diff --git a/backup/agents/models.ts b/backup/agents/models.ts new file mode 100644 index 0000000..16c7193 --- /dev/null +++ b/backup/agents/models.ts @@ -0,0 +1,36 @@ +import { ChatOpenAI, ChatOpenAIFields } from "@langchain/openai"; + +export const openAI = (config: ChatOpenAIFields = {}) => { + return new ChatOpenAI({ + modelName: "gpt-4.1", + temperature: 1, + configuration: { + apiKey: process.env.AI_OPENAI_KEY, + baseURL: process.env.AI_OPENAI_URL, + }, + ...config, + }); +}; + +export const doubao = (config: ChatOpenAIFields = {}) => { + return new ChatOpenAI({ + model: "doubao-seed-1-6-flash-250828", + temperature: 1, + configuration: { + apiKey: process.env.AI_TIKTOK_KEY, + baseURL: process.env.AI_TIKTOK_URL, + }, + ...config, + }); +}; + +export const deepseek = (config: ChatOpenAIFields = {}) => + new ChatOpenAI({ + model: "DeepSeek-V3.2", + temperature: 1, + configuration: { + apiKey: process.env.AI_DEEPSEEK_KEY, + baseURL: process.env.AI_DEEPSEEK_URL, + }, + ...config, + }); diff --git a/backup/agents/outlineScript/index.ts b/backup/agents/outlineScript/index.ts new file mode 100644 index 0000000..304c7c0 --- /dev/null +++ b/backup/agents/outlineScript/index.ts @@ -0,0 +1,769 @@ +// @/agents/outlineScript.ts +import u from "@/utils"; +import { createAgent } from "langchain"; +import { EventEmitter } from "events"; +import { openAI } from "@/agents/models"; +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import type { DB } from "@/types/database"; +// ==================== 类型定义 ==================== + +type AgentType = "AI1" | "AI2" | "director"; +type AssetType = "角色" | "道具" | "场景"; +type RefreshEvent = "storyline" | "outline" | "assets"; + +interface AssetItem { + name: string; + description: string; +} + +interface EpisodeData { + episodeIndex: number; + title: string; + chapterRange: number[]; + scenes: AssetItem[]; // 按 outline 出场顺序排列 + characters: AssetItem[]; // 按 outline 出场顺序排列 + props: AssetItem[]; // 按 outline 出场顺序排列 + coreConflict: string; + outline: string; // 最高优先级,剧本生成的唯一权威 + openingHook: string; // outline 第一句话的视觉化,开篇第一个镜头 + keyEvents: string[]; // 4个元素:[起, 承, 转, 合],严格按 outline 顺序 + emotionalCurve: string; // 对应 keyEvents 各阶段 + visualHighlights: string[]; // 按 outline 顺序排列的标志性镜头 + endingHook: string; // outline 之后的悬念延伸 + classicQuotes: string[]; +} + +// ==================== Schema 定义 ==================== + +const sceneItemSchema = z.object({ + name: z.string().describe("场景名称,如'五星酒店宴会厅'、'老旧出租屋'"), + description: z.string().describe("环境描写:空间结构、光线氛围、装饰陈设、环境细节"), +}); + +const characterItemSchema = z.object({ + name: z.string().describe("角色姓名(必须是具体人名,禁止'众人'、'群众'等集合描述)"), + description: z.string().describe("人设样貌:年龄体态、五官特征、发型妆容、服装配饰、气质神态"), +}); + +const propItemSchema = z.object({ + name: z.string().describe("道具名称"), + description: z.string().describe("样式描写:材质质感、颜色图案、形状尺寸、磨损痕迹、特殊标记"), +}); + +const episodeSchema = z.object({ + episodeIndex: z.number().describe("集数索引,从1开始递增"), + title: z.string().describe("8字内标题,疑问/感叹句,含情绪爆点"), + chapterRange: z.array(z.number()).describe("关联章节号数组"), + scenes: z.array(sceneItemSchema).describe("场景列表,按 outline 出场顺序排列"), + characters: z.array(characterItemSchema).describe("角色列表,按 outline 出场顺序排列,必须是独立个体"), + props: z.array(propItemSchema).describe("道具列表,按 outline 出场顺序排列,至少3个"), + coreConflict: z.string().describe("核心矛盾:A想要X vs B阻碍X"), + outline: z.string().describe("100-300字剧情主干,最高优先级,剧本生成的唯一权威,按时间顺序完整叙述"), + openingHook: z.string().describe("开场镜头:outline 第一句话的视觉化,必须作为剧本第一个镜头"), + keyEvents: z.array(z.string()).length(4).describe("4个元素的数组:[起, 承, 转, 合],严格按 outline 顺序从中提取"), + emotionalCurve: z.string().describe("情绪曲线,如:2(压抑)→5(反抗)→9(爆发)→3(余波),对应 keyEvents 各阶段"), + visualHighlights: z.array(z.string()).describe("3-5个标志性镜头,按 outline 叙事顺序排列"), + endingHook: z.string().describe("结尾悬念:outline 之后的延伸,勾引下集"), + classicQuotes: z.array(z.string()).describe("1-2句金句,每句≤15字,必须从原文提取"), +}); + +// ==================== 常量配置 ==================== + +// ==================== 主类 ==================== + +export default class OutlineScript { + private readonly projectId: number; + readonly emitter = new EventEmitter(); + history: Array<[string, string]> = []; + novelChapters: DB["t_novel"][] = []; + + modelName = "gpt-4.1"; + apiKey = ""; + baseURL = ""; + + constructor(projectId: number) { + this.projectId = projectId; + } + + // ==================== 公共方法 ==================== + + get events() { + return this.emitter; + } + + setNovel(chapters: DB["t_novel"][]) { + this.novelChapters = chapters; + } + + // ==================== 私有工具方法 ==================== + + private emit(event: string, data?: any) { + this.emitter.emit(event, data); + } + + private refresh(type: RefreshEvent) { + this.emit("refresh", type); + } + + private log(action: string, detail?: string) { + const msg = detail ? `${action}: ${detail}` : action; + console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`); + } + + private safeParseJson(str: string, fallback: T): T { + try { + return JSON.parse(str); + } catch { + return fallback; + } + } + + private uniqueByName(items: T[]): T[] { + return Array.from(new Map(items.map((item) => [item.name, item])).values()); + } + + // ==================== 数据库操作 ==================== + + private async getProjectInfo(): Promise { + return u.db("t_project").where({ id: this.projectId }).first(); + } + + private async getNovelInfo(asString = false): Promise { + const info = await this.getProjectInfo(); + if (!info) return asString ? "未查询到项目信息" : null; + + if (asString) { + const fields = [ + `小说名称: ${info.name}`, + `小说简介: ${info.intro}`, + `小说类型: ${info.type}`, + `目标短剧类型: ${info.artStyle}`, + `短剧画幅: ${info.videoRatio}`, + ]; + return fields.join("\n"); + } + return info; + } + + // ==================== 故事线操作 ==================== + + private async findStoryline() { + return u.db("t_storyline").where({ projectId: this.projectId }).first(); + } + + private async upsertStorylineContent(content: string) { + const existing = await this.findStoryline(); + if (existing) { + await u.db("t_storyline").where({ projectId: this.projectId }).update({ content }); + } else { + await u.db("t_storyline").insert({ projectId: this.projectId, content }); + } + this.refresh("storyline"); + } + + private async deleteStorylineContent() { + const deleted = await u.db("t_storyline").where({ projectId: this.projectId }).del(); + this.refresh("storyline"); + return deleted; + } + + // ==================== 大纲操作 ==================== + + private async findOutlines() { + return u.db("t_outline").where({ projectId: this.projectId }).orderBy("episode", "asc"); + } + + private async findOutlineById(id: number) { + return u.db("t_outline").where({ id, projectId: this.projectId }).first(); + } + + private async getMaxEpisode(): Promise { + const result: any = await u.db("t_outline").where({ projectId: this.projectId }).max("episode as max").first(); + return result?.max ?? 0; + } + + private async clearOutlinesAndScripts() { + const outlines = await u.db("t_outline").select("id").where({ projectId: this.projectId }); + if (outlines.length === 0) return 0; + + const outlineIds = outlines.map((o) => o.id); + await u.db("t_script").whereIn("outlineId", outlineIds).del(); + await u.db("t_outline").where({ projectId: this.projectId }).del(); + + return outlines.length; + } + + private async insertOutlines(episodes: EpisodeData[], startEpisode: number) { + const insertList = episodes.map((ep, idx) => ({ + projectId: this.projectId, + data: JSON.stringify({ ...ep, episodeIndex: startEpisode + idx }), + episode: startEpisode + idx, + })); + + await u.db("t_outline").insert(insertList); + return insertList.length; + } + + private async createEmptyScripts(outlineIds: Array<{ id: number; data: string }>) { + const scripts = outlineIds.map((item) => { + const data = this.safeParseJson>(item.data, {}); + return { + name: `第${data.episodeIndex ?? ""}集`, + content: "", + projectId: this.projectId, + outlineId: item.id, + }; + }); + + if (scripts.length > 0) { + await u.db("t_script").insert(scripts); + } + return scripts.length; + } + + private async saveOutlineData(episodes: EpisodeData[], overwrite: boolean, startEpisode?: number) { + if (overwrite) { + const cleared = await this.clearOutlinesAndScripts(); + if (cleared > 0) { + this.log("清理旧数据", `删除了 ${cleared} 条大纲及关联剧本`); + } + } + + const actualStart = overwrite ? 1 : startEpisode ?? (await this.getMaxEpisode()) + 1; + const insertedCount = await this.insertOutlines(episodes, actualStart); + + const newOutlines = await u + .db("t_outline") + .select("id", "data") + .where({ projectId: this.projectId }) + .orderBy("episode", "desc") + .limit(insertedCount); + + const scriptCount = await this.createEmptyScripts(newOutlines as Array<{ id: number; data: string }>); + + this.refresh("outline"); + return { insertedCount, scriptCount }; + } + + private async updateOutlineData(id: number, data: EpisodeData) { + const existing = await this.findOutlineById(id); + if (!existing) return false; + + await u + .db("t_outline") + .where({ id }) + .update({ data: JSON.stringify(data) }); + this.refresh("outline"); + return true; + } + + private async deleteOutlineData(ids: number[]) { + const results = await Promise.allSettled(ids.map((id) => u.deleteOutline(id, this.projectId))); + this.refresh("outline"); + return results; + } + + private formatOutlineDetail(ep: any): string { + const formatList = (items: any[], formatter: (item: any) => string) => + items?.map((item, i) => ` ${i + 1}. ${formatter(item)}`).join("\n") || " 无"; + + // keyEvents 按顺序显示:起、承、转、合 + const keyEventsLabels = ["起", "承", "转", "合"]; + const formatKeyEvents = (events: string[]) => events?.map((e, i) => ` 【${keyEventsLabels[i] || i + 1}】${e}`).join("\n") || " 无"; + + return ` +大纲ID: ${ep.id} +第 ${ep.episodeIndex} 集: ${ep.title || ""} +${"=".repeat(50)} +章节范围: ${ep.chapterRange?.join(", ") || ""} +核心矛盾: ${ep.coreConflict || ""} + +【剧情主干】(最高优先级,剧本生成的唯一权威): +${ep.outline || "无"} + +【开场镜头】(必须作为剧本第一个镜头): +${ep.openingHook || "无"} + +【剧情节点】(严格按顺序:起→承→转→合): +${formatKeyEvents(ep.keyEvents)} + +情绪曲线: ${ep.emotionalCurve || ""} + +【视觉重点】(按剧情主干顺序排列): +${formatList(ep.visualHighlights, (v) => v)} + +【结尾悬念】: +${ep.endingHook || "无"} + +【经典台词】: +${formatList(ep.classicQuotes, (q) => q)} + +角色(按出场顺序): ${ep.characters?.map((c: AssetItem) => `${c.name}(${c.description})`).join("; ") || "无"} +场景(按出场顺序): ${ep.scenes?.map((s: AssetItem) => `${s.name}(${s.description})`).join("; ") || "无"} +道具(按出场顺序): ${ep.props?.map((p: AssetItem) => `${p.name}(${p.description})`).join("; ") || "无"}`; + } + + private async getOutlineText(simplified: boolean): Promise { + const records = await this.findOutlines(); + + if (!records.length) return "当前项目暂无大纲"; + + const episodes = records.map((r) => ({ + id: r.id, + episode: r.episode, + ...this.safeParseJson>(r.data ?? "{}", {}), + })); + + if (simplified) { + const list = episodes.map((ep) => `第 ${ep.episodeIndex ?? ep.episode} 集 (id=${ep.id})`).join("\n"); + return `项目大纲 (共 ${episodes.length} 集):\n${list}`; + } + + const details = episodes.map((ep) => this.formatOutlineDetail(ep)).join("\n"); + return `项目大纲 (共 ${episodes.length} 集)\n\n${details}`; + } + + // ==================== 资产操作 ==================== + + private async findAssetByTypeAndName(type: AssetType, name: string) { + return u.db("t_assets").where({ projectId: this.projectId, type, name }).first(); + } + + private async upsertAsset(type: AssetType, item: AssetItem): Promise<"inserted" | "updated" | "skipped"> { + const existing = await this.findAssetByTypeAndName(type, item.name); + + if (!existing) { + await u.db("t_assets").insert({ + projectId: this.projectId, + type, + name: item.name, + intro: item.description, + prompt: item.description, + }); + return "inserted"; + } + + if (existing.intro !== item.description) { + await u.db("t_assets").where({ id: existing.id }).update({ + intro: item.description, + prompt: item.description, + }); + return "updated"; + } + + return "skipped"; + } + + private extractAssetsFromOutlines(outlines: Array<{ data?: string | null | undefined }>): { + characters: AssetItem[]; + props: AssetItem[]; + scenes: AssetItem[]; + } { + const result = { characters: [] as AssetItem[], props: [] as AssetItem[], scenes: [] as AssetItem[] }; + + for (const outline of outlines) { + const data = this.safeParseJson>(outline.data ?? "{}", {}); + if (data.characters) result.characters.push(...data.characters); + if (data.props) result.props.push(...data.props); + if (data.scenes) result.scenes.push(...data.scenes); + } + + return { + characters: this.uniqueByName(result.characters), + props: this.uniqueByName(result.props), + scenes: this.uniqueByName(result.scenes), + }; + } + + private async generateAssetsFromOutlines() { + const outlines = await u.db("t_outline").select("data").where({ projectId: this.projectId }); + + if (!outlines.length) return { inserted: 0, updated: 0, skipped: 0 }; + + const { characters, props, scenes } = this.extractAssetsFromOutlines(outlines); + + // 只做新增和更新,不做删除 + const stats = { inserted: 0, updated: 0, skipped: 0 }; + + const processItems = async (items: AssetItem[], type: AssetType) => { + for (const item of items) { + const result = await this.upsertAsset(type, item); + stats[result]++; + } + }; + + await processItems(characters, "角色"); + await processItems(props, "道具"); + await processItems(scenes, "场景"); + + this.refresh("assets"); + return { ...stats }; + } + + // ==================== Tool 定义:故事线 ==================== + + getStoryline = tool( + async () => { + this.log("获取故事线"); + const storyline = await this.findStoryline(); + return storyline?.content ?? "当前项目暂无故事线"; + }, + { + name: "getStoryline", + description: "获取当前项目的故事线内容", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + saveStoryline = tool( + async ({ content }) => { + this.log("保存故事线"); + await this.upsertStorylineContent(content); + return "故事线保存成功"; + }, + { + name: "saveStoryline", + description: "保存或更新当前项目的故事线,会覆盖已有内容", + schema: z.object({ + content: z.string().describe("故事线完整内容"), + }), + verboseParsingErrors: true, + }, + ); + + deleteStoryline = tool( + async () => { + this.log("删除故事线"); + const deleted = await this.deleteStorylineContent(); + return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线"; + }, + { + name: "deleteStoryline", + description: "删除当前项目的故事线", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== Tool 定义:大纲 ==================== + + getOutline = tool( + async ({ simplified = false }) => { + this.log("获取大纲", `简化模式: ${simplified}`); + return this.getOutlineText(simplified); + }, + { + name: "getOutline", + description: "获取项目大纲。simplified=true返回简化列表,false返回完整内容", + schema: z.object({ + simplified: z.boolean().default(false).describe("是否返回简化版本"), + }), + verboseParsingErrors: true, + }, + ); + + saveOutline = tool( + async ({ episodes, overwrite = true, startEpisode }) => { + this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`); + const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode); + return `大纲保存成功:插入 ${insertedCount} 集大纲,创建 ${scriptCount} 个剧本记录`; + }, + { + name: "saveOutline", + description: "保存大纲数据。overwrite=true会清空现有大纲后写入,false则追加到末尾", + schema: z.object({ + episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"), + overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"), + startEpisode: z.number().optional().describe("追加模式下的起始集数(不填则自动递增)"), + }), + verboseParsingErrors: true, + }, + ); + + updateOutline = tool( + async ({ id, data }) => { + this.log("更新大纲", `ID: ${id}`); + const success = await this.updateOutlineData(id, data as EpisodeData); + return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`; + }, + { + name: "updateOutline", + description: "更新指定ID的单集大纲内容", + schema: z.object({ + id: z.number().describe("大纲ID"), + data: episodeSchema.describe("更新后的大纲数据"), + }), + verboseParsingErrors: true, + }, + ); + + deleteOutline = tool( + async ({ ids }) => { + this.log("删除大纲", `IDs: ${ids.join(", ")}`); + const results = await this.deleteOutlineData(ids); + const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", "); + return `删除结果: ${summary}`; + }, + { + name: "deleteOutline", + description: "根据大纲ID删除指定大纲及关联数据", + schema: z.object({ + ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"), + }), + verboseParsingErrors: true, + }, + ); + + // ==================== Tool 定义:章节 ==================== + + getChapter = tool( + async ({ chapterNumbers }) => { + this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`); + + const results = await Promise.all( + chapterNumbers.map(async (num) => { + const chapter = await u + .db("t_novel") + .where({ projectId: this.projectId, chapterIndex: num }) + .select("chapterData", "chapterIndex", "chapter") + .first(); + + if (chapter) { + return `\n【第${chapter.chapterIndex}章 ${chapter.chapter || ""}】\n${chapter.chapterData}`; + } + return `\n【第${num}章】未找到`; + }), + ); + + return results.join("\n\n---\n"); + }, + { + name: "getChapter", + description: "根据章节编号获取小说章节的完整原文内容,支持批量获取", + schema: z.object({ + chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"), + }), + verboseParsingErrors: true, + }, + ); + + // ==================== Tool 定义:资产 ==================== + + generateAssets = tool( + async () => { + this.log("生成资产"); + const stats = await this.generateAssetsFromOutlines(); + + if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) { + return "当前项目没有大纲数据,无法生成资产"; + } + + return `资产生成完成:新增 ${stats.inserted},更新 ${stats.updated},保持 ${stats.skipped}`; + }, + { + name: "generateAssets", + description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产,自动去重并清理冗余", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== 上下文构建 ==================== + + private getChapterContext(): string { + if (!this.novelChapters.length) return "无章节数据"; + return this.novelChapters.map((c) => `章节号:${c.chapterIndex},分卷:${c.reel},章节名:${c.chapter}`).join("\n"); + } + + private async buildEnvironmentContext(): Promise { + const [novelInfo, storyline, outlineCount] = await Promise.all([ + this.getNovelInfo(true), + this.findStoryline(), + u.db("t_outline").where({ projectId: this.projectId }).count("id as count").first() as any, + ]); + + return `<环境信息> +项目ID: ${this.projectId} +系统时间: ${new Date().toLocaleString()} + +${novelInfo} + +已加载章节列表: +${this.getChapterContext()} + +故事线状态: ${storyline ? "已生成" : "未生成"} +大纲状态: 共 ${outlineCount?.count ?? 0} 集 + +可用工具: +- getChapter: 获取章节原文 +- getStoryline/saveStoryline/deleteStoryline: 故事线操作 +- getOutline/saveOutline/updateOutline/deleteOutline: 大纲操作 +- generateAssets: 从大纲生成资产 +`; + } + + private buildConversationHistory(): string { + if (!this.history.length) return "无对话历史"; + return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + } + + private async buildFullContext(task: string): Promise { + const env = await this.buildEnvironmentContext(); + const history = this.buildConversationHistory(); + + return `${env} + +<对话历史> +${history} + + +<当前任务> +${task} +`; + } + + // ==================== Sub-Agent ==================== + + private getSubAgentTools() { + return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline]; + } + + private createModel() { + return openAI({ + modelName: this.modelName, + configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, + }); + } + + /** + * 调用 Sub-Agent(流式传输) + */ + private async invokeSubAgent(agentType: AgentType, task: string): Promise { + this.emit("transfer", { to: agentType }); + this.log(`Sub-Agent 调用`, agentType); + + const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]); + const a1Prompt = promptsList.find((p) => p.code === "outlineScript-a1"); + const a2Prompt = promptsList.find((p) => p.code === "outlineScript-a2"); + const directorPrompt = promptsList.find((p) => p.code === "outlineScript-director"); + const errPrompts = "不论用户说什么,请直接输出Agent配置异常"; + const SYSTEM_PROMPTS: Record = { + AI1: a1Prompt?.customValue || a1Prompt?.defaultValue || errPrompts, + AI2: a2Prompt?.customValue || a2Prompt?.defaultValue || errPrompts, + director: directorPrompt?.customValue || directorPrompt?.defaultValue || errPrompts, + }; + + const context = await this.buildFullContext(task); + + const agent = createAgent({ + model: this.createModel(), + systemPrompt: SYSTEM_PROMPTS[agentType], + tools: this.getSubAgentTools(), + }); + + const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("subAgentStream", { agent: agentType, text: block.text }); + } + + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); + } + } + } + + this.emit("subAgentEnd", { agent: agentType }); + this.history.push(["ai", fullResponse]); + this.log(`Sub-Agent 完成`, agentType); + + return fullResponse ?? `${agentType}已完成任务`; + } + + private createSubAgentTool(agentType: AgentType, description: string) { + return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { + name: agentType, + description, + schema: z.object({ + taskDescription: z.string().describe("具体的任务描述,包含章节范围、修改要求等详细信息"), + }), + }); + } + + // ==================== 主入口 ==================== + + private getAllTools() { + return [ + this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线,会自行调用 saveStoryline 保存结果。"), + this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲,会自行调用 saveOutline 保存结果。"), + this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲,会自行调用 updateOutline 或 saveStoryline 进行修改。"), + this.getChapter, + this.getStoryline, + this.saveStoryline, + this.deleteStoryline, + this.getOutline, + this.saveOutline, + this.updateOutline, + this.deleteOutline, + this.generateAssets, + ]; + } + + async call(msg: string): Promise { + this.history.push(["user", msg]); + + const envContext = await this.buildEnvironmentContext(); + + const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first(); + + const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么,请直接输出Agent配置异常"; + + const mainAgent = createAgent({ + model: this.createModel(), + tools: this.getAllTools(), + systemPrompt: `${envContext}\n${mainPrompts}`, + }); + const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("data", block.text); + } + + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); + } + } + } + + this.history.push(["assistant", fullResponse]); + this.emit("response", fullResponse); + + return fullResponse; + } +} diff --git a/backup/agents/storyboard/generateImagePromptsTool.ts b/backup/agents/storyboard/generateImagePromptsTool.ts new file mode 100644 index 0000000..4057221 --- /dev/null +++ b/backup/agents/storyboard/generateImagePromptsTool.ts @@ -0,0 +1,130 @@ +import u from "@/utils"; + +type AspectRatio = "16:9" | "9:16" | "21:9" | "1:1" | "4:3" | "3:4" | "3:2" | "2:3"; + +interface GridLayoutResult { + cols: number; + rows: number; + totalCells: number; + placeholderCount: number; +} + +interface GridPromptOptions { + prompts: string[]; + style: string; + aspectRatio: AspectRatio; + assetsName: { name: string; intro: string }[]; +} + +interface GridPromptResult { + prompt: string; + gridLayout: GridLayoutResult; +} + +/** + * 根据prompts数量计算宫格布局 + */ +function calculateGridLayout(count: number): GridLayoutResult { + let cols: number; + let rows: number; + if (count <= 0) { + cols = 1; + rows = 1; + } else if (count === 1) { + cols = 1; + rows = 1; + } else if (count === 2) { + cols = 2; + rows = 1; + } else if (count === 3) { + cols = 3; + rows = 1; + } else if (count === 4) { + cols = 2; + rows = 2; + } else if (count <= 9) { + cols = 3; + rows = 3; + } else { + cols = 3; + rows = Math.ceil(count / 3); + } + const totalCells = cols * rows; + const placeholderCount = totalCells - count; + return { cols, rows, totalCells, placeholderCount }; +} + +/** + * 获取宽高比描述 + */ +function getAspectRatioDescription(aspectRatio: AspectRatio): string { + const descriptions: Record = { + "16:9": "电影宽银幕", + "9:16": "竖屏短剧", + "21:9": "超宽银幕史诗感", + "1:1": "方形构图", + "4:3": "经典银幕", + "3:4": "竖版经典", + "3:2": "摄影标准", + "2:3": "竖版摄影", + }; + return descriptions[aspectRatio] || "标准比例"; +} + +/** + * 生成电影级宫格分镜提示词 + */ +async function generateGridPrompt(options: GridPromptOptions): Promise { + const { prompts, style, aspectRatio, assetsName } = options; + const layout = calculateGridLayout(prompts.length); + const aspectRatioDesc = getAspectRatioDescription(aspectRatio); + + // 构建宫格位置描述 + const gridPositions: string[] = []; + for (let i = 0; i < layout.totalCells; i++) { + const row = Math.floor(i / layout.cols) + 1; + const col = (i % layout.cols) + 1; + if (i < prompts.length) { + gridPositions.push(`[第${row}行第${col}列]: ${prompts[i]}`); + } else { + gridPositions.push(`[第${row}行第${col}列]: 纯黑图`); + } + } + + // 构建资产说明 + const assetsSection = + assetsName.length > 0 + ? `\n【可用资产】\n${assetsName.map((a) => `- ${a.name}:${a.intro}`).join("\n")}\n\n⚠️ 必须使用完整资产名称,禁止简称或代词。` + : ""; + + const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first(); + + const mainPrompts = promptsData?.customValue || promptsData?.defaultValue; + const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`; + + if (!mainPrompts) return { prompt: errData, gridLayout: layout }; + + const chatModel = await u.ai.text({}); + + const result = await chatModel!.invoke({ + messages: [ + { + role: "system", + content: mainPrompts, + }, + { + role: "user", + content: `请优化以下分镜提示词:\n\n【布局】${layout.cols}列×${layout.rows}行=${ + layout.totalCells + }格\n【比例】${aspectRatio}(${aspectRatioDesc})\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`, + }, + ], + }); + + return { + prompt: result?.text ?? errData, + gridLayout: layout, + }; +} + +export default generateGridPrompt; diff --git a/backup/agents/storyboard/generateImageTool.ts b/backup/agents/storyboard/generateImageTool.ts new file mode 100644 index 0000000..151c830 --- /dev/null +++ b/backup/agents/storyboard/generateImageTool.ts @@ -0,0 +1,334 @@ +import generateImagePromptsTool from "@/agents/storyboard/generateImagePromptsTool"; +import u from "@/utils"; +import sharp from "sharp"; +import { z } from "zod"; + +interface AssetItem { + name: string; + description: string; +} + +interface EpisodeData { + episodeIndex: number; + title: string; + chapterRange: number[]; + scenes: AssetItem[]; + characters: AssetItem[]; + props: AssetItem[]; + coreConflict: string; + openingHook: string; + outline: string; + keyEvents: string[]; + emotionalCurve: string; + visualHighlights: string[]; + endingHook: string; + classicQuotes: string[]; +} + +interface ImageInfo { + name: string; + type: string; + filePath: string; +} + +interface ResourceItem { + name: string; + intro: string; +} + +// 资产过滤响应的 schema +const filteredAssetsSchema = z.object({ + relevantAssets: z + .array( + z.object({ + name: z.string().describe("资产名称"), + reason: z.string().describe("选择该资产的原因"), + }), + ) + .describe("与分镜内容相关的资产列表"), +}); + +// 压缩图片直到不超过指定大小 +async function compressImage(buffer: Buffer, maxSizeBytes: number = 3 * 1024 * 1024): Promise { + if (buffer.length <= maxSizeBytes) { + return buffer; + } + let quality = 90; + let compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer(); + while (compressedBuffer.length > maxSizeBytes && quality > 10) { + quality -= 10; + compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer(); + } + if (compressedBuffer.length > maxSizeBytes) { + const metadata = await sharp(buffer).metadata(); + let scale = 0.9; + while (compressedBuffer.length > maxSizeBytes && scale > 0.1) { + const newWidth = Math.round((metadata.width || 1000) * scale); + const newHeight = Math.round((metadata.height || 1000) * scale); + compressedBuffer = await sharp(buffer) + .resize(newWidth, newHeight, { fit: "inside" }) + .jpeg({ quality: Math.max(quality, 30) }) + .toBuffer(); + scale -= 0.1; + } + } + return compressedBuffer; +} + +// 拼接多张图片为一张 +async function mergeImages(imagePaths: string[]): Promise { + const imageBuffers = await Promise.all(imagePaths.map((path) => u.oss.getFile(path))); + const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata())); + const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0)); + const resizedImages = await Promise.all( + imageBuffers.map(async (buffer, index) => { + const metadata = imageMetadatas[index]; + const aspectRatio = (metadata.width || 1) / (metadata.height || 1); + const newWidth = Math.round(maxHeight * aspectRatio); + return { + buffer: await sharp(buffer).resize(newWidth, maxHeight, { fit: "cover" }).toBuffer(), + width: newWidth, + }; + }), + ); + let currentX = 0; + const compositeInputs = resizedImages.map(({ buffer, width }) => { + const input = { + input: buffer, + left: currentX, + top: 0, + }; + currentX += width; + return input; + }); + const mergedImage = await sharp({ + create: { + width: currentX, + height: maxHeight, + channels: 4, + background: { r: 255, g: 255, b: 255, alpha: 1 }, + }, + }) + .composite(compositeInputs) + .jpeg({ quality: 90 }) + .toBuffer(); + return compressImage(mergedImage); +} + +// 进一步压缩单张图片到指定大小 +async function compressToSize(buffer: Buffer, targetSize: number): Promise { + if (buffer.length <= targetSize) { + return buffer; + } + + const metadata = await sharp(buffer).metadata(); + let quality = 80; + let scale = 1.0; + let compressedBuffer = buffer; + + // 先尝试降低质量 + while (compressedBuffer.length > targetSize && quality > 10) { + compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer(); + quality -= 10; + } + + // 如果还是太大,缩小尺寸 + while (compressedBuffer.length > targetSize && scale > 0.2) { + scale -= 0.1; + const newWidth = Math.round((metadata.width || 1000) * scale); + const newHeight = Math.round((metadata.height || 1000) * scale); + compressedBuffer = await sharp(buffer) + .resize(newWidth, newHeight, { fit: "inside" }) + .jpeg({ quality: Math.max(quality, 20) }) + .toBuffer(); + } + + return compressedBuffer; +} + +// 确保图片列表总大小不超过指定限制 +async function ensureTotalSizeLimit(buffers: Buffer[], maxTotalBytes: number = 10 * 1024 * 1024): Promise { + let totalSize = buffers.reduce((sum, buf) => sum + buf.length, 0); + + if (totalSize <= maxTotalBytes) { + return buffers; + } + + // 计算每张图片的平均目标大小 + const avgTargetSize = Math.floor(maxTotalBytes / buffers.length); + + // 按大小降序排列,优先压缩大图片 + const indexedBuffers = buffers.map((buf, idx) => ({ buf, idx, size: buf.length })); + indexedBuffers.sort((a, b) => b.size - a.size); + + const result = [...buffers]; + + for (const item of indexedBuffers) { + totalSize = result.reduce((sum, buf) => sum + buf.length, 0); + if (totalSize <= maxTotalBytes) { + break; + } + + // 计算这张图片需要压缩到的目标大小 + const excessSize = totalSize - maxTotalBytes; + const targetSize = Math.max(item.buf.length - excessSize, avgTargetSize, 100 * 1024); // 最小100KB + + if (item.buf.length > targetSize) { + result[item.idx] = await compressToSize(item.buf, targetSize); + } + } + + return result; +} + +// 处理图片列表,确保不超过10张且每张不超过3MB,总大小不超过10MB +async function processImages(images: ImageInfo[]): Promise { + const maxImages = 10; + let processedBuffers: Buffer[]; + + if (images.length <= maxImages) { + const buffers = await Promise.all(images.map((img) => u.oss.getFile(img.filePath))); + processedBuffers = await Promise.all(buffers.map((buffer) => compressImage(buffer))); + } else { + const mergeStartIndex = maxImages - 1; + const firstBuffers = await Promise.all(images.slice(0, mergeStartIndex).map((img) => u.oss.getFile(img.filePath))); + const compressedFirstImages = await Promise.all(firstBuffers.map((buffer) => compressImage(buffer))); + const imagesToMergeList = images.slice(mergeStartIndex).map((img) => img.filePath); + const mergedImage = await mergeImages(imagesToMergeList); + processedBuffers = [...compressedFirstImages, mergedImage]; + } + + // 确保总大小不超过10MB + return ensureTotalSizeLimit(processedBuffers); +} + +// 使用 AI 过滤与分镜相关的资产 +async function filterRelevantAssets(prompts: string[], allResources: ResourceItem[], availableImages: ImageInfo[]): Promise { + if (allResources.length === 0 || availableImages.length === 0) { + return availableImages; + } + + const availableNames = new Set(availableImages.map((img) => img.name)); + const availableResources = allResources.filter((r) => availableNames.has(r.name)); + + if (availableResources.length === 0) { + return availableImages; + } + + const chatModel = await u.ai.text({}); + const result = await chatModel!.invoke({ + messages: [ + { + role: "user", + content: `请分析以下分镜描述,从可用资产中筛选出与分镜内容直接相关的资产。 + +分镜描述: +${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")} + +可用资产列表: +${availableResources.map((r) => `- ${r.name}:${r.intro}`).join("\n")} + +请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, + }, + ], + responseFormat: { + type: "json_schema", + jsonSchema: { + name: "filteredAssets", + strict: true, + schema: z.toJSONSchema(filteredAssetsSchema), + }, + }, + }); + + const data = result?.json as z.infer; + + if (!data?.relevantAssets || data.relevantAssets.length === 0) { + return availableImages; + } + + const relevantNames = new Set(data.relevantAssets.map((a) => a.name)); + const filteredImages = availableImages.filter((img) => relevantNames.has(img.name)); + + return filteredImages.length > 0 ? filteredImages : availableImages; +} + +// 构建资产映射提示词 +function buildResourcesMapPrompts(images: ImageInfo[]): string { + if (images.length === 0) return ""; + + const mapping = images.map((item, index) => { + if (index < 9) { + return `${item.name}=图片${index + 1}`; + } else { + return `${item.name}=图10-${index - 8}`; + } + }); + + return `其中人物、场景、道具参考对照关系如下:${mapping.join(", ")}。`; +} + +export default async (cells: { prompt: string }[], scriptId: number, projectId: number) => { + const scriptData = await u.db("t_script").where({ id: scriptId, projectId }).first(); + const projectInfo = await u.db("t_project").where({ id: projectId }).first(); + + const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId }).first(); + const outline: EpisodeData | null = row?.data ? JSON.parse(row.data) : null; + + const resources: ResourceItem[] = outline + ? (["characters", "props", "scenes"] as const).flatMap((k) => outline[k]?.map((i) => ({ name: i.name, intro: i.description })) ?? []) + : []; + + const resourceNames = resources.map((r) => r.name); + const imagesRaw = await u.db("t_assets").whereIn("name", resourceNames).andWhere({ projectId }).select("name", "type", "filePath"); + + const allImages = imagesRaw + .sort((a, b) => { + const order = ["角色", "场景", "道具"]; + return order.indexOf(a.type!) - order.indexOf(b.type!); + }) + .filter((img) => img.filePath) as ImageInfo[]; + + if (allImages.length === 0) { + throw new Error("未找到可用的图片资源"); + } + + const cellPrompts = cells.map((c) => c.prompt); + + // 使用 AI 过滤相关资产 + const filteredImages = await filterRelevantAssets(cellPrompts, resources, allImages); + + const resourcesMapPrompts = buildResourcesMapPrompts(filteredImages); + console.log("====润色前:", cellPrompts); + const promptsData = await generateImagePromptsTool({ + prompts: cellPrompts, + style: `类型:${projectInfo?.type!},风格:${projectInfo?.artStyle!}`, + aspectRatio: projectInfo?.videoRatio! as any, + assetsName: resources, + }); + + // const prompts = `请生成${promptsData.gridLayout.totalCells}格,${promptsData.gridLayout.cols}列×${promptsData.gridLayout.rows}行宫格图。 + + // ${promptsData.prompt} + + // 注意:请严格按照提示词内容生成图片,确保人物样貌、艺术风格、色调光影一致。 + // `; + const prompts = promptsData.prompt; + console.log("====润色后:", prompts); + + const processedImages = await processImages(filteredImages); + + const contentStr = await u.ai.generateImage({ + systemPrompt: resourcesMapPrompts, + prompt: prompts, + size: "4K", + aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9", + imageBase64: processedImages.map((buf) => buf.toString("base64")), + }); + + const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/); + const base64Str = match?.[1] ?? contentStr; + const buffer = Buffer.from(base64Str, "base64"); + + return buffer; +}; diff --git a/backup/agents/storyboard/imageSplitting.ts b/backup/agents/storyboard/imageSplitting.ts new file mode 100644 index 0000000..5488194 --- /dev/null +++ b/backup/agents/storyboard/imageSplitting.ts @@ -0,0 +1,94 @@ +import sharp from "sharp"; + +interface GridLayoutResult { + cols: number; + rows: number; + totalCells: number; + placeholderCount: number; +} + +/** + * 计算宫格布局 + * 1张: 1x1 + * 2张: 2x1 + * 3张: 3x1 + * 4张: 2x2 + * 5-9张: 3x3 + * 10-12张: 3x4 + * 13-15张: 3x5 + * ...以此类推(3列,行数递增) + */ +function calculateGridLayout(count: number): GridLayoutResult { + let cols: number; + let rows: number; + if (count <= 0) { + cols = 1; + rows = 1; + } else if (count === 1) { + cols = 1; + rows = 1; + } else if (count === 2) { + cols = 2; + rows = 1; + } else if (count === 3) { + cols = 3; + rows = 1; + } else if (count === 4) { + cols = 2; + rows = 2; + } else if (count <= 9) { + // 5-9格统一用3x3 + cols = 3; + rows = 3; + } else { + cols = 3; + rows = Math.ceil(count / 3); + } + const totalCells = cols * rows; + const placeholderCount = totalCells - count; + return { cols, rows, totalCells, placeholderCount }; +} + +/** + * 分割宫格图片 + * @param image - 输入的宫格图片 Buffer + * @param length - 实际需要的图片数量(不包含占位图) + * @returns 分割后的单张图片 Buffer 数组 + */ +export default async (image: Buffer, length: number): Promise => { + const metadata = await sharp(image).metadata(); + const { width: totalWidth, height: totalHeight } = metadata; + + if (!totalWidth || !totalHeight) { + throw new Error("无法获取图片尺寸"); + } + + const { cols, rows } = calculateGridLayout(length); + + const cellWidth = Math.floor(totalWidth / cols); + const cellHeight = Math.floor(totalHeight / rows); + + const buffers: Buffer[] = []; + + for (let i = 0; i < length; i++) { + const row = Math.floor(i / cols); + const col = i % cols; + + const left = col * cellWidth; + const top = row * cellHeight; + + const cellBuffer = await sharp(image) + .extract({ + left, + top, + width: cellWidth, + height: cellHeight, + }) + .png() + .toBuffer(); + + buffers.push(cellBuffer); + } + + return buffers; +}; diff --git a/backup/agents/storyboard/index.ts b/backup/agents/storyboard/index.ts new file mode 100644 index 0000000..f215888 --- /dev/null +++ b/backup/agents/storyboard/index.ts @@ -0,0 +1,737 @@ +// @/agents/Storyboard.ts +import u from "@/utils"; +import { createAgent } from "langchain"; +import { EventEmitter } from "events"; +import { openAI } from "@/agents/models"; +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import type { DB } from "@/types/database"; +import generateImageTool from "./generateImageTool"; +import imageSplitting from "./imageSplitting"; + +// ==================== 类型定义 ==================== + +type AgentType = "segmentAgent" | "shotAgent"; +type RefreshEvent = "storyline" | "outline" | "assets"; + +// ==================== 常量配置 ==================== + +// const SYSTEM_PROMPTS: Record = { +// segmentAgent: segmentPrompts, +// shotAgent: shotPrompts, +// director: directorPrompts, +// }; + +// ==================== 类型定义:片段和画面 ==================== + +interface Segment { + index: number; + description: string; + emotion?: string; + action?: string; +} + +interface Shot { + id: number; // 分镜独立ID + segmentId: number; // 所属片段ID + title: string; + x: number; + y: number; + cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组,每个cell是一个镜头 +} + +// ==================== 主类 ==================== + +export default class Storyboard { + private readonly projectId: number; + private readonly scriptId: number; + readonly emitter = new EventEmitter(); + history: Array<[string, string]> = []; + novelChapters: DB["t_novel"][] = []; + + // 存储 segmentAgent 生成的片段结果 + private segments: Segment[] = []; + // 存储 shotAgent 生成的分镜结果 + private shots: Shot[] = []; + // 分镜ID计数器 + private shotIdCounter: number = 0; + // 存储正在生成分镜图的分镜ID + private generatingShots: Set = new Set(); + + modelName = "gpt-4.1"; + apiKey = ""; + baseURL = ""; + + constructor(projectId: number, scriptId: number) { + this.projectId = projectId; + this.scriptId = scriptId; + } + + // 更新shopts + public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) { + console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId); + console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId); + console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell); + const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId); + if (shotIndex === -1) { + return `分镜 ${segmentId} 不存在,请检查分镜ID是否正确`; + } + const cellIndex = this.shots[shotIndex].cells.findIndex((item) => item.id === cellId.toString()); + if (cellIndex === -1) { + return `镜头 ${cellId} 不存在,请检查镜头ID是否正确`; + } + this.shots[shotIndex].cells[cellIndex] = { ...this.shots[shotIndex].cells[cellIndex], ...cell }; + } + + // ==================== 公共方法 ==================== + + get events() { + return this.emitter; + } + // ==================== 私有工具方法 ==================== + + private emit(event: string, data?: any) { + this.emitter.emit(event, data); + } + + private refresh(type: RefreshEvent) { + this.emit("refresh", type); + } + + private log(action: string, detail?: string) { + const msg = detail ? `${action}: ${detail}` : action; + console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`); + } + + // ==================== 剧本相关操作 ==================== + + getScript = tool( + async () => { + this.log("获取剧本", `scriptId: ${this.scriptId}`); + const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first(); + if (!script) throw new Error("剧本不存在"); + return `剧本集:${script.name}\n\n内容:\n\`\`\`${script.content}\`\`\``; + }, + { + name: "getScript", + description: "获取剧本内容", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== 资产相关操作 ==================== + + /** + * 获取资产列表(供 segmentAgent 和 shotAgent 调用) + */ + getAssets = tool( + async () => { + this.log("获取资产列表", `scriptId: ${this.scriptId}`); + const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first(); + const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first(); + const outline: any | null = row?.data ? JSON.parse(row.data) : null; + + if (!outline) { + return "暂无资产数据"; + } + + // 提取资源名称和描述(与generateImageTool保持一致的字段名) + const resources = outline + ? (["characters", "props", "scenes"] as const).flatMap( + (k) => outline[k]?.map((i: any) => ({ name: i.name, description: i.description })) ?? [], + ) + : []; + + if (resources.length === 0) { + return "暂无资产数据"; + } + + // 分类提取资源并格式化 + const characters = outline?.characters?.map((item: any) => `- ${item.name}${item.description ? `:${item.description}` : ""}`) ?? []; + const props = outline?.props?.map((item: any) => `- ${item.name}${item.description ? `:${item.description}` : ""}`) ?? []; + const scenes = outline?.scenes?.map((item: any) => `- ${item.name}${item.description ? `:${item.description}` : ""}`) ?? []; + + const sections = [ + characters.length ? `【角色】\n${characters.join("\n")}` : "", + props.length ? `【道具】\n${props.join("\n")}` : "", + scenes.length ? `【场景】\n${scenes.join("\n")}` : "", + ].filter(Boolean); + + if (sections.length === 0) { + return "暂无资产数据"; + } + + return `<资产列表> +${sections.join("\n\n")} + + +⚠️ 重要规则: +1. 必须原封不动地使用上述资产名称,禁止使用近义词、缩写或任何变体 +2. 禁止在资产名称前后添加修饰词 +3. 禁止捏造资产列表中不存在的角色、场景、道具`; + }, + { + name: "getAssets", + description: "获取资产列表(角色、道具、场景),包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息,确保名称一致性", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== 片段和分镜工具 ==================== + + /** + * 获取当前存储的片段数据(供 shotAgent 调用) + */ + getSegments = tool( + async () => { + this.log("获取片段数据", `共 ${this.segments.length} 个片段`); + if (this.segments.length === 0) { + return "暂无片段数据,请先调用 segmentAgent 生成片段"; + } + return JSON.stringify(this.segments, null, 2); + }, + { + name: "getSegments", + description: "获取当前已生成的片段数据,用于生成分镜", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + /** + * 更新/存储片段数据(供 segmentAgent 调用) + */ + updateSegments = tool( + async ({ segments }: { segments: Segment[] }) => { + this.log("更新片段数据", `共 ${segments.length} 个片段`); + this.segments = segments; + this.emit("segmentsUpdated", this.segments); + return `成功存储 ${segments.length} 个片段`; + }, + { + name: "updateSegments", + description: "存储生成的片段数据,segmentAgent 在生成片段后必须调用此工具保存结果", + schema: z.object({ + segments: z + .array( + z.object({ + index: z.number().describe("片段序号"), + description: z.string().describe("片段描述"), + emotion: z.string().optional().describe("情绪氛围"), + action: z.string().optional().describe("主要动作"), + }), + ) + .describe("片段数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 添加分镜(供 shotAgent 调用) + */ + addShots = tool( + async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => { + const added: { id: number; segmentIndex: number }[] = []; + const skipped: number[] = []; + + for (const item of shots) { + const exists = this.shots.some((f) => f.segmentId === item.segmentIndex); + if (exists) { + skipped.push(item.segmentIndex); + continue; + } + // 分配独立的分镜ID + this.shotIdCounter++; + const shotId = this.shotIdCounter; + this.shots.push({ + id: shotId, + segmentId: item.segmentIndex, + title: `分镜 ${shotId}`, + x: 0, + y: 0, + cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })), + }); + added.push({ id: shotId, segmentIndex: item.segmentIndex }); + } + + const addedInfo = added.map((a) => `分镜${a.id}(片段${a.segmentIndex})`).join(", "); + this.log("添加分镜", `新增: [${addedInfo}], 跳过片段: [${skipped.join(", ")}]`); + this.emit("shotsUpdated", this.shots); + + if (skipped.length) { + return `已添加${addedInfo};片段 ${skipped.join(", ")} 已存在分镜被跳过。当前共 ${this.shots.length} 个分镜`; + } + return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`; + }, + { + name: "addShots", + description: "添加新的分镜。每个分镜有独立ID,包含多个镜头(每个镜头对应一个提示词)。如果片段已存在分镜会跳过", + schema: z.object({ + shots: z + .array( + z.object({ + segmentIndex: z.number().describe("对应的片段序号"), + prompts: z.array(z.string()).describe("镜头提示词数组,每个提示词对应一个镜头(中文)"), + }), + ) + .describe("要添加的分镜数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 更新指定分镜(供 shotAgent 调用) + * 保留原有 cells 的 id 和 src 字段,只更新 prompt + */ + updateShots = tool( + async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => { + const existingIndex = this.shots.findIndex((item) => item.id === shotId); + + if (existingIndex === -1) { + return `分镜 ${shotId} 不存在,请检查分镜ID是否正确`; + } + + const existingCells = this.shots[existingIndex].cells; + + // 更新 cells,保留原有的 id 和 src 字段 + this.shots[existingIndex].cells = prompts.map((prompt, i) => { + const existingCell = existingCells[i]; + if (existingCell) { + // 保留原有 cell 的 id 和 src,只更新 prompt + return { ...existingCell, prompt }; + } else { + // 新增的 cell + return { id: u.uuid(), prompt }; + } + }); + + this.log("更新分镜", `分镜 ${shotId}`); + this.emit("shotsUpdated", this.shots); + + return `已更新分镜 ${shotId}`; + }, + { + name: "updateShots", + description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜", + schema: z.object({ + shotId: z.number().describe("要更新的分镜ID"), + prompts: z.array(z.string()).describe("新的镜头提示词数组,每个提示词对应一个镜头"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 删除指定分镜(供 shotAgent 调用) + */ + deleteShots = tool( + async ({ shotIds }: { shotIds: number[] }) => { + const deleted: number[] = []; + const notFound: number[] = []; + + for (const shotId of shotIds) { + const idx = this.shots.findIndex((item) => item.id === shotId); + if (idx === -1) { + notFound.push(shotId); + } else { + this.shots.splice(idx, 1); + deleted.push(shotId); + } + } + + this.log("删除分镜", `删除: [分镜${deleted.join(", 分镜")}], 未找到: [分镜${notFound.join(", 分镜")}]`); + this.emit("shotsUpdated", this.shots); + + if (notFound.length) { + return `已删除分镜 ${deleted.join(", ")};分镜 ${notFound.join(", ")} 不存在。当前共 ${this.shots.length} 个分镜`; + } + return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`; + }, + { + name: "deleteShots", + description: "删除指定的分镜。通过分镜ID指定要删除的分镜", + schema: z.object({ + shotIds: z.array(z.number()).describe("要删除的分镜ID数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 生成分镜图(异步执行,使用 nanoBanana) + */ + generateShotImage = tool( + async ({ shotIds }: { shotIds: number[] }) => { + const toGenerate: number[] = []; + const alreadyGenerating: number[] = []; + const notFound: number[] = []; + + for (const shotId of shotIds) { + const shot = this.shots.find((f) => f.id === shotId); + if (!shot) { + notFound.push(shotId); + continue; + } + if (this.generatingShots.has(shotId)) { + alreadyGenerating.push(shotId); + continue; + } + toGenerate.push(shotId); + } + + if (toGenerate.length === 0) { + if (notFound.length) { + return `分镜 ${notFound.join(", ")} 不存在,请检查分镜ID是否正确`; + } + if (alreadyGenerating.length) { + return `分镜 ${alreadyGenerating.join(", ")} 正在生成中,请稍候`; + } + return "没有需要生成的分镜"; + } + + // 标记为正在生成 + for (const id of toGenerate) { + this.generatingShots.add(id); + } + + // 通知前端开始生成 + this.emit("shotImageGenerateStart", { shotIds: toGenerate }); + this.log("开始生成分镜图", `分镜: [${toGenerate.join(", ")}]`); + + // 异步执行图片生成(不阻塞 Agent 流程) + this.executeShotImageGeneration(toGenerate).catch((err) => { + this.log("分镜图生成错误", err.message); + this.emit("shotImageGenerateError", { shotIds: toGenerate, error: err.message }); + }); + + let result = `已开始为分镜 ${toGenerate.join(", ")} 生成分镜图,生成过程在后台进行`; + if (alreadyGenerating.length) { + result += `;分镜 ${alreadyGenerating.join(", ")} 正在生成中`; + } + if (notFound.length) { + result += `;分镜 ${notFound.join(", ")} 不存在`; + } + return result; + }, + { + name: "generateShotImage", + description: + "为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图,然后自动分割为单格图片。通过分镜ID指定,不需要指定具体格子,整个分镜是一个完整的生成单元", + schema: z.object({ + shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 执行分镜图生成的具体逻辑(异步并发) + * 每个分镜包含多个镜头,所有镜头的提示词合并生成一张宫格图,再分割为单张镜头图片 + */ + async executeShotImageGeneration(shotIds: number[]): Promise { + await Promise.all(shotIds.map((shotId) => this.generateSingleShotImage(shotId))); + } + + /** + * 生成单个分镜的图片 + */ + private async generateSingleShotImage(shotId: number): Promise { + try { + const shot = this.shots.find((f) => f.id === shotId); + if (!shot) return; + + // 提取所有镜头的有效提示词 + const prompts: string[] = shot.cells.map((c) => c.prompt).filter((p): p is string => Boolean(p)); + + if (prompts.length === 0) { + this.log("跳过分镜图生成", `分镜 ${shotId} 没有有效的镜头提示词`); + this.generatingShots.delete(shotId); + return; + } + + // 通知前端正在生成该分镜 + this.emit("shotImageGenerateProgress", { shotId, status: "generating", message: "正在调用 AI 生成宫格图片" }); + + // 根据所有镜头提示词生成宫格图片 + const gridImage = await generateImageTool( + prompts.map((p) => ({ prompt: p })), + this.scriptId, + this.projectId, + ); + + // 通知前端正在分割图片 + this.emit("shotImageGenerateProgress", { shotId, status: "splitting", message: "正在分割宫格图片为单张镜头图" }); + + // 分割宫格图片为单张镜头图片 + const imageBuffers = await imageSplitting(gridImage, prompts.length); + + // 通知前端正在保存图片 + this.emit("shotImageGenerateProgress", { shotId, status: "saving", message: `正在保存 ${imageBuffers.length} 张镜头图片` }); + + // 保存分割后的镜头图片到 OSS,并获取文件路径 + const timestamp = Date.now(); + const imagePaths: string[] = []; + + for (let i = 0; i < imageBuffers.length; i++) { + const fileName = `${this.projectId}/chat/${this.scriptId}/storyboard/shot_${shotId}_take_${i}_${timestamp}.png`; + await u.oss.writeFile(fileName, imageBuffers[i]); + const imageUrl = await u.oss.getFileUrl(fileName); + imagePaths.push(imageUrl); + + // 每保存一张镜头图片通知进度 + this.emit("shotImageGenerateProgress", { + shotId, + status: "saving", + message: `已保存 ${i + 1}/${imageBuffers.length} 张镜头图片`, + progress: Math.round(((i + 1) / imageBuffers.length) * 100), + }); + } + + // 更新每个镜头的 src 字段 + shot.cells = shot.cells.map((cell, i) => ({ + id: u.uuid(), + ...cell, + src: imagePaths[i] || cell.src, + })); + + // 生成完成后更新状态 + this.generatingShots.delete(shotId); + this.emit("shotImageGenerateComplete", { shotId, shot, imagePaths }); + this.emit("shotsUpdated", this.shots); + this.log("分镜图生成完成", `分镜 ${shotId},共 ${imagePaths.length} 张镜头图片`); + } catch (err: any) { + this.generatingShots.delete(shotId); + this.emit("shotImageGenerateError", { shotId, error: err.message }); + this.log("分镜图生成失败", `分镜 ${shotId}: ${err.message}`); + } + } + + // ==================== 公共访问器 ==================== + + /** + * 获取当前片段数据 + */ + getSegmentsData(): Segment[] { + return this.segments; + } + + /** + * 获取当前分镜数据 + */ + getShotsData(): Shot[] { + return this.shots; + } + + // ==================== 上下文构建 ==================== + + private async buildEnvironmentContext(): Promise { + const projectInfo = await u.db("t_project").where({ id: this.projectId }).first(); + + const row = await u.db("t_outline").where({ id: this.scriptId, projectId: this.projectId }).first(); + const outline: any | null = row?.data ? JSON.parse(row.data) : null; + + // 分类提取资源名称 + const characters = outline?.characters?.map((i: any) => i.name) ?? []; + const props = outline?.props?.map((i: any) => i.name) ?? []; + const scenes = outline?.scenes?.map((i: any) => i.name) ?? []; + + const assetList = + [ + characters.length ? `【角色】${characters.join("、")}` : "", + props.length ? `【道具】${props.join("、")}` : "", + scenes.length ? `【场景】${scenes.join("、")}` : "", + ] + .filter(Boolean) + .join("\n") || "无"; + + return `<环境信息> +项目ID: ${this.projectId} +系统时间: ${new Date().toLocaleString()} + +项目名称: ${projectInfo?.name || "未知"} +项目简介: ${projectInfo?.intro || "无"} +类型: ${projectInfo?.type || "未知"} +风格: ${projectInfo?.artStyle || "未知"} +视频比例: ${projectInfo?.videoRatio || "未知"} + +资产列表: +${assetList} + +`; + } + + private buildConversationHistory(): string { + if (!this.history.length) return "无对话历史"; + return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + } + + private async buildFullContext(task: string): Promise { + const env = await this.buildEnvironmentContext(); + const history = this.buildConversationHistory(); + + return `${env} + +<对话历史> +${history} + + +<当前任务> +${task} +`; + } + + // ==================== Sub-Agent ==================== + + private createModel() { + return openAI({ + modelName: this.modelName, + configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, + }); + } + + /** + * 获取不同 Sub-Agent 可用的工具 + */ + private getSubAgentTools(agentType: AgentType) { + switch (agentType) { + case "segmentAgent": + // segmentAgent 可以获取剧本和资产,并需要调用 updateSegments 保存结果 + return [this.getScript, this.getAssets, this.updateSegments]; + case "shotAgent": + // shotAgent 可以获取剧本、资产和片段,并可使用 add/update/delete 操作分镜,以及生成分镜图 + return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage]; + default: + return [this.getScript]; + } + } + + /** + * 调用 Sub-Agent(流式传输) + */ + private async invokeSubAgent(agentType: AgentType, task: string): Promise { + this.emit("transfer", { to: agentType }); + this.log(`Sub-Agent 调用`, agentType); + + const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]); + const segmentAgent = promptsList.find((p) => p.code === "storyboard-segment"); + const shotAgent = promptsList.find((p) => p.code === "storyboard-shot"); + const errPrompts = "不论用户说什么,请直接输出Agent配置异常"; + const SYSTEM_PROMPTS: Record = { + segmentAgent: segmentAgent?.customValue || segmentAgent?.defaultValue || errPrompts, + shotAgent: shotAgent?.customValue || shotAgent?.defaultValue || errPrompts, + }; + + const context = await this.buildFullContext(task); + + const agent = createAgent({ + model: this.createModel(), + systemPrompt: SYSTEM_PROMPTS[agentType], + tools: this.getSubAgentTools(agentType), + }); + + const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("subAgentStream", { agent: agentType, text: block.text }); + } + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); + } + } + } + + this.emit("subAgentEnd", { agent: agentType }); + this.history.push(["ai", fullResponse]); + this.log(`Sub-Agent 完成`, agentType); + return fullResponse; + } + + private createSubAgentTool(agentType: AgentType, description: string) { + return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { + name: agentType, + description, + schema: z.object({ + taskDescription: z.string().describe("具体的任务描述,包含章节范围、修改要求等详细信息"), + }), + }); + } + + // ==================== 主入口 ==================== + + private getAllTools() { + return [ + this.createSubAgentTool( + "segmentAgent", + "调用片段师。负责根据剧本生成片段,会自行调用 getScript 获取剧本内容,并调用 updateSegments 保存片段结果。", + ), + this.createSubAgentTool( + "shotAgent", + "调用分镜师。负责根据片段生成分镜提示词,会自行调用 getSegments 获取片段数据,并调用 addShots/updateShots 保存分镜结果。", + ), + // this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲,会自行调用 updateOutline 或 saveStoryline 进行修改。"), + this.getScript, + this.getSegments, + this.generateShotImage, + ...this.getSubAgentTools("segmentAgent"), + ...this.getSubAgentTools("shotAgent"), + ]; + } + + async call(msg: string): Promise { + console.log("模型名称:", this.modelName); + this.history.push(["user", msg]); + + const envContext = await this.buildEnvironmentContext(); + + const prompts = await u.db("t_prompts").where("code", "storyboard-main").first(); + + const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么,请直接输出Agent配置异常"; + + const mainAgent = createAgent({ + model: this.createModel(), + tools: this.getAllTools(), + systemPrompt: `${envContext}\n${mainPrompts}`, + }); + const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("data", block.text); + } + + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); + } + } + } + + this.history.push(["assistant", fullResponse]); + this.emit("response", fullResponse); + + return fullResponse; + } +} diff --git a/package.json b/package.json index a005f94..f8e872d 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "langchain": "^1.2.10", "morgan": "^1.10.1", "qwen-ai-provider": "^0.1.1", + "serialize-error": "^13.0.1", "sharp": "^0.34.5", "sqlite3": "^5.1.7", "zhipu-ai-provider": "^0.2.2", diff --git a/src/agents/outlineScript/index.ts b/src/agents/outlineScript/index.ts index 304c7c0..60e919f 100644 --- a/src/agents/outlineScript/index.ts +++ b/src/agents/outlineScript/index.ts @@ -1,10 +1,8 @@ // @/agents/outlineScript.ts import u from "@/utils"; -import { createAgent } from "langchain"; import { EventEmitter } from "events"; -import { openAI } from "@/agents/models"; +import { tool, ModelMessage } from "ai"; import { z } from "zod"; -import { tool } from "@langchain/core/tools"; import type { DB } from "@/types/database"; // ==================== 类型定义 ==================== @@ -75,13 +73,9 @@ const episodeSchema = z.object({ export default class OutlineScript { private readonly projectId: number; readonly emitter = new EventEmitter(); - history: Array<[string, string]> = []; + history: Array = []; novelChapters: DB["t_novel"][] = []; - modelName = "gpt-4.1"; - apiKey = ""; - baseURL = ""; - constructor(projectId: number) { this.projectId = projectId; } @@ -403,123 +397,107 @@ ${formatList(ep.classicQuotes, (q) => q)} // ==================== Tool 定义:故事线 ==================== - getStoryline = tool( - async () => { + getStoryline = tool({ + title: "getStoryline", + description: "Get the weather in a location", + inputSchema: z.object({}), + execute: async () => { this.log("获取故事线"); const storyline = await this.findStoryline(); return storyline?.content ?? "当前项目暂无故事线"; }, - { - name: "getStoryline", - description: "获取当前项目的故事线内容", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); - saveStoryline = tool( - async ({ content }) => { + saveStoryline = tool({ + title: "saveStoryline", + description: "保存或更新当前项目的故事线,会覆盖已有内容", + inputSchema: z.object({ + content: z.string().describe("故事线完整内容"), + }), + execute: async ({ content }) => { this.log("保存故事线"); await this.upsertStorylineContent(content); return "故事线保存成功"; }, - { - name: "saveStoryline", - description: "保存或更新当前项目的故事线,会覆盖已有内容", - schema: z.object({ - content: z.string().describe("故事线完整内容"), - }), - verboseParsingErrors: true, - }, - ); + }); - deleteStoryline = tool( - async () => { + deleteStoryline = tool({ + title: "deleteStoryline", + description: "删除当前项目的故事线", + inputSchema: z.object({}), + execute: async () => { this.log("删除故事线"); const deleted = await this.deleteStorylineContent(); return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线"; }, - { - name: "deleteStoryline", - description: "删除当前项目的故事线", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); // ==================== Tool 定义:大纲 ==================== - getOutline = tool( - async ({ simplified = false }) => { + getOutline = tool({ + title: "getOutline", + description: "获取项目大纲。simplified=true返回简化列表,false返回完整内容", + inputSchema: z.object({ + simplified: z.boolean().default(false).describe("是否返回简化版本"), + }), + execute: async ({ simplified }) => { this.log("获取大纲", `简化模式: ${simplified}`); return this.getOutlineText(simplified); }, - { - name: "getOutline", - description: "获取项目大纲。simplified=true返回简化列表,false返回完整内容", - schema: z.object({ - simplified: z.boolean().default(false).describe("是否返回简化版本"), - }), - verboseParsingErrors: true, - }, - ); + }); - saveOutline = tool( - async ({ episodes, overwrite = true, startEpisode }) => { + saveOutline = tool({ + title: "saveOutline", + description: "保存大纲数据。overwrite=true会清空现有大纲后写入,false则追加到末尾", + inputSchema: z.object({ + episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"), + overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"), + startEpisode: z.number().optional().describe("追加模式下的起始集数(不填则自动递增)"), + }), + execute: async ({ episodes, overwrite = true, startEpisode }) => { this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`); const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode); return `大纲保存成功:插入 ${insertedCount} 集大纲,创建 ${scriptCount} 个剧本记录`; }, - { - name: "saveOutline", - description: "保存大纲数据。overwrite=true会清空现有大纲后写入,false则追加到末尾", - schema: z.object({ - episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"), - overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"), - startEpisode: z.number().optional().describe("追加模式下的起始集数(不填则自动递增)"), - }), - verboseParsingErrors: true, - }, - ); + }); - updateOutline = tool( - async ({ id, data }) => { + updateOutline = tool({ + title: "updateOutline", + description: "更新指定ID的单集大纲内容", + inputSchema: z.object({ + id: z.number().describe("大纲ID"), + data: episodeSchema.describe("更新后的大纲数据"), + }), + execute: async ({ id, data }) => { this.log("更新大纲", `ID: ${id}`); const success = await this.updateOutlineData(id, data as EpisodeData); return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`; }, - { - name: "updateOutline", - description: "更新指定ID的单集大纲内容", - schema: z.object({ - id: z.number().describe("大纲ID"), - data: episodeSchema.describe("更新后的大纲数据"), - }), - verboseParsingErrors: true, - }, - ); + }); - deleteOutline = tool( - async ({ ids }) => { + deleteOutline = tool({ + title: "deleteOutline", + description: "根据大纲ID删除指定大纲及关联数据", + inputSchema: z.object({ + ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"), + }), + execute: async ({ ids }) => { this.log("删除大纲", `IDs: ${ids.join(", ")}`); const results = await this.deleteOutlineData(ids); const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", "); return `删除结果: ${summary}`; }, - { - name: "deleteOutline", - description: "根据大纲ID删除指定大纲及关联数据", - schema: z.object({ - ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"), - }), - verboseParsingErrors: true, - }, - ); + }); // ==================== Tool 定义:章节 ==================== - getChapter = tool( - async ({ chapterNumbers }) => { + getChapter = tool({ + title: "getChapter", + description: "根据章节编号获取小说章节的完整原文内容,支持批量获取", + inputSchema: z.object({ + chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"), + }), + execute: async ({ chapterNumbers }) => { this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`); const results = await Promise.all( @@ -539,36 +517,24 @@ ${formatList(ep.classicQuotes, (q) => q)} return results.join("\n\n---\n"); }, - { - name: "getChapter", - description: "根据章节编号获取小说章节的完整原文内容,支持批量获取", - schema: z.object({ - chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"), - }), - verboseParsingErrors: true, - }, - ); + }); // ==================== Tool 定义:资产 ==================== - generateAssets = tool( - async () => { + generateAssets = tool({ + title: "generateAssets", + description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产,自动去重并清理冗余", + inputSchema: z.object({}), + execute: async () => { this.log("生成资产"); const stats = await this.generateAssetsFromOutlines(); if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) { return "当前项目没有大纲数据,无法生成资产"; } - return `资产生成完成:新增 ${stats.inserted},更新 ${stats.updated},保持 ${stats.skipped}`; }, - { - name: "generateAssets", - description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产,自动去重并清理冗余", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); // ==================== 上下文构建 ==================== @@ -606,7 +572,7 @@ ${this.getChapterContext()} private buildConversationHistory(): string { if (!this.history.length) return "无对话历史"; - return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n"); } private async buildFullContext(task: string): Promise { @@ -627,14 +593,14 @@ ${task} // ==================== Sub-Agent ==================== private getSubAgentTools() { - return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline]; - } - - private createModel() { - return openAI({ - modelName: this.modelName, - configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, - }); + return { + getChapter: this.getChapter, + getStoryline: this.getStoryline, + saveStoryline: this.saveStoryline, + getOutline: this.getOutline, + saveOutline: this.saveOutline, + updateOutline: this.updateOutline, + }; } /** @@ -657,74 +623,69 @@ ${task} const context = await this.buildFullContext(task); - const agent = createAgent({ - model: this.createModel(), - systemPrompt: SYSTEM_PROMPTS[agentType], + const { fullStream } = await u.ai.text.stream({ + system: SYSTEM_PROMPTS[agentType], tools: this.getSubAgentTools(), + messages: [{ role: "user", content: context }], + maxStep: 100, }); - const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); - let fullResponse = ""; - - for await (const [mode, chunk] of stream) { - if (mode !== "messages") continue; - - const [token] = chunk as any; - const block = token.contentBlocks?.[0]; - - // 处理 AI 文本流 - if (token.type === "ai" && block?.text) { - fullResponse += block.text; - this.emit("subAgentStream", { agent: agentType, text: block.text }); + for await (const item of fullStream) { + if (item.type == "tool-call") { + this.emit("toolCall", { agent: "main", name: item.title, args: null }); } - - // 处理 tool 调用 - if (token.type === "ai" && token.tool_calls?.length) { - for (const toolCall of token.tool_calls) { - this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); - } + if (item.type == "text-delta") { + fullResponse += item.text; + this.emit("subAgentStream", { agent: agentType, text: item.text }); } } this.emit("subAgentEnd", { agent: agentType }); - this.history.push(["ai", fullResponse]); + this.history.push({ + role: "assistant", + content: fullResponse, + }); this.log(`Sub-Agent 完成`, agentType); return fullResponse ?? `${agentType}已完成任务`; } private createSubAgentTool(agentType: AgentType, description: string) { - return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { - name: agentType, + return tool({ + title: agentType, description, - schema: z.object({ + inputSchema: z.object({ taskDescription: z.string().describe("具体的任务描述,包含章节范围、修改要求等详细信息"), }), + execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), }); } // ==================== 主入口 ==================== private getAllTools() { - return [ - this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线,会自行调用 saveStoryline 保存结果。"), - this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲,会自行调用 saveOutline 保存结果。"), - this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲,会自行调用 updateOutline 或 saveStoryline 进行修改。"), - this.getChapter, - this.getStoryline, - this.saveStoryline, - this.deleteStoryline, - this.getOutline, - this.saveOutline, - this.updateOutline, - this.deleteOutline, - this.generateAssets, - ]; + return { + AI1: this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线,会自行调用 saveStoryline 保存结果。"), + AI2: this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲,会自行调用 saveOutline 保存结果。"), + director: this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲,会自行调用 updateOutline 或 saveStoryline 进行修改。"), + getChapter: this.getChapter, + getStoryline: this.getStoryline, + saveStoryline: this.saveStoryline, + deleteStoryline: this.deleteStoryline, + getOutline: this.getOutline, + saveOutline: this.saveOutline, + updateOutline: this.updateOutline, + deleteOutline: this.deleteOutline, + generateAssets: this.generateAssets, + }; } async call(msg: string): Promise { - this.history.push(["user", msg]); + this.history.push({ + role: "user", + content: msg, + }); const envContext = await this.buildEnvironmentContext(); @@ -732,36 +693,28 @@ ${task} const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么,请直接输出Agent配置异常"; - const mainAgent = createAgent({ - model: this.createModel(), + const { fullStream } = await u.ai.text.stream({ + system: `${envContext}\n${mainPrompts}`, tools: this.getAllTools(), - systemPrompt: `${envContext}\n${mainPrompts}`, + messages: this.history, + maxStep: 100, }); - const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); let fullResponse = ""; - - for await (const [mode, chunk] of stream) { - if (mode !== "messages") continue; - - const [token] = chunk as any; - const block = token.contentBlocks?.[0]; - - // 处理 AI 文本流 - if (token.type === "ai" && block?.text) { - fullResponse += block.text; - this.emit("data", block.text); + for await (const item of fullStream) { + if (item.type == "tool-call") { + this.emit("toolCall", { agent: "main", name: item.title, args: null }); } - - // 处理 tool 调用 - if (token.type === "ai" && token.tool_calls?.length) { - for (const toolCall of token.tool_calls) { - this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); - } + if (item.type == "text-delta") { + fullResponse += item.text; + this.emit("data", item.text); } } + this.history.push({ + role: "assistant", + content: fullResponse, + }); - this.history.push(["assistant", fullResponse]); this.emit("response", fullResponse); return fullResponse; diff --git a/src/agents/storyboard/index.ts b/src/agents/storyboard/index.ts index f215888..efbb10b 100644 --- a/src/agents/storyboard/index.ts +++ b/src/agents/storyboard/index.ts @@ -1,10 +1,8 @@ // @/agents/Storyboard.ts import u from "@/utils"; -import { createAgent } from "langchain"; +import { tool, ModelMessage, Tool } from "ai"; import { EventEmitter } from "events"; -import { openAI } from "@/agents/models"; import { z } from "zod"; -import { tool } from "@langchain/core/tools"; import type { DB } from "@/types/database"; import generateImageTool from "./generateImageTool"; import imageSplitting from "./imageSplitting"; @@ -46,7 +44,7 @@ export default class Storyboard { private readonly projectId: number; private readonly scriptId: number; readonly emitter = new EventEmitter(); - history: Array<[string, string]> = []; + history: ModelMessage[] = []; novelChapters: DB["t_novel"][] = []; // 存储 segmentAgent 生成的片段结果 @@ -58,10 +56,6 @@ export default class Storyboard { // 存储正在生成分镜图的分镜ID private generatingShots: Set = new Set(); - modelName = "gpt-4.1"; - apiKey = ""; - baseURL = ""; - constructor(projectId: number, scriptId: number) { this.projectId = projectId; this.scriptId = scriptId; @@ -105,28 +99,28 @@ export default class Storyboard { // ==================== 剧本相关操作 ==================== - getScript = tool( - async () => { + getScript = tool({ + title: "getScript", + description: "获取剧本内容", + inputSchema: z.object({}), + execute: async () => { this.log("获取剧本", `scriptId: ${this.scriptId}`); const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first(); if (!script) throw new Error("剧本不存在"); return `剧本集:${script.name}\n\n内容:\n\`\`\`${script.content}\`\`\``; }, - { - name: "getScript", - description: "获取剧本内容", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); // ==================== 资产相关操作 ==================== /** * 获取资产列表(供 segmentAgent 和 shotAgent 调用) */ - getAssets = tool( - async () => { + getAssets = tool({ + title: "getAssets", + description: "获取资产列表(角色、道具、场景),包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息,确保名称一致性", + inputSchema: z.object({}), + execute: async () => { this.log("获取资产列表", `scriptId: ${this.scriptId}`); const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first(); const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first(); @@ -171,69 +165,69 @@ ${sections.join("\n\n")} 2. 禁止在资产名称前后添加修饰词 3. 禁止捏造资产列表中不存在的角色、场景、道具`; }, - { - name: "getAssets", - description: "获取资产列表(角色、道具、场景),包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息,确保名称一致性", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); // ==================== 片段和分镜工具 ==================== /** * 获取当前存储的片段数据(供 shotAgent 调用) */ - getSegments = tool( - async () => { + getSegments = tool({ + title: "getSegments", + description: "获取当前已生成的片段数据,用于生成分镜", + inputSchema: z.object({}), + execute: async () => { this.log("获取片段数据", `共 ${this.segments.length} 个片段`); if (this.segments.length === 0) { return "暂无片段数据,请先调用 segmentAgent 生成片段"; } return JSON.stringify(this.segments, null, 2); }, - { - name: "getSegments", - description: "获取当前已生成的片段数据,用于生成分镜", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); /** * 更新/存储片段数据(供 segmentAgent 调用) */ - updateSegments = tool( - async ({ segments }: { segments: Segment[] }) => { + updateSegments = tool({ + title: "updateSegments", + description: "存储生成的片段数据,segmentAgent 在生成片段后必须调用此工具保存结果", + inputSchema: z.object({ + segments: z + .array( + z.object({ + index: z.number().describe("片段序号"), + description: z.string().describe("片段描述"), + emotion: z.string().optional().describe("情绪氛围"), + action: z.string().optional().describe("主要动作"), + }), + ) + .describe("片段数组"), + }), + execute: async ({ segments }: { segments: Segment[] }) => { this.log("更新片段数据", `共 ${segments.length} 个片段`); this.segments = segments; this.emit("segmentsUpdated", this.segments); return `成功存储 ${segments.length} 个片段`; }, - { - name: "updateSegments", - description: "存储生成的片段数据,segmentAgent 在生成片段后必须调用此工具保存结果", - schema: z.object({ - segments: z - .array( - z.object({ - index: z.number().describe("片段序号"), - description: z.string().describe("片段描述"), - emotion: z.string().optional().describe("情绪氛围"), - action: z.string().optional().describe("主要动作"), - }), - ) - .describe("片段数组"), - }), - verboseParsingErrors: true, - }, - ); + }); /** * 添加分镜(供 shotAgent 调用) */ - addShots = tool( - async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => { + addShots = tool({ + title: "addShots", + description: "添加新的分镜。每个分镜有独立ID,包含多个镜头(每个镜头对应一个提示词)。如果片段已存在分镜会跳过", + inputSchema: z.object({ + shots: z + .array( + z.object({ + segmentIndex: z.number().describe("对应的片段序号"), + prompts: z.array(z.string()).describe("镜头提示词数组,每个提示词对应一个镜头(中文)"), + }), + ) + .describe("要添加的分镜数组"), + }), + execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => { const added: { id: number; segmentIndex: number }[] = []; const skipped: number[] = []; @@ -266,29 +260,20 @@ ${sections.join("\n\n")} } return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`; }, - { - name: "addShots", - description: "添加新的分镜。每个分镜有独立ID,包含多个镜头(每个镜头对应一个提示词)。如果片段已存在分镜会跳过", - schema: z.object({ - shots: z - .array( - z.object({ - segmentIndex: z.number().describe("对应的片段序号"), - prompts: z.array(z.string()).describe("镜头提示词数组,每个提示词对应一个镜头(中文)"), - }), - ) - .describe("要添加的分镜数组"), - }), - verboseParsingErrors: true, - }, - ); + }); /** * 更新指定分镜(供 shotAgent 调用) * 保留原有 cells 的 id 和 src 字段,只更新 prompt */ - updateShots = tool( - async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => { + updateShots = tool({ + title: "updateShots", + description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜", + inputSchema: z.object({ + shotId: z.number().describe("要更新的分镜ID"), + prompts: z.array(z.string()).describe("新的镜头提示词数组,每个提示词对应一个镜头"), + }), + execute: async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => { const existingIndex = this.shots.findIndex((item) => item.id === shotId); if (existingIndex === -1) { @@ -314,22 +299,18 @@ ${sections.join("\n\n")} return `已更新分镜 ${shotId}`; }, - { - name: "updateShots", - description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜", - schema: z.object({ - shotId: z.number().describe("要更新的分镜ID"), - prompts: z.array(z.string()).describe("新的镜头提示词数组,每个提示词对应一个镜头"), - }), - verboseParsingErrors: true, - }, - ); + }); /** * 删除指定分镜(供 shotAgent 调用) */ - deleteShots = tool( - async ({ shotIds }: { shotIds: number[] }) => { + deleteShots = tool({ + title: "deleteShots", + description: "删除指定的分镜。通过分镜ID指定要删除的分镜", + inputSchema: z.object({ + shotIds: z.array(z.number()).describe("要删除的分镜ID数组"), + }), + execute: async ({ shotIds }: { shotIds: number[] }) => { const deleted: number[] = []; const notFound: number[] = []; @@ -351,21 +332,19 @@ ${sections.join("\n\n")} } return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`; }, - { - name: "deleteShots", - description: "删除指定的分镜。通过分镜ID指定要删除的分镜", - schema: z.object({ - shotIds: z.array(z.number()).describe("要删除的分镜ID数组"), - }), - verboseParsingErrors: true, - }, - ); + }); /** * 生成分镜图(异步执行,使用 nanoBanana) */ - generateShotImage = tool( - async ({ shotIds }: { shotIds: number[] }) => { + generateShotImage = tool({ + title: "generateShotImage", + description: + "为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图,然后自动分割为单格图片。通过分镜ID指定,不需要指定具体格子,整个分镜是一个完整的生成单元", + inputSchema: z.object({ + shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"), + }), + execute: async ({ shotIds }: { shotIds: number[] }) => { const toGenerate: number[] = []; const alreadyGenerating: number[] = []; const notFound: number[] = []; @@ -417,16 +396,7 @@ ${sections.join("\n\n")} } return result; }, - { - name: "generateShotImage", - description: - "为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图,然后自动分割为单格图片。通过分镜ID指定,不需要指定具体格子,整个分镜是一个完整的生成单元", - schema: z.object({ - shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"), - }), - verboseParsingErrors: true, - }, - ); + }); /** * 执行分镜图生成的具体逻辑(异步并发) @@ -566,7 +536,7 @@ ${assetList} private buildConversationHistory(): string { if (!this.history.length) return "无对话历史"; - return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n"); } private async buildFullContext(task: string): Promise { @@ -586,26 +556,33 @@ ${task} // ==================== Sub-Agent ==================== - private createModel() { - return openAI({ - modelName: this.modelName, - configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, - }); - } - /** * 获取不同 Sub-Agent 可用的工具 */ - private getSubAgentTools(agentType: AgentType) { + private getSubAgentTools(agentType: AgentType): Record { switch (agentType) { case "segmentAgent": // segmentAgent 可以获取剧本和资产,并需要调用 updateSegments 保存结果 - return [this.getScript, this.getAssets, this.updateSegments]; + return { + getScript: this.getScript, + getAssets: this.getAssets, + updateSegments: this.updateSegments, + }; case "shotAgent": // shotAgent 可以获取剧本、资产和片段,并可使用 add/update/delete 操作分镜,以及生成分镜图 - return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage]; + return { + getScript: this.getScript, + getAssets: this.getAssets, + getSegments: this.getSegments, + addShots: this.addShots, + updateShots: this.updateShots, + deleteShots: this.deleteShots, + generateShotImage: this.generateShotImage, + }; default: - return [this.getScript]; + return { + getScript: this.getScript, + }; } } @@ -627,74 +604,71 @@ ${task} const context = await this.buildFullContext(task); - const agent = createAgent({ - model: this.createModel(), - systemPrompt: SYSTEM_PROMPTS[agentType], + const { fullStream } = await u.ai.text.stream({ + system: SYSTEM_PROMPTS[agentType], tools: this.getSubAgentTools(agentType), + messages: [{ role: "user", content: context }], + maxStep: 100, }); - const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); - let fullResponse = ""; - - for await (const [mode, chunk] of stream) { - if (mode !== "messages") continue; - const [token] = chunk as any; - const block = token.contentBlocks?.[0]; - - // 处理 AI 文本流 - if (token.type === "ai" && block?.text) { - fullResponse += block.text; - this.emit("subAgentStream", { agent: agentType, text: block.text }); + for await (const item of fullStream) { + if (item.type == "tool-call") { + this.emit("toolCall", { agent: "main", name: item.title, args: null }); } - // 处理 tool 调用 - if (token.type === "ai" && token.tool_calls?.length) { - for (const toolCall of token.tool_calls) { - this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); - } + if (item.type == "text-delta") { + fullResponse += item.text; + this.emit("subAgentStream", { agent: agentType, text: item.text }); } } this.emit("subAgentEnd", { agent: agentType }); - this.history.push(["ai", fullResponse]); + this.history.push({ + role: "assistant", + content: fullResponse, + }); this.log(`Sub-Agent 完成`, agentType); - return fullResponse; + + return fullResponse ?? `${agentType}已完成任务`; } private createSubAgentTool(agentType: AgentType, description: string) { - return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { - name: agentType, + return tool({ + title: agentType, description, - schema: z.object({ + inputSchema: z.object({ taskDescription: z.string().describe("具体的任务描述,包含章节范围、修改要求等详细信息"), }), + execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), }); } // ==================== 主入口 ==================== private getAllTools() { - return [ - this.createSubAgentTool( + return { + segmentAgent: this.createSubAgentTool( "segmentAgent", "调用片段师。负责根据剧本生成片段,会自行调用 getScript 获取剧本内容,并调用 updateSegments 保存片段结果。", ), - this.createSubAgentTool( + shotAgent: this.createSubAgentTool( "shotAgent", "调用分镜师。负责根据片段生成分镜提示词,会自行调用 getSegments 获取片段数据,并调用 addShots/updateShots 保存分镜结果。", ), // this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲,会自行调用 updateOutline 或 saveStoryline 进行修改。"), - this.getScript, - this.getSegments, - this.generateShotImage, + getScript: this.getScript, + getSegments: this.getSegments, + generateShotImage: this.generateShotImage, ...this.getSubAgentTools("segmentAgent"), ...this.getSubAgentTools("shotAgent"), - ]; + }; } async call(msg: string): Promise { - console.log("模型名称:", this.modelName); - this.history.push(["user", msg]); + this.history.push({ + role: "user", + content: msg, + }); const envContext = await this.buildEnvironmentContext(); @@ -702,34 +676,28 @@ ${task} const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么,请直接输出Agent配置异常"; - const mainAgent = createAgent({ - model: this.createModel(), + const { fullStream } = await u.ai.text.stream({ + system: `${envContext}\n${mainPrompts}`, tools: this.getAllTools(), - systemPrompt: `${envContext}\n${mainPrompts}`, + messages: this.history, + maxStep: 100, }); - const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); let fullResponse = ""; - - for await (const [mode, chunk] of stream) { - if (mode !== "messages") continue; - const [token] = chunk as any; - const block = token.contentBlocks?.[0]; - // 处理 AI 文本流 - if (token.type === "ai" && block?.text) { - fullResponse += block.text; - this.emit("data", block.text); + for await (const item of fullStream) { + if (item.type == "tool-call") { + this.emit("toolCall", { agent: "main", name: item.title, args: null }); } - - // 处理 tool 调用 - if (token.type === "ai" && token.tool_calls?.length) { - for (const toolCall of token.tool_calls) { - this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); - } + if (item.type == "text-delta") { + fullResponse += item.text; + this.emit("data", item.text); } } + this.history.push({ + role: "assistant", + content: fullResponse, + }); - this.history.push(["assistant", fullResponse]); this.emit("response", fullResponse); return fullResponse; diff --git a/src/routes/assets/saveAssets.ts b/src/routes/assets/saveAssets.ts index 174c45f..0396352 100644 --- a/src/routes/assets/saveAssets.ts +++ b/src/routes/assets/saveAssets.ts @@ -78,7 +78,9 @@ export default router.post( } // 更新提示信息 - await u.db("t_assets").where("id", id).update({ prompt }); + if (prompt !== undefined && prompt !== null && prompt !== "") { + await u.db("t_assets").where("id", id).update({ prompt }); + } res.status(200).send(success({ message: "保存资产图片成功" })); }, diff --git a/src/routes/other/testImage.ts b/src/routes/other/testImage.ts index 11d52f3..1fc3ef1 100644 --- a/src/routes/other/testImage.ts +++ b/src/routes/other/testImage.ts @@ -16,25 +16,34 @@ export default router.post( }), async (req, res) => { const { modelName, apiKey, baseURL, manufacturer } = req.body; - try { - const contentStr = await u.ai.generateImage( - { - prompt: "2D cat", - imageBase64: [], - aspectRatio: "16:9", - size: "1K", - }, - { - model: modelName, - apiKey, - baseURL, - manufacturer, - }, - ); - res.status(200).send(success(contentStr)); - } catch (err: any) { - const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败"; - res.status(500).send(error(message)); - } + + const image =await u.ai.image({ + prompt: "2D cat", + imageBase64: [], + aspectRatio: "16:9", + size: "1K", + }); + res.status(200).send(success(image)); + + // try { + // const contentStr = await u.ai.generateImage( + // { + // prompt: "2D cat", + // imageBase64: [], + // aspectRatio: "16:9", + // size: "1K", + // }, + // { + // model: modelName, + // apiKey, + // baseURL, + // manufacturer, + // }, + // ); + // res.status(200).send(success(contentStr)); + // } catch (err: any) { + // const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败"; + // res.status(500).send(error(message)); + // } }, ); diff --git a/src/routes/outline/agentsOutline.ts b/src/routes/outline/agentsOutline.ts index 2059272..7ca6032 100644 --- a/src/routes/outline/agentsOutline.ts +++ b/src/routes/outline/agentsOutline.ts @@ -8,7 +8,6 @@ expressWs(router as unknown as Application); router.ws("/", async (ws, req) => { let agent: OutlineScript; - const config = await u.getConfig("language"); const projectId = req.query.projectId; if (!projectId || typeof projectId !== "string") { @@ -19,10 +18,6 @@ router.ws("/", async (ws, req) => { agent = new OutlineScript(Number(projectId)); - agent.modelName = config.model ?? ""; - agent.baseURL = config.baseURL ?? ""; - agent.apiKey = config.apiKey ?? ""; - // const existing = await u // .db("t_chatHistory") // .where({ projectId: Number(projectId) }) diff --git a/src/utils.ts b/src/utils.ts index cb43ce2..4022fd3 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -6,21 +6,24 @@ import number2Chinese from "@/utils/number2Chinese"; import deleteOutline from "@/utils/deleteOutline"; import getConfig from "./utils/getConfig"; import { v4 as uuid } from "uuid"; +import error from "@/utils/error"; +import * as imageTools from "@/utils/imageTools"; + +import AIText from "@/utils/ai/text/index"; +import AIImage from "@/utils/ai/image/index"; -import AIText from "@/utils/ai/text"; -import generateVideo from "@/utils/ai/generateVideo"; -import generateImage from "@/utils/ai/generateImage"; export default { db, oss, ai: { text: AIText, - generateVideo, - generateImage, + image: AIImage, }, editImage, number2Chinese, deleteOutline, getConfig, uuid, + error, + imageTools, }; diff --git a/src/utils/ai/image/index.ts b/src/utils/ai/image/index.ts new file mode 100644 index 0000000..8aefebe --- /dev/null +++ b/src/utils/ai/image/index.ts @@ -0,0 +1,44 @@ +import "./type"; +import u from "@/utils"; +import modelList from "./modelList"; +import axios from "axios"; + +import volcengine from "./owned/volcengine"; +import kling from "./owned/kling"; + + +interface AIConfig { + model?: string; + apiKey?: string; + baseURL?: string; +} + +const urlToBase64 = async (url: string): Promise => { + const res = await axios.get(url, { responseType: "arraybuffer" }); + const base64 = Buffer.from(res.data).toString("base64"); + const mimeType = res.headers["content-type"] || "image/png"; + return `data:${mimeType};base64,${base64}`; +}; + +const modelInstance = { + gemini: null, + volcengine: volcengine, + kling: kling, + vidu: null, + runninghub: null, + apimart: null, +} as const; + +export default async (input: ImageConfig, config?: AIConfig) => { + const sqlTextModelConfig = await u.getConfig("image"); + const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config }; + const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance]; + if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商"); + const owned = modelList.find((m) => m.model === model); + if (!owned) throw new Error("不支持的模型"); + + let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL }); + if (!input.resType) input.resType = "b64"; + if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl); + return input; +}; diff --git a/src/utils/ai/image/modelList.ts b/src/utils/ai/image/modelList.ts new file mode 100644 index 0000000..2b9bae4 --- /dev/null +++ b/src/utils/ai/image/modelList.ts @@ -0,0 +1,77 @@ +interface Owned { + manufacturer: string; + model: string; + grid: boolean; + type: "t2i" | "ti2i" | "i2i"; +} + +const modelList: Owned[] = [ + // 火山引擎 + { + manufacturer: "volcengine", + model: "doubao-seedream-4-5-251128", + grid: false, + type: "ti2i", + }, + { + manufacturer: "volcengine", + model: "doubao-seedream-4-0-250828", + grid: false, + type: "ti2i", + }, + //可灵 + { + manufacturer: "kling", + model: "kling-image-o1", + grid: false, + type: "ti2i", + }, + //gemini + { + manufacturer: "gemini", + model: "gemini-2.5-flash-image", + grid: true, + type: "ti2i", + }, + { + manufacturer: "gemini", + model: "gemini-2.5-flash-image-preview", + grid: true, + type: "ti2i", + }, + { + manufacturer: "gemini", + model: "gemini-2.5-flash-image-preview-all", + grid: true, + type: "ti2i", + }, + { + manufacturer: "gemini", + model: "gemini-3-pro-image-preview", + grid: true, + type: "ti2i", + }, + //Vidu + { + manufacturer: "vidu", + model: "viduq2", + grid: false, + type: "ti2i", + }, + //RunningHub + { + manufacturer: "runninghub", + model: "nanobanana", + grid: true, + type: "ti2i", + }, + //ApiMart + { + manufacturer: "apimart", + model: "nanobanana", + grid: true, + type: "ti2i", + }, +]; + +export default modelList; diff --git a/src/utils/ai/image/owned/gemini.ts b/src/utils/ai/image/owned/gemini.ts new file mode 100644 index 0000000..e710879 --- /dev/null +++ b/src/utils/ai/image/owned/gemini.ts @@ -0,0 +1,34 @@ +import "../type"; +import { createGoogleGenerativeAI } from "@ai-sdk/google"; +import { generateImage } from "ai"; + +export default async (input: ImageConfig, config: AIConfig): Promise => { + if (!config.model) throw new Error("缺少Model名称"); + if (!config.apiKey) throw new Error("缺少API Key"); + if (!input.prompt) throw new Error("缺少提示词"); + + const google = createGoogleGenerativeAI({ + apiKey: config.apiKey, + baseURL: config.baseURL, + }); + + // 构建完整的提示词 + const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt; + + // 根据 size 配置映射到具体尺寸 + const sizeMap: Record = { + "1K": "1024x1024", + "2K": "2048x2048", + "4K": "4096x4096", + }; + + const { image } = await generateImage({ + model: google.image(config.model), + prompt: fullPrompt, + aspectRatio: input.aspectRatio as "1:1" | "3:4" | "4:3" | "9:16" | "16:9", + size: sizeMap[input.size] ?? "1024x1024", + }); + + // 返回生成的图片 base64 + return image.base64; +}; diff --git a/src/utils/ai/image/owned/kling.ts b/src/utils/ai/image/owned/kling.ts new file mode 100644 index 0000000..fa32aa4 --- /dev/null +++ b/src/utils/ai/image/owned/kling.ts @@ -0,0 +1,107 @@ +import "../type"; +import axios from "axios"; +import jwt from "jsonwebtoken"; +import u from "@/utils"; +import { pollTask } from "@/utils/ai/utils"; + +function generateJwtToken(ak: string, sk: string): string { + const now = Math.floor(Date.now() / 1000); + const payload = { + iss: ak, + exp: now + 1800, + nbf: now - 5, + }; + return jwt.sign(payload, sk, { + algorithm: "HS256", + header: { alg: "HS256", typ: "JWT" }, + }); +} + +function getApiToken(apiKey: string): string { + const trimmedKey = apiKey.replace(/^Bearer\s+/i, "").trim(); + + if (trimmedKey.includes("|")) { + const parts = trimmedKey.split("|"); + if (parts.length !== 2 || !parts[0].trim() || !parts[1].trim()) { + throw new Error("API Key格式错误,请使用 ak|sk 格式"); + } + return generateJwtToken(parts[0].trim(), parts[1].trim()); + } + + return trimmedKey; +} + +async function processImages(imageBase64: string[]): Promise> { + let images = imageBase64.filter((img) => img?.trim()); + if (images.length === 0) return []; + + // 压缩所有图片到10MB以内 + images = await Promise.all(images.map((img) => u.imageTools.compressImage(img, "10mb"))); + + // 参考主体数量和参考图片数量之和不得超过10 + if (images.length > 10) { + const mergeImageList = images.splice(9); + const mergedImage = await u.imageTools.mergeImages(mergeImageList, "10mb"); + images.push(mergedImage); + } + + return images.map((img) => ({ + image: img.replace(/^data:image\/[a-z]+;base64,/i, ""), + })); +} + +export default async (input: ImageConfig, config: AIConfig): Promise => { + if (!config.apiKey) throw new Error("缺少API Key"); + if (!input.prompt) throw new Error("缺少提示词,prompt为必填项"); + + const authorization = `Bearer ${getApiToken(config.apiKey)}`; + const baseURL = (config.baseURL ?? "https://api-beijing.klingai.com/v1/images/omni-image").replace(/\/+$/, ""); + const imageList = await processImages(input.imageBase64); + + const body: Record = { + model_name: config.model || "kling-image-o1", + prompt: input.prompt, + n: 1, + ...(input.size !== "4K" && { resolution: input.size.toLowerCase() }), + ...(imageList.length > 0 && { image_list: imageList }), + }; + + const headers = { + "Content-Type": "application/json", + Authorization: authorization, + }; + + try { + const { data: createData } = await axios.post(baseURL, body, { headers }); + + if (createData.code !== 0) { + throw new Error(createData.message || "创建任务失败"); + } + + const taskId = createData.data?.task_id; + if (!taskId) throw new Error("未获取到任务ID"); + + const queryUrl = `${baseURL}/${taskId}`; + return await pollTask(async () => { + const { data: queryData } = await axios.get(queryUrl, { headers }); + + if (queryData.code !== 0) { + return { completed: false, error: queryData.message || "查询任务失败" }; + } + + const { task_status, task_status_msg, task_result } = queryData.data || {}; + + if (task_status === "failed") { + return { completed: false, error: task_status_msg || "图片生成失败" }; + } + + if (task_status === "succeed") { + return { completed: true, imageUrl: task_result?.images?.[0]?.url }; + } + + return { completed: false }; + }); + } catch (error) { + throw new Error(u.error(error).message || "可灵图片生成失败"); + } +} diff --git a/src/utils/ai/image/owned/volcengine.ts b/src/utils/ai/image/owned/volcengine.ts new file mode 100644 index 0000000..2d93fdf --- /dev/null +++ b/src/utils/ai/image/owned/volcengine.ts @@ -0,0 +1,31 @@ +import "../type"; +import axios from "axios"; +import u from "@/utils"; + +export default async (input: ImageConfig, config: AIConfig): Promise => { + if (!config.model) throw new Error("缺少Model名称"); + if (!config.apiKey) throw new Error("缺少API Key"); + + const apiKey = "Bearer " + config.apiKey.replace(/Bearer\s+/g, "").trim(); + const size = input.size === "1K" ? "2K" : input.size; + + const body: Record = { + model: config.model, + prompt: input.prompt, + size, + response_format: "url", + sequential_image_generation: "disabled", + stream: false, + watermark: false, + ...(input.imageBase64 && { image: input.imageBase64 }), + }; + + const url = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/images/generations"; + try { + const { data } = await axios.post(url, body, { headers: { Authorization: apiKey } }); + return data.data[0]?.url; + } catch (error) { + const msg = u.error(error).message || "Volcengine 图片生成失败"; + throw new Error(msg); + } +} diff --git a/src/utils/ai/image/type.ts b/src/utils/ai/image/type.ts new file mode 100644 index 0000000..bbd7eec --- /dev/null +++ b/src/utils/ai/image/type.ts @@ -0,0 +1,14 @@ +interface ImageConfig { + systemPrompt?: string; + prompt: string; + imageBase64: string[]; + size: "1K" | "2K" | "4K"; + aspectRatio: string; + resType?: "url" | "b64"; +} + +interface AIConfig { + model?: string; + apiKey?: string; + baseURL?: string; +} \ No newline at end of file diff --git a/src/utils/ai/text.ts b/src/utils/ai/text/index.ts similarity index 100% rename from src/utils/ai/text.ts rename to src/utils/ai/text/index.ts diff --git a/src/utils/ai/modelList.ts b/src/utils/ai/text/modelList.ts similarity index 100% rename from src/utils/ai/modelList.ts rename to src/utils/ai/text/modelList.ts diff --git a/src/utils/ai/utils.ts b/src/utils/ai/utils.ts new file mode 100644 index 0000000..9bcc2a9 --- /dev/null +++ b/src/utils/ai/utils.ts @@ -0,0 +1,13 @@ +export const pollTask = async ( + queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, + maxAttempts = 500, + interval = 2000, +): Promise => { + for (let i = 0; i < maxAttempts; i++) { + await new Promise((resolve) => setTimeout(resolve, interval)); + const { completed, imageUrl, error } = await queryFn(); + if (error) throw new Error(error); + if (completed && imageUrl) return imageUrl; + } + throw new Error(`任务轮询超时,已尝试 ${maxAttempts} 次`); +}; \ No newline at end of file diff --git a/src/utils/ai/video/owned/volcengine.ts b/src/utils/ai/video/owned/volcengine.ts new file mode 100644 index 0000000..8729fad --- /dev/null +++ b/src/utils/ai/video/owned/volcengine.ts @@ -0,0 +1,70 @@ +import "../type"; +import axios from "axios"; +import u from "@/utils"; + +interface DoubaoVideoConfig { + prompt: string; + savePath: string; + imageBase64?: string[]; // 单张参考图片 base64 + duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒 + aspectRatio: "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive"; + audio?: boolean; +} + +const pollTask = async ( + queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, + maxAttempts = 500, + interval = 2000, +): Promise => { + for (let i = 0; i < maxAttempts; i++) { + await new Promise((resolve) => setTimeout(resolve, interval)); + const { completed, imageUrl, error } = await queryFn(); + if (error) throw new Error(error); + if (completed && imageUrl) return imageUrl; + } + throw new Error(`任务轮询超时,已尝试 ${maxAttempts} 次`); +}; + +export default async (input: ImageConfig, config: AIConfig) => { + console.log("%c Line:5 🍓 input", "background:#7f2b82", input); + console.log("%c Line:5 🍎 config", "background:#93c0a4", config); + if (!config.model) throw new Error("缺少Model名称"); + if (!config.apiKey) throw new Error("缺少API Key"); + + const key = "Bearer " + config.apiKey.replaceAll("Bearer ", "").trim(); + + const doubaoConfig = config as DoubaoVideoConfig; + const createRes = await axios.post( + config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", + { + model: "doubao-seedance-1-5-pro-251215", + content: [ + { type: "text", text: input.prompt }, + ...(doubaoConfig.imageBase64 + ? doubaoConfig.imageBase64.map((base64, i) => ({ + type: "image_url", + image_url: { url: base64 }, + role: i === 0 ? "first_frame" : "last_frame", + })) + : []), + ], + generate_audio: doubaoConfig.audio ?? false, + duration: doubaoConfig.duration, + resolution: doubaoConfig.aspectRatio, + watermark: false, + }, + { headers: { "Content-Type": "application/json", Authorization: key } }, + ); + const taskId = createRes.data.id; + if (!taskId) throw new Error("视频任务创建失败"); + return await pollTask(async () => { + const res = await axios.get(`${config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, { + headers: { Authorization: key }, + }); + const { status, content } = res.data; + if (status === "succeeded") return { completed: true, imageUrl: content?.video_url }; + if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` }; + if (["queued", "running"].includes(status)) return { completed: false }; + return { completed: false, error: `未知状态: ${status}` }; + }); +}; diff --git a/src/utils/error.ts b/src/utils/error.ts new file mode 100644 index 0000000..a08789b --- /dev/null +++ b/src/utils/error.ts @@ -0,0 +1,68 @@ +// utils/error.ts +import { serializeError } from "serialize-error"; +import { isAxiosError } from "axios"; + +export interface NormalizedError { + name: string; + message: string; + code?: string; + status?: number; + stack?: string; + cause?: NormalizedError; + responseData?: unknown; + meta?: Record; +} + +export function normalizeError(error: unknown): NormalizedError { + // Axios 特殊处理 + if (isAxiosError(error)) { + return { + name: "AxiosError", + message: error.response?.data?.error?.message || error.response?.data?.message || error.message, + code: error.code, + status: error.response?.status, + stack: error.stack, + responseData: error.response?.data, + meta: { + url: error.config?.url, + method: error.config?.method, + }, + }; + } + + // 普通 Error,用 serialize-error 处理 + if (error instanceof Error) { + const serialized = serializeError(error); + return { + name: serialized.name || "Error", + message: serialized.message || "未知错误", + code: (serialized as any).code, + stack: serialized.stack, + cause: error.cause ? normalizeError(error.cause) : undefined, + meta: extractMeta(serialized), + }; + } + + // 非 Error + return { + name: "UnknownError", + message: String(error), + meta: { raw: serializeError(error) }, + }; +} + +// 提取自定义属性 +function extractMeta(obj: Record): Record | undefined { + const standardKeys = ["name", "message", "stack", "cause"]; + const meta: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + if (!standardKeys.includes(key) && value !== undefined) { + meta[key] = value; + } + } + + return Object.keys(meta).length > 0 ? meta : undefined; +} + +export default normalizeError; diff --git a/src/utils/getConfig.ts b/src/utils/getConfig.ts index 8825ea6..cb27c09 100644 --- a/src/utils/getConfig.ts +++ b/src/utils/getConfig.ts @@ -13,9 +13,9 @@ interface TextResData extends BaseConfig { manufacturer: "deepseek" | "openAi" | "doubao"; } +// 图像模型配置接口 interface ImageResData extends BaseConfig { - baseURL: string; - manufacturer: "openAi" | "gemini" | "volcengine" | "runninghub" | "apimart"; + manufacturer: "gemini" | "volcengine" | "kling" | "vidu" | "runninghub" | "apimart"; } interface VideoResData extends BaseConfig { diff --git a/src/utils/imageTools.ts b/src/utils/imageTools.ts new file mode 100644 index 0000000..82b519b --- /dev/null +++ b/src/utils/imageTools.ts @@ -0,0 +1,122 @@ +import sharp from "sharp"; + +/** + * 解析大小字符串为字节数 + */ +function parseSize(size: string): number { + const match = size.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*(kb|mb|gb|b)?$/); + if (!match) { + throw new Error(`无效的大小格式: ${size}`); + } + const value = parseFloat(match[1]); + const unit = match[2] || "b"; + const multipliers: Record = { + b: 1, + kb: 1024, + mb: 1024 * 1024, + gb: 1024 * 1024 * 1024, + }; + return Math.floor(value * multipliers[unit]); +} + +/** + * 将base64字符串转换为Buffer + */ +function base64ToBuffer(base64: string): Buffer { + const base64Data = base64.replace(/^data:image\/\w+;base64,/, ""); + return Buffer.from(base64Data, "base64"); +} + +/** + * 压缩Buffer到指定大小以内 + */ +async function compressToSize(imageBuffer: Buffer, maxBytes: number, originalWidth: number, originalHeight: number): Promise { + let quality = 90; + let scale = 1; + + while (true) { + const targetWidth = Math.round(originalWidth * scale); + const targetHeight = Math.round(originalHeight * scale); + + const resultBuffer = await sharp(imageBuffer).resize(targetWidth, targetHeight, { fit: "fill" }).jpeg({ quality }).toBuffer(); + + if (resultBuffer.length <= maxBytes) { + return resultBuffer; + } + + if (quality > 10) { + quality -= 10; + } else { + quality = 90; + scale *= 0.8; + } + } +} + +/** + * 压缩单张图片到指定大小以内 + * @param imageBase64 - base64编码的图片 + * @param maxSize - 最大输出大小,支持格式如 "10mb", "5MB", "1024kb" 等 + * @returns 压缩后的图片base64字符串 + */ +export async function compressImage(imageBase64: string, maxSize = "10mb"): Promise { + const maxBytes = parseSize(maxSize); + const imageBuffer = base64ToBuffer(imageBase64); + const metadata = await sharp(imageBuffer).metadata(); + const resultBuffer = await compressToSize(imageBuffer, maxBytes, metadata.width || 1, metadata.height || 1); + return resultBuffer.toString("base64"); +} + +/** + * 将多张图片横向拼接为一张,并确保输出大小不超过指定限制 + * @param imageBase64List - base64编码的图片数组 + * @param maxSize - 最大输出大小,支持格式如 "10mb", "5MB", "1024kb" 等 + * @returns 拼接后的图片base64字符串 + */ +export async function mergeImages(imageBase64List: string[], maxSize = "10mb"): Promise { + if (imageBase64List.length === 0) { + throw new Error("图片列表不能为空"); + } + + const maxBytes = parseSize(maxSize); + const imageBuffers = imageBase64List.map(base64ToBuffer); + const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata())); + const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0)); + + // 计算各图片调整后的宽度 + const imageWidths = imageMetadatas.map((metadata) => { + const aspectRatio = (metadata.width || 1) / (metadata.height || 1); + return Math.round(maxHeight * aspectRatio); + }); + const totalWidth = imageWidths.reduce((sum, w) => sum + w, 0); + + // 拼接图片 + const resizedImages = await Promise.all( + imageBuffers.map(async (buffer, index) => { + return sharp(buffer).resize(imageWidths[index], maxHeight, { fit: "cover" }).toBuffer(); + }), + ); + + let currentX = 0; + const compositeInputs = resizedImages.map((buffer, index) => { + const input = { input: buffer, left: currentX, top: 0 }; + currentX += imageWidths[index]; + return input; + }); + + const mergedBuffer = await sharp({ + create: { + width: totalWidth, + height: maxHeight, + channels: 4, + background: { r: 255, g: 255, b: 255, alpha: 1 }, + }, + }) + .composite(compositeInputs) + .jpeg({ quality: 90 }) + .toBuffer(); + + // 复用压缩逻辑 + const resultBuffer = await compressToSize(mergedBuffer, maxBytes, totalWidth, maxHeight); + return resultBuffer.toString("base64"); +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index f52c0c3..a00a10f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4564,6 +4564,11 @@ nodemon@^3.1.11: touch "^3.1.0" undefsafe "^2.0.5" +non-error@^0.1.0: + version "0.1.0" + resolved "https://registry.npmmirror.com/non-error/-/non-error-0.1.0.tgz#b78b7d9a67ccb03ac979f9758813336ca7521cf2" + integrity sha512-TMB1uHiGsHRGv1uYclfhivcnf0/PdFp2pNqRxXjncaAsjYMoisaQJI+SSZCqRq+VliwRTC8tsMQfmrWjDMhkPQ== + nopt@^4.0.1: version "4.0.3" resolved "https://registry.npmmirror.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" @@ -5330,6 +5335,14 @@ send@^1.1.0, send@^1.2.0: range-parser "^1.2.1" statuses "^2.0.2" +serialize-error@^13.0.1: + version "13.0.1" + resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-13.0.1.tgz#dd1e1bf6d3e3d01037d126bd95e919f48b0c8ec0" + integrity sha512-bBZaRwLH9PN5HbLCjPId4dP5bNGEtumcErgOX952IsvOhVPrm3/AeK1y0UHA/QaPG701eg0yEnOKsCOC6X/kaA== + dependencies: + non-error "^0.1.0" + type-fest "^5.4.1" + serialize-error@^7.0.1: version "7.0.1" resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18" @@ -5760,6 +5773,11 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +tagged-tag@^1.0.0: + version "1.0.0" + resolved "https://registry.npmmirror.com/tagged-tag/-/tagged-tag-1.0.0.tgz#a0b5917c2864cba54841495abfa3f6b13edcf4d6" + integrity sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng== + tar-fs@^2.0.0: version "2.1.4" resolved "https://registry.npmmirror.com/tar-fs/-/tar-fs-2.1.4.tgz#800824dbf4ef06ded9afea4acafe71c67c76b930" @@ -5929,6 +5947,13 @@ type-fest@^0.13.1: resolved "https://registry.npmmirror.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== +type-fest@^5.4.1: + version "5.4.3" + resolved "https://registry.npmmirror.com/type-fest/-/type-fest-5.4.3.tgz#b4c7e028da129098911ee2162a0c30df8a1be904" + integrity sha512-AXSAQJu79WGc79/3e9/CR77I/KQgeY1AhNvcShIH4PTcGYyC4xv6H4R4AUOwkPS5799KlVDAu8zExeCrkGquiA== + dependencies: + tagged-tag "^1.0.0" + type-is@^2.0.1: version "2.0.1" resolved "https://registry.npmmirror.com/type-is/-/type-is-2.0.1.tgz#64f6cf03f92fce4015c2b224793f6bdd4b068c97"