From 88a1b829ccb5074406cb5b423b45a1197ff9bbdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ACT=E4=B8=B6=E6=B5=81=E6=98=9F=E9=9B=A8?= <1340145680@qq.com> Date: Wed, 4 Feb 2026 10:05:48 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=85=E4=BA=8B=E7=BA=BFAgent=E5=8E=BB?= =?UTF-8?q?=E9=99=A4langchain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backup/agents/models.ts | 36 + backup/agents/outlineScript/index.ts | 769 ++++++++++++++++++ .../storyboard/generateImagePromptsTool.ts | 130 +++ backup/agents/storyboard/generateImageTool.ts | 334 ++++++++ backup/agents/storyboard/imageSplitting.ts | 94 +++ backup/agents/storyboard/index.ts | 737 +++++++++++++++++ src/agents/outlineScript/index.ts | 307 +++---- src/routes/outline/agentsOutline.ts | 5 - 8 files changed, 2230 insertions(+), 182 deletions(-) create mode 100644 backup/agents/models.ts create mode 100644 backup/agents/outlineScript/index.ts create mode 100644 backup/agents/storyboard/generateImagePromptsTool.ts create mode 100644 backup/agents/storyboard/generateImageTool.ts create mode 100644 backup/agents/storyboard/imageSplitting.ts create mode 100644 backup/agents/storyboard/index.ts diff --git a/backup/agents/models.ts b/backup/agents/models.ts new file mode 100644 index 0000000..16c7193 --- /dev/null +++ b/backup/agents/models.ts @@ -0,0 +1,36 @@ +import { ChatOpenAI, ChatOpenAIFields } from "@langchain/openai"; + +export const openAI = (config: ChatOpenAIFields = {}) => { + return new ChatOpenAI({ + modelName: "gpt-4.1", + temperature: 1, + configuration: { + apiKey: process.env.AI_OPENAI_KEY, + baseURL: process.env.AI_OPENAI_URL, + }, + ...config, + }); +}; + +export const doubao = (config: ChatOpenAIFields = {}) => { + return new ChatOpenAI({ + model: "doubao-seed-1-6-flash-250828", + temperature: 1, + configuration: { + apiKey: process.env.AI_TIKTOK_KEY, + baseURL: process.env.AI_TIKTOK_URL, + }, + ...config, + }); +}; + +export const deepseek = (config: ChatOpenAIFields = {}) => + new ChatOpenAI({ + model: "DeepSeek-V3.2", + temperature: 1, + configuration: { + apiKey: process.env.AI_DEEPSEEK_KEY, + baseURL: process.env.AI_DEEPSEEK_URL, + }, + ...config, + }); diff --git a/backup/agents/outlineScript/index.ts b/backup/agents/outlineScript/index.ts new file mode 100644 index 0000000..304c7c0 --- /dev/null +++ b/backup/agents/outlineScript/index.ts @@ -0,0 +1,769 @@ +// @/agents/outlineScript.ts +import u from "@/utils"; +import { createAgent } from "langchain"; +import { EventEmitter } from "events"; +import { openAI } from "@/agents/models"; +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import type { DB } from "@/types/database"; +// ==================== 类型定义 ==================== + +type AgentType = "AI1" | "AI2" | "director"; +type AssetType = "角色" | "道具" | "场景"; +type RefreshEvent = "storyline" | "outline" | "assets"; + +interface AssetItem { + name: string; + description: string; +} + +interface EpisodeData { + episodeIndex: number; + title: string; + chapterRange: number[]; + scenes: AssetItem[]; // 按 outline 出场顺序排列 + characters: AssetItem[]; // 按 outline 出场顺序排列 + props: AssetItem[]; // 按 outline 出场顺序排列 + coreConflict: string; + outline: string; // 最高优先级，剧本生成的唯一权威 + openingHook: string; // outline 第一句话的视觉化，开篇第一个镜头 + keyEvents: string[]; // 4个元素：[起, 承, 转, 合]，严格按 outline 顺序 + emotionalCurve: string; // 对应 keyEvents 各阶段 + visualHighlights: string[]; // 按 outline 顺序排列的标志性镜头 + endingHook: string; // outline 之后的悬念延伸 + classicQuotes: string[]; +} + +// ==================== Schema 定义 ==================== + +const sceneItemSchema = z.object({ + name: z.string().describe("场景名称，如'五星酒店宴会厅'、'老旧出租屋'"), + description: z.string().describe("环境描写：空间结构、光线氛围、装饰陈设、环境细节"), +}); + +const characterItemSchema = z.object({ + name: z.string().describe("角色姓名（必须是具体人名，禁止'众人'、'群众'等集合描述）"), + description: z.string().describe("人设样貌：年龄体态、五官特征、发型妆容、服装配饰、气质神态"), +}); + +const propItemSchema = z.object({ + name: z.string().describe("道具名称"), + description: z.string().describe("样式描写：材质质感、颜色图案、形状尺寸、磨损痕迹、特殊标记"), +}); + +const episodeSchema = z.object({ + episodeIndex: z.number().describe("集数索引，从1开始递增"), + title: z.string().describe("8字内标题，疑问/感叹句，含情绪爆点"), + chapterRange: z.array(z.number()).describe("关联章节号数组"), + scenes: z.array(sceneItemSchema).describe("场景列表，按 outline 出场顺序排列"), + characters: z.array(characterItemSchema).describe("角色列表，按 outline 出场顺序排列，必须是独立个体"), + props: z.array(propItemSchema).describe("道具列表，按 outline 出场顺序排列，至少3个"), + coreConflict: z.string().describe("核心矛盾：A想要X vs B阻碍X"), + outline: z.string().describe("100-300字剧情主干，最高优先级，剧本生成的唯一权威，按时间顺序完整叙述"), + openingHook: z.string().describe("开场镜头：outline 第一句话的视觉化，必须作为剧本第一个镜头"), + keyEvents: z.array(z.string()).length(4).describe("4个元素的数组：[起, 承, 转, 合]，严格按 outline 顺序从中提取"), + emotionalCurve: z.string().describe("情绪曲线，如：2(压抑)→5(反抗)→9(爆发)→3(余波)，对应 keyEvents 各阶段"), + visualHighlights: z.array(z.string()).describe("3-5个标志性镜头，按 outline 叙事顺序排列"), + endingHook: z.string().describe("结尾悬念：outline 之后的延伸，勾引下集"), + classicQuotes: z.array(z.string()).describe("1-2句金句，每句≤15字，必须从原文提取"), +}); + +// ==================== 常量配置 ==================== + +// ==================== 主类 ==================== + +export default class OutlineScript { + private readonly projectId: number; + readonly emitter = new EventEmitter(); + history: Array<[string, string]> = []; + novelChapters: DB["t_novel"][] = []; + + modelName = "gpt-4.1"; + apiKey = ""; + baseURL = ""; + + constructor(projectId: number) { + this.projectId = projectId; + } + + // ==================== 公共方法 ==================== + + get events() { + return this.emitter; + } + + setNovel(chapters: DB["t_novel"][]) { + this.novelChapters = chapters; + } + + // ==================== 私有工具方法 ==================== + + private emit(event: string, data?: any) { + this.emitter.emit(event, data); + } + + private refresh(type: RefreshEvent) { + this.emit("refresh", type); + } + + private log(action: string, detail?: string) { + const msg = detail ? `${action}: ${detail}` : action; + console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`); + } + + private safeParseJson(str: string, fallback: T): T { + try { + return JSON.parse(str); + } catch { + return fallback; + } + } + + private uniqueByName(items: T[]): T[] { + return Array.from(new Map(items.map((item) => [item.name, item])).values()); + } + + // ==================== 数据库操作 ==================== + + private async getProjectInfo(): Promise { + return u.db("t_project").where({ id: this.projectId }).first(); + } + + private async getNovelInfo(asString = false): Promise { + const info = await this.getProjectInfo(); + if (!info) return asString ? "未查询到项目信息" : null; + + if (asString) { + const fields = [ + `小说名称: ${info.name}`, + `小说简介: ${info.intro}`, + `小说类型: ${info.type}`, + `目标短剧类型: ${info.artStyle}`, + `短剧画幅: ${info.videoRatio}`, + ]; + return fields.join("\n"); + } + return info; + } + + // ==================== 故事线操作 ==================== + + private async findStoryline() { + return u.db("t_storyline").where({ projectId: this.projectId }).first(); + } + + private async upsertStorylineContent(content: string) { + const existing = await this.findStoryline(); + if (existing) { + await u.db("t_storyline").where({ projectId: this.projectId }).update({ content }); + } else { + await u.db("t_storyline").insert({ projectId: this.projectId, content }); + } + this.refresh("storyline"); + } + + private async deleteStorylineContent() { + const deleted = await u.db("t_storyline").where({ projectId: this.projectId }).del(); + this.refresh("storyline"); + return deleted; + } + + // ==================== 大纲操作 ==================== + + private async findOutlines() { + return u.db("t_outline").where({ projectId: this.projectId }).orderBy("episode", "asc"); + } + + private async findOutlineById(id: number) { + return u.db("t_outline").where({ id, projectId: this.projectId }).first(); + } + + private async getMaxEpisode(): Promise { + const result: any = await u.db("t_outline").where({ projectId: this.projectId }).max("episode as max").first(); + return result?.max ?? 0; + } + + private async clearOutlinesAndScripts() { + const outlines = await u.db("t_outline").select("id").where({ projectId: this.projectId }); + if (outlines.length === 0) return 0; + + const outlineIds = outlines.map((o) => o.id); + await u.db("t_script").whereIn("outlineId", outlineIds).del(); + await u.db("t_outline").where({ projectId: this.projectId }).del(); + + return outlines.length; + } + + private async insertOutlines(episodes: EpisodeData[], startEpisode: number) { + const insertList = episodes.map((ep, idx) => ({ + projectId: this.projectId, + data: JSON.stringify({ ...ep, episodeIndex: startEpisode + idx }), + episode: startEpisode + idx, + })); + + await u.db("t_outline").insert(insertList); + return insertList.length; + } + + private async createEmptyScripts(outlineIds: Array<{ id: number; data: string }>) { + const scripts = outlineIds.map((item) => { + const data = this.safeParseJson>(item.data, {}); + return { + name: `第${data.episodeIndex ?? ""}集`, + content: "", + projectId: this.projectId, + outlineId: item.id, + }; + }); + + if (scripts.length > 0) { + await u.db("t_script").insert(scripts); + } + return scripts.length; + } + + private async saveOutlineData(episodes: EpisodeData[], overwrite: boolean, startEpisode?: number) { + if (overwrite) { + const cleared = await this.clearOutlinesAndScripts(); + if (cleared > 0) { + this.log("清理旧数据", `删除了 ${cleared} 条大纲及关联剧本`); + } + } + + const actualStart = overwrite ? 1 : startEpisode ?? (await this.getMaxEpisode()) + 1; + const insertedCount = await this.insertOutlines(episodes, actualStart); + + const newOutlines = await u + .db("t_outline") + .select("id", "data") + .where({ projectId: this.projectId }) + .orderBy("episode", "desc") + .limit(insertedCount); + + const scriptCount = await this.createEmptyScripts(newOutlines as Array<{ id: number; data: string }>); + + this.refresh("outline"); + return { insertedCount, scriptCount }; + } + + private async updateOutlineData(id: number, data: EpisodeData) { + const existing = await this.findOutlineById(id); + if (!existing) return false; + + await u + .db("t_outline") + .where({ id }) + .update({ data: JSON.stringify(data) }); + this.refresh("outline"); + return true; + } + + private async deleteOutlineData(ids: number[]) { + const results = await Promise.allSettled(ids.map((id) => u.deleteOutline(id, this.projectId))); + this.refresh("outline"); + return results; + } + + private formatOutlineDetail(ep: any): string { + const formatList = (items: any[], formatter: (item: any) => string) => + items?.map((item, i) => ` ${i + 1}. ${formatter(item)}`).join("\n") || " 无"; + + // keyEvents 按顺序显示：起、承、转、合 + const keyEventsLabels = ["起", "承", "转", "合"]; + const formatKeyEvents = (events: string[]) => events?.map((e, i) => ` 【${keyEventsLabels[i] || i + 1}】${e}`).join("\n") || " 无"; + + return ` +大纲ID: ${ep.id} +第 ${ep.episodeIndex} 集: ${ep.title || ""} +${"=".repeat(50)} +章节范围: ${ep.chapterRange?.join(", ") || ""} +核心矛盾: ${ep.coreConflict || ""} + +【剧情主干】(最高优先级，剧本生成的唯一权威): +${ep.outline || "无"} + +【开场镜头】(必须作为剧本第一个镜头): +${ep.openingHook || "无"} + +【剧情节点】(严格按顺序：起→承→转→合): +${formatKeyEvents(ep.keyEvents)} + +情绪曲线: ${ep.emotionalCurve || ""} + +【视觉重点】(按剧情主干顺序排列): +${formatList(ep.visualHighlights, (v) => v)} + +【结尾悬念】: +${ep.endingHook || "无"} + +【经典台词】: +${formatList(ep.classicQuotes, (q) => q)} + +角色(按出场顺序): ${ep.characters?.map((c: AssetItem) => `${c.name}(${c.description})`).join("; ") || "无"} +场景(按出场顺序): ${ep.scenes?.map((s: AssetItem) => `${s.name}(${s.description})`).join("; ") || "无"} +道具(按出场顺序): ${ep.props?.map((p: AssetItem) => `${p.name}(${p.description})`).join("; ") || "无"}`; + } + + private async getOutlineText(simplified: boolean): Promise { + const records = await this.findOutlines(); + + if (!records.length) return "当前项目暂无大纲"; + + const episodes = records.map((r) => ({ + id: r.id, + episode: r.episode, + ...this.safeParseJson>(r.data ?? "{}", {}), + })); + + if (simplified) { + const list = episodes.map((ep) => `第 ${ep.episodeIndex ?? ep.episode} 集 (id=${ep.id})`).join("\n"); + return `项目大纲 (共 ${episodes.length} 集):\n${list}`; + } + + const details = episodes.map((ep) => this.formatOutlineDetail(ep)).join("\n"); + return `项目大纲 (共 ${episodes.length} 集)\n\n${details}`; + } + + // ==================== 资产操作 ==================== + + private async findAssetByTypeAndName(type: AssetType, name: string) { + return u.db("t_assets").where({ projectId: this.projectId, type, name }).first(); + } + + private async upsertAsset(type: AssetType, item: AssetItem): Promise<"inserted" | "updated" | "skipped"> { + const existing = await this.findAssetByTypeAndName(type, item.name); + + if (!existing) { + await u.db("t_assets").insert({ + projectId: this.projectId, + type, + name: item.name, + intro: item.description, + prompt: item.description, + }); + return "inserted"; + } + + if (existing.intro !== item.description) { + await u.db("t_assets").where({ id: existing.id }).update({ + intro: item.description, + prompt: item.description, + }); + return "updated"; + } + + return "skipped"; + } + + private extractAssetsFromOutlines(outlines: Array<{ data?: string | null | undefined }>): { + characters: AssetItem[]; + props: AssetItem[]; + scenes: AssetItem[]; + } { + const result = { characters: [] as AssetItem[], props: [] as AssetItem[], scenes: [] as AssetItem[] }; + + for (const outline of outlines) { + const data = this.safeParseJson>(outline.data ?? "{}", {}); + if (data.characters) result.characters.push(...data.characters); + if (data.props) result.props.push(...data.props); + if (data.scenes) result.scenes.push(...data.scenes); + } + + return { + characters: this.uniqueByName(result.characters), + props: this.uniqueByName(result.props), + scenes: this.uniqueByName(result.scenes), + }; + } + + private async generateAssetsFromOutlines() { + const outlines = await u.db("t_outline").select("data").where({ projectId: this.projectId }); + + if (!outlines.length) return { inserted: 0, updated: 0, skipped: 0 }; + + const { characters, props, scenes } = this.extractAssetsFromOutlines(outlines); + + // 只做新增和更新，不做删除 + const stats = { inserted: 0, updated: 0, skipped: 0 }; + + const processItems = async (items: AssetItem[], type: AssetType) => { + for (const item of items) { + const result = await this.upsertAsset(type, item); + stats[result]++; + } + }; + + await processItems(characters, "角色"); + await processItems(props, "道具"); + await processItems(scenes, "场景"); + + this.refresh("assets"); + return { ...stats }; + } + + // ==================== Tool 定义：故事线 ==================== + + getStoryline = tool( + async () => { + this.log("获取故事线"); + const storyline = await this.findStoryline(); + return storyline?.content ?? "当前项目暂无故事线"; + }, + { + name: "getStoryline", + description: "获取当前项目的故事线内容", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + saveStoryline = tool( + async ({ content }) => { + this.log("保存故事线"); + await this.upsertStorylineContent(content); + return "故事线保存成功"; + }, + { + name: "saveStoryline", + description: "保存或更新当前项目的故事线，会覆盖已有内容", + schema: z.object({ + content: z.string().describe("故事线完整内容"), + }), + verboseParsingErrors: true, + }, + ); + + deleteStoryline = tool( + async () => { + this.log("删除故事线"); + const deleted = await this.deleteStorylineContent(); + return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线"; + }, + { + name: "deleteStoryline", + description: "删除当前项目的故事线", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== Tool 定义：大纲 ==================== + + getOutline = tool( + async ({ simplified = false }) => { + this.log("获取大纲", `简化模式: ${simplified}`); + return this.getOutlineText(simplified); + }, + { + name: "getOutline", + description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容", + schema: z.object({ + simplified: z.boolean().default(false).describe("是否返回简化版本"), + }), + verboseParsingErrors: true, + }, + ); + + saveOutline = tool( + async ({ episodes, overwrite = true, startEpisode }) => { + this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`); + const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode); + return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`; + }, + { + name: "saveOutline", + description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾", + schema: z.object({ + episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"), + overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"), + startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"), + }), + verboseParsingErrors: true, + }, + ); + + updateOutline = tool( + async ({ id, data }) => { + this.log("更新大纲", `ID: ${id}`); + const success = await this.updateOutlineData(id, data as EpisodeData); + return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`; + }, + { + name: "updateOutline", + description: "更新指定ID的单集大纲内容", + schema: z.object({ + id: z.number().describe("大纲ID"), + data: episodeSchema.describe("更新后的大纲数据"), + }), + verboseParsingErrors: true, + }, + ); + + deleteOutline = tool( + async ({ ids }) => { + this.log("删除大纲", `IDs: ${ids.join(", ")}`); + const results = await this.deleteOutlineData(ids); + const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", "); + return `删除结果: ${summary}`; + }, + { + name: "deleteOutline", + description: "根据大纲ID删除指定大纲及关联数据", + schema: z.object({ + ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"), + }), + verboseParsingErrors: true, + }, + ); + + // ==================== Tool 定义：章节 ==================== + + getChapter = tool( + async ({ chapterNumbers }) => { + this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`); + + const results = await Promise.all( + chapterNumbers.map(async (num) => { + const chapter = await u + .db("t_novel") + .where({ projectId: this.projectId, chapterIndex: num }) + .select("chapterData", "chapterIndex", "chapter") + .first(); + + if (chapter) { + return `\n【第${chapter.chapterIndex}章 ${chapter.chapter || ""}】\n${chapter.chapterData}`; + } + return `\n【第${num}章】未找到`; + }), + ); + + return results.join("\n\n---\n"); + }, + { + name: "getChapter", + description: "根据章节编号获取小说章节的完整原文内容，支持批量获取", + schema: z.object({ + chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"), + }), + verboseParsingErrors: true, + }, + ); + + // ==================== Tool 定义：资产 ==================== + + generateAssets = tool( + async () => { + this.log("生成资产"); + const stats = await this.generateAssetsFromOutlines(); + + if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) { + return "当前项目没有大纲数据，无法生成资产"; + } + + return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`; + }, + { + name: "generateAssets", + description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== 上下文构建 ==================== + + private getChapterContext(): string { + if (!this.novelChapters.length) return "无章节数据"; + return this.novelChapters.map((c) => `章节号:${c.chapterIndex}，分卷:${c.reel}，章节名:${c.chapter}`).join("\n"); + } + + private async buildEnvironmentContext(): Promise { + const [novelInfo, storyline, outlineCount] = await Promise.all([ + this.getNovelInfo(true), + this.findStoryline(), + u.db("t_outline").where({ projectId: this.projectId }).count("id as count").first() as any, + ]); + + return `<环境信息> +项目ID: ${this.projectId} +系统时间: ${new Date().toLocaleString()} + +${novelInfo} + +已加载章节列表: +${this.getChapterContext()} + +故事线状态: ${storyline ? "已生成" : "未生成"} +大纲状态: 共 ${outlineCount?.count ?? 0} 集 + +可用工具: +- getChapter: 获取章节原文 +- getStoryline/saveStoryline/deleteStoryline: 故事线操作 +- getOutline/saveOutline/updateOutline/deleteOutline: 大纲操作 +- generateAssets: 从大纲生成资产 +`; + } + + private buildConversationHistory(): string { + if (!this.history.length) return "无对话历史"; + return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + } + + private async buildFullContext(task: string): Promise { + const env = await this.buildEnvironmentContext(); + const history = this.buildConversationHistory(); + + return `${env} + +<对话历史> +${history} + + +<当前任务> +${task} +`; + } + + // ==================== Sub-Agent ==================== + + private getSubAgentTools() { + return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline]; + } + + private createModel() { + return openAI({ + modelName: this.modelName, + configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, + }); + } + + /** + * 调用 Sub-Agent（流式传输） + */ + private async invokeSubAgent(agentType: AgentType, task: string): Promise { + this.emit("transfer", { to: agentType }); + this.log(`Sub-Agent 调用`, agentType); + + const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]); + const a1Prompt = promptsList.find((p) => p.code === "outlineScript-a1"); + const a2Prompt = promptsList.find((p) => p.code === "outlineScript-a2"); + const directorPrompt = promptsList.find((p) => p.code === "outlineScript-director"); + const errPrompts = "不论用户说什么，请直接输出Agent配置异常"; + const SYSTEM_PROMPTS: Record = { + AI1: a1Prompt?.customValue || a1Prompt?.defaultValue || errPrompts, + AI2: a2Prompt?.customValue || a2Prompt?.defaultValue || errPrompts, + director: directorPrompt?.customValue || directorPrompt?.defaultValue || errPrompts, + }; + + const context = await this.buildFullContext(task); + + const agent = createAgent({ + model: this.createModel(), + systemPrompt: SYSTEM_PROMPTS[agentType], + tools: this.getSubAgentTools(), + }); + + const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("subAgentStream", { agent: agentType, text: block.text }); + } + + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); + } + } + } + + this.emit("subAgentEnd", { agent: agentType }); + this.history.push(["ai", fullResponse]); + this.log(`Sub-Agent 完成`, agentType); + + return fullResponse ?? `${agentType}已完成任务`; + } + + private createSubAgentTool(agentType: AgentType, description: string) { + return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { + name: agentType, + description, + schema: z.object({ + taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"), + }), + }); + } + + // ==================== 主入口 ==================== + + private getAllTools() { + return [ + this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"), + this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"), + this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"), + this.getChapter, + this.getStoryline, + this.saveStoryline, + this.deleteStoryline, + this.getOutline, + this.saveOutline, + this.updateOutline, + this.deleteOutline, + this.generateAssets, + ]; + } + + async call(msg: string): Promise { + this.history.push(["user", msg]); + + const envContext = await this.buildEnvironmentContext(); + + const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first(); + + const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常"; + + const mainAgent = createAgent({ + model: this.createModel(), + tools: this.getAllTools(), + systemPrompt: `${envContext}\n${mainPrompts}`, + }); + const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("data", block.text); + } + + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); + } + } + } + + this.history.push(["assistant", fullResponse]); + this.emit("response", fullResponse); + + return fullResponse; + } +} diff --git a/backup/agents/storyboard/generateImagePromptsTool.ts b/backup/agents/storyboard/generateImagePromptsTool.ts new file mode 100644 index 0000000..4057221 --- /dev/null +++ b/backup/agents/storyboard/generateImagePromptsTool.ts @@ -0,0 +1,130 @@ +import u from "@/utils"; + +type AspectRatio = "16:9" | "9:16" | "21:9" | "1:1" | "4:3" | "3:4" | "3:2" | "2:3"; + +interface GridLayoutResult { + cols: number; + rows: number; + totalCells: number; + placeholderCount: number; +} + +interface GridPromptOptions { + prompts: string[]; + style: string; + aspectRatio: AspectRatio; + assetsName: { name: string; intro: string }[]; +} + +interface GridPromptResult { + prompt: string; + gridLayout: GridLayoutResult; +} + +/** + * 根据prompts数量计算宫格布局 + */ +function calculateGridLayout(count: number): GridLayoutResult { + let cols: number; + let rows: number; + if (count <= 0) { + cols = 1; + rows = 1; + } else if (count === 1) { + cols = 1; + rows = 1; + } else if (count === 2) { + cols = 2; + rows = 1; + } else if (count === 3) { + cols = 3; + rows = 1; + } else if (count === 4) { + cols = 2; + rows = 2; + } else if (count <= 9) { + cols = 3; + rows = 3; + } else { + cols = 3; + rows = Math.ceil(count / 3); + } + const totalCells = cols * rows; + const placeholderCount = totalCells - count; + return { cols, rows, totalCells, placeholderCount }; +} + +/** + * 获取宽高比描述 + */ +function getAspectRatioDescription(aspectRatio: AspectRatio): string { + const descriptions: Record = { + "16:9": "电影宽银幕", + "9:16": "竖屏短剧", + "21:9": "超宽银幕史诗感", + "1:1": "方形构图", + "4:3": "经典银幕", + "3:4": "竖版经典", + "3:2": "摄影标准", + "2:3": "竖版摄影", + }; + return descriptions[aspectRatio] || "标准比例"; +} + +/** + * 生成电影级宫格分镜提示词 + */ +async function generateGridPrompt(options: GridPromptOptions): Promise { + const { prompts, style, aspectRatio, assetsName } = options; + const layout = calculateGridLayout(prompts.length); + const aspectRatioDesc = getAspectRatioDescription(aspectRatio); + + // 构建宫格位置描述 + const gridPositions: string[] = []; + for (let i = 0; i < layout.totalCells; i++) { + const row = Math.floor(i / layout.cols) + 1; + const col = (i % layout.cols) + 1; + if (i < prompts.length) { + gridPositions.push(`[第${row}行第${col}列]: ${prompts[i]}`); + } else { + gridPositions.push(`[第${row}行第${col}列]: 纯黑图`); + } + } + + // 构建资产说明 + const assetsSection = + assetsName.length > 0 + ? `\n【可用资产】\n${assetsName.map((a) => `- ${a.name}：${a.intro}`).join("\n")}\n\n⚠️ 必须使用完整资产名称，禁止简称或代词。` + : ""; + + const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first(); + + const mainPrompts = promptsData?.customValue || promptsData?.defaultValue; + const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`; + + if (!mainPrompts) return { prompt: errData, gridLayout: layout }; + + const chatModel = await u.ai.text({}); + + const result = await chatModel!.invoke({ + messages: [ + { + role: "system", + content: mainPrompts, + }, + { + role: "user", + content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${ + layout.totalCells + }格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`, + }, + ], + }); + + return { + prompt: result?.text ?? errData, + gridLayout: layout, + }; +} + +export default generateGridPrompt; diff --git a/backup/agents/storyboard/generateImageTool.ts b/backup/agents/storyboard/generateImageTool.ts new file mode 100644 index 0000000..151c830 --- /dev/null +++ b/backup/agents/storyboard/generateImageTool.ts @@ -0,0 +1,334 @@ +import generateImagePromptsTool from "@/agents/storyboard/generateImagePromptsTool"; +import u from "@/utils"; +import sharp from "sharp"; +import { z } from "zod"; + +interface AssetItem { + name: string; + description: string; +} + +interface EpisodeData { + episodeIndex: number; + title: string; + chapterRange: number[]; + scenes: AssetItem[]; + characters: AssetItem[]; + props: AssetItem[]; + coreConflict: string; + openingHook: string; + outline: string; + keyEvents: string[]; + emotionalCurve: string; + visualHighlights: string[]; + endingHook: string; + classicQuotes: string[]; +} + +interface ImageInfo { + name: string; + type: string; + filePath: string; +} + +interface ResourceItem { + name: string; + intro: string; +} + +// 资产过滤响应的 schema +const filteredAssetsSchema = z.object({ + relevantAssets: z + .array( + z.object({ + name: z.string().describe("资产名称"), + reason: z.string().describe("选择该资产的原因"), + }), + ) + .describe("与分镜内容相关的资产列表"), +}); + +// 压缩图片直到不超过指定大小 +async function compressImage(buffer: Buffer, maxSizeBytes: number = 3 * 1024 * 1024): Promise { + if (buffer.length <= maxSizeBytes) { + return buffer; + } + let quality = 90; + let compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer(); + while (compressedBuffer.length > maxSizeBytes && quality > 10) { + quality -= 10; + compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer(); + } + if (compressedBuffer.length > maxSizeBytes) { + const metadata = await sharp(buffer).metadata(); + let scale = 0.9; + while (compressedBuffer.length > maxSizeBytes && scale > 0.1) { + const newWidth = Math.round((metadata.width || 1000) * scale); + const newHeight = Math.round((metadata.height || 1000) * scale); + compressedBuffer = await sharp(buffer) + .resize(newWidth, newHeight, { fit: "inside" }) + .jpeg({ quality: Math.max(quality, 30) }) + .toBuffer(); + scale -= 0.1; + } + } + return compressedBuffer; +} + +// 拼接多张图片为一张 +async function mergeImages(imagePaths: string[]): Promise { + const imageBuffers = await Promise.all(imagePaths.map((path) => u.oss.getFile(path))); + const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata())); + const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0)); + const resizedImages = await Promise.all( + imageBuffers.map(async (buffer, index) => { + const metadata = imageMetadatas[index]; + const aspectRatio = (metadata.width || 1) / (metadata.height || 1); + const newWidth = Math.round(maxHeight * aspectRatio); + return { + buffer: await sharp(buffer).resize(newWidth, maxHeight, { fit: "cover" }).toBuffer(), + width: newWidth, + }; + }), + ); + let currentX = 0; + const compositeInputs = resizedImages.map(({ buffer, width }) => { + const input = { + input: buffer, + left: currentX, + top: 0, + }; + currentX += width; + return input; + }); + const mergedImage = await sharp({ + create: { + width: currentX, + height: maxHeight, + channels: 4, + background: { r: 255, g: 255, b: 255, alpha: 1 }, + }, + }) + .composite(compositeInputs) + .jpeg({ quality: 90 }) + .toBuffer(); + return compressImage(mergedImage); +} + +// 进一步压缩单张图片到指定大小 +async function compressToSize(buffer: Buffer, targetSize: number): Promise { + if (buffer.length <= targetSize) { + return buffer; + } + + const metadata = await sharp(buffer).metadata(); + let quality = 80; + let scale = 1.0; + let compressedBuffer = buffer; + + // 先尝试降低质量 + while (compressedBuffer.length > targetSize && quality > 10) { + compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer(); + quality -= 10; + } + + // 如果还是太大，缩小尺寸 + while (compressedBuffer.length > targetSize && scale > 0.2) { + scale -= 0.1; + const newWidth = Math.round((metadata.width || 1000) * scale); + const newHeight = Math.round((metadata.height || 1000) * scale); + compressedBuffer = await sharp(buffer) + .resize(newWidth, newHeight, { fit: "inside" }) + .jpeg({ quality: Math.max(quality, 20) }) + .toBuffer(); + } + + return compressedBuffer; +} + +// 确保图片列表总大小不超过指定限制 +async function ensureTotalSizeLimit(buffers: Buffer[], maxTotalBytes: number = 10 * 1024 * 1024): Promise { + let totalSize = buffers.reduce((sum, buf) => sum + buf.length, 0); + + if (totalSize <= maxTotalBytes) { + return buffers; + } + + // 计算每张图片的平均目标大小 + const avgTargetSize = Math.floor(maxTotalBytes / buffers.length); + + // 按大小降序排列，优先压缩大图片 + const indexedBuffers = buffers.map((buf, idx) => ({ buf, idx, size: buf.length })); + indexedBuffers.sort((a, b) => b.size - a.size); + + const result = [...buffers]; + + for (const item of indexedBuffers) { + totalSize = result.reduce((sum, buf) => sum + buf.length, 0); + if (totalSize <= maxTotalBytes) { + break; + } + + // 计算这张图片需要压缩到的目标大小 + const excessSize = totalSize - maxTotalBytes; + const targetSize = Math.max(item.buf.length - excessSize, avgTargetSize, 100 * 1024); // 最小100KB + + if (item.buf.length > targetSize) { + result[item.idx] = await compressToSize(item.buf, targetSize); + } + } + + return result; +} + +// 处理图片列表，确保不超过10张且每张不超过3MB，总大小不超过10MB +async function processImages(images: ImageInfo[]): Promise { + const maxImages = 10; + let processedBuffers: Buffer[]; + + if (images.length <= maxImages) { + const buffers = await Promise.all(images.map((img) => u.oss.getFile(img.filePath))); + processedBuffers = await Promise.all(buffers.map((buffer) => compressImage(buffer))); + } else { + const mergeStartIndex = maxImages - 1; + const firstBuffers = await Promise.all(images.slice(0, mergeStartIndex).map((img) => u.oss.getFile(img.filePath))); + const compressedFirstImages = await Promise.all(firstBuffers.map((buffer) => compressImage(buffer))); + const imagesToMergeList = images.slice(mergeStartIndex).map((img) => img.filePath); + const mergedImage = await mergeImages(imagesToMergeList); + processedBuffers = [...compressedFirstImages, mergedImage]; + } + + // 确保总大小不超过10MB + return ensureTotalSizeLimit(processedBuffers); +} + +// 使用 AI 过滤与分镜相关的资产 +async function filterRelevantAssets(prompts: string[], allResources: ResourceItem[], availableImages: ImageInfo[]): Promise { + if (allResources.length === 0 || availableImages.length === 0) { + return availableImages; + } + + const availableNames = new Set(availableImages.map((img) => img.name)); + const availableResources = allResources.filter((r) => availableNames.has(r.name)); + + if (availableResources.length === 0) { + return availableImages; + } + + const chatModel = await u.ai.text({}); + const result = await chatModel!.invoke({ + messages: [ + { + role: "user", + content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。 + +分镜描述： +${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")} + +可用资产列表： +${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")} + +请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, + }, + ], + responseFormat: { + type: "json_schema", + jsonSchema: { + name: "filteredAssets", + strict: true, + schema: z.toJSONSchema(filteredAssetsSchema), + }, + }, + }); + + const data = result?.json as z.infer; + + if (!data?.relevantAssets || data.relevantAssets.length === 0) { + return availableImages; + } + + const relevantNames = new Set(data.relevantAssets.map((a) => a.name)); + const filteredImages = availableImages.filter((img) => relevantNames.has(img.name)); + + return filteredImages.length > 0 ? filteredImages : availableImages; +} + +// 构建资产映射提示词 +function buildResourcesMapPrompts(images: ImageInfo[]): string { + if (images.length === 0) return ""; + + const mapping = images.map((item, index) => { + if (index < 9) { + return `${item.name}=图片${index + 1}`; + } else { + return `${item.name}=图10-${index - 8}`; + } + }); + + return `其中人物、场景、道具参考对照关系如下：${mapping.join(", ")}。`; +} + +export default async (cells: { prompt: string }[], scriptId: number, projectId: number) => { + const scriptData = await u.db("t_script").where({ id: scriptId, projectId }).first(); + const projectInfo = await u.db("t_project").where({ id: projectId }).first(); + + const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId }).first(); + const outline: EpisodeData | null = row?.data ? JSON.parse(row.data) : null; + + const resources: ResourceItem[] = outline + ? (["characters", "props", "scenes"] as const).flatMap((k) => outline[k]?.map((i) => ({ name: i.name, intro: i.description })) ?? []) + : []; + + const resourceNames = resources.map((r) => r.name); + const imagesRaw = await u.db("t_assets").whereIn("name", resourceNames).andWhere({ projectId }).select("name", "type", "filePath"); + + const allImages = imagesRaw + .sort((a, b) => { + const order = ["角色", "场景", "道具"]; + return order.indexOf(a.type!) - order.indexOf(b.type!); + }) + .filter((img) => img.filePath) as ImageInfo[]; + + if (allImages.length === 0) { + throw new Error("未找到可用的图片资源"); + } + + const cellPrompts = cells.map((c) => c.prompt); + + // 使用 AI 过滤相关资产 + const filteredImages = await filterRelevantAssets(cellPrompts, resources, allImages); + + const resourcesMapPrompts = buildResourcesMapPrompts(filteredImages); + console.log("====润色前：", cellPrompts); + const promptsData = await generateImagePromptsTool({ + prompts: cellPrompts, + style: `类型：${projectInfo?.type!}，风格：${projectInfo?.artStyle!}`, + aspectRatio: projectInfo?.videoRatio! as any, + assetsName: resources, + }); + + // const prompts = `请生成${promptsData.gridLayout.totalCells}格,${promptsData.gridLayout.cols}列×${promptsData.gridLayout.rows}行宫格图。 + + // ${promptsData.prompt} + + // 注意：请严格按照提示词内容生成图片，确保人物样貌、艺术风格、色调光影一致。 + // `; + const prompts = promptsData.prompt; + console.log("====润色后：", prompts); + + const processedImages = await processImages(filteredImages); + + const contentStr = await u.ai.generateImage({ + systemPrompt: resourcesMapPrompts, + prompt: prompts, + size: "4K", + aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9", + imageBase64: processedImages.map((buf) => buf.toString("base64")), + }); + + const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/); + const base64Str = match?.[1] ?? contentStr; + const buffer = Buffer.from(base64Str, "base64"); + + return buffer; +}; diff --git a/backup/agents/storyboard/imageSplitting.ts b/backup/agents/storyboard/imageSplitting.ts new file mode 100644 index 0000000..5488194 --- /dev/null +++ b/backup/agents/storyboard/imageSplitting.ts @@ -0,0 +1,94 @@ +import sharp from "sharp"; + +interface GridLayoutResult { + cols: number; + rows: number; + totalCells: number; + placeholderCount: number; +} + +/** + * 计算宫格布局 + * 1张: 1x1 + * 2张: 2x1 + * 3张: 3x1 + * 4张: 2x2 + * 5-9张: 3x3 + * 10-12张: 3x4 + * 13-15张: 3x5 + * ...以此类推（3列，行数递增） + */ +function calculateGridLayout(count: number): GridLayoutResult { + let cols: number; + let rows: number; + if (count <= 0) { + cols = 1; + rows = 1; + } else if (count === 1) { + cols = 1; + rows = 1; + } else if (count === 2) { + cols = 2; + rows = 1; + } else if (count === 3) { + cols = 3; + rows = 1; + } else if (count === 4) { + cols = 2; + rows = 2; + } else if (count <= 9) { + // 5-9格统一用3x3 + cols = 3; + rows = 3; + } else { + cols = 3; + rows = Math.ceil(count / 3); + } + const totalCells = cols * rows; + const placeholderCount = totalCells - count; + return { cols, rows, totalCells, placeholderCount }; +} + +/** + * 分割宫格图片 + * @param image - 输入的宫格图片 Buffer + * @param length - 实际需要的图片数量（不包含占位图） + * @returns 分割后的单张图片 Buffer 数组 + */ +export default async (image: Buffer, length: number): Promise => { + const metadata = await sharp(image).metadata(); + const { width: totalWidth, height: totalHeight } = metadata; + + if (!totalWidth || !totalHeight) { + throw new Error("无法获取图片尺寸"); + } + + const { cols, rows } = calculateGridLayout(length); + + const cellWidth = Math.floor(totalWidth / cols); + const cellHeight = Math.floor(totalHeight / rows); + + const buffers: Buffer[] = []; + + for (let i = 0; i < length; i++) { + const row = Math.floor(i / cols); + const col = i % cols; + + const left = col * cellWidth; + const top = row * cellHeight; + + const cellBuffer = await sharp(image) + .extract({ + left, + top, + width: cellWidth, + height: cellHeight, + }) + .png() + .toBuffer(); + + buffers.push(cellBuffer); + } + + return buffers; +}; diff --git a/backup/agents/storyboard/index.ts b/backup/agents/storyboard/index.ts new file mode 100644 index 0000000..f215888 --- /dev/null +++ b/backup/agents/storyboard/index.ts @@ -0,0 +1,737 @@ +// @/agents/Storyboard.ts +import u from "@/utils"; +import { createAgent } from "langchain"; +import { EventEmitter } from "events"; +import { openAI } from "@/agents/models"; +import { z } from "zod"; +import { tool } from "@langchain/core/tools"; +import type { DB } from "@/types/database"; +import generateImageTool from "./generateImageTool"; +import imageSplitting from "./imageSplitting"; + +// ==================== 类型定义 ==================== + +type AgentType = "segmentAgent" | "shotAgent"; +type RefreshEvent = "storyline" | "outline" | "assets"; + +// ==================== 常量配置 ==================== + +// const SYSTEM_PROMPTS: Record = { +// segmentAgent: segmentPrompts, +// shotAgent: shotPrompts, +// director: directorPrompts, +// }; + +// ==================== 类型定义：片段和画面 ==================== + +interface Segment { + index: number; + description: string; + emotion?: string; + action?: string; +} + +interface Shot { + id: number; // 分镜独立ID + segmentId: number; // 所属片段ID + title: string; + x: number; + y: number; + cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头 +} + +// ==================== 主类 ==================== + +export default class Storyboard { + private readonly projectId: number; + private readonly scriptId: number; + readonly emitter = new EventEmitter(); + history: Array<[string, string]> = []; + novelChapters: DB["t_novel"][] = []; + + // 存储 segmentAgent 生成的片段结果 + private segments: Segment[] = []; + // 存储 shotAgent 生成的分镜结果 + private shots: Shot[] = []; + // 分镜ID计数器 + private shotIdCounter: number = 0; + // 存储正在生成分镜图的分镜ID + private generatingShots: Set = new Set(); + + modelName = "gpt-4.1"; + apiKey = ""; + baseURL = ""; + + constructor(projectId: number, scriptId: number) { + this.projectId = projectId; + this.scriptId = scriptId; + } + + // 更新shopts + public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) { + console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId); + console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId); + console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell); + const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId); + if (shotIndex === -1) { + return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`; + } + const cellIndex = this.shots[shotIndex].cells.findIndex((item) => item.id === cellId.toString()); + if (cellIndex === -1) { + return `镜头 ${cellId} 不存在，请检查镜头ID是否正确`; + } + this.shots[shotIndex].cells[cellIndex] = { ...this.shots[shotIndex].cells[cellIndex], ...cell }; + } + + // ==================== 公共方法 ==================== + + get events() { + return this.emitter; + } + // ==================== 私有工具方法 ==================== + + private emit(event: string, data?: any) { + this.emitter.emit(event, data); + } + + private refresh(type: RefreshEvent) { + this.emit("refresh", type); + } + + private log(action: string, detail?: string) { + const msg = detail ? `${action}: ${detail}` : action; + console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`); + } + + // ==================== 剧本相关操作 ==================== + + getScript = tool( + async () => { + this.log("获取剧本", `scriptId: ${this.scriptId}`); + const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first(); + if (!script) throw new Error("剧本不存在"); + return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``; + }, + { + name: "getScript", + description: "获取剧本内容", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== 资产相关操作 ==================== + + /** + * 获取资产列表（供 segmentAgent 和 shotAgent 调用） + */ + getAssets = tool( + async () => { + this.log("获取资产列表", `scriptId: ${this.scriptId}`); + const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first(); + const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first(); + const outline: any | null = row?.data ? JSON.parse(row.data) : null; + + if (!outline) { + return "暂无资产数据"; + } + + // 提取资源名称和描述（与generateImageTool保持一致的字段名） + const resources = outline + ? (["characters", "props", "scenes"] as const).flatMap( + (k) => outline[k]?.map((i: any) => ({ name: i.name, description: i.description })) ?? [], + ) + : []; + + if (resources.length === 0) { + return "暂无资产数据"; + } + + // 分类提取资源并格式化 + const characters = outline?.characters?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? []; + const props = outline?.props?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? []; + const scenes = outline?.scenes?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? []; + + const sections = [ + characters.length ? `【角色】\n${characters.join("\n")}` : "", + props.length ? `【道具】\n${props.join("\n")}` : "", + scenes.length ? `【场景】\n${scenes.join("\n")}` : "", + ].filter(Boolean); + + if (sections.length === 0) { + return "暂无资产数据"; + } + + return `<资产列表> +${sections.join("\n\n")} + + +⚠️ 重要规则： +1. 必须原封不动地使用上述资产名称，禁止使用近义词、缩写或任何变体 +2. 禁止在资产名称前后添加修饰词 +3. 禁止捏造资产列表中不存在的角色、场景、道具`; + }, + { + name: "getAssets", + description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + // ==================== 片段和分镜工具 ==================== + + /** + * 获取当前存储的片段数据（供 shotAgent 调用） + */ + getSegments = tool( + async () => { + this.log("获取片段数据", `共 ${this.segments.length} 个片段`); + if (this.segments.length === 0) { + return "暂无片段数据，请先调用 segmentAgent 生成片段"; + } + return JSON.stringify(this.segments, null, 2); + }, + { + name: "getSegments", + description: "获取当前已生成的片段数据，用于生成分镜", + schema: z.object({}), + verboseParsingErrors: true, + }, + ); + + /** + * 更新/存储片段数据（供 segmentAgent 调用） + */ + updateSegments = tool( + async ({ segments }: { segments: Segment[] }) => { + this.log("更新片段数据", `共 ${segments.length} 个片段`); + this.segments = segments; + this.emit("segmentsUpdated", this.segments); + return `成功存储 ${segments.length} 个片段`; + }, + { + name: "updateSegments", + description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果", + schema: z.object({ + segments: z + .array( + z.object({ + index: z.number().describe("片段序号"), + description: z.string().describe("片段描述"), + emotion: z.string().optional().describe("情绪氛围"), + action: z.string().optional().describe("主要动作"), + }), + ) + .describe("片段数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 添加分镜（供 shotAgent 调用） + */ + addShots = tool( + async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => { + const added: { id: number; segmentIndex: number }[] = []; + const skipped: number[] = []; + + for (const item of shots) { + const exists = this.shots.some((f) => f.segmentId === item.segmentIndex); + if (exists) { + skipped.push(item.segmentIndex); + continue; + } + // 分配独立的分镜ID + this.shotIdCounter++; + const shotId = this.shotIdCounter; + this.shots.push({ + id: shotId, + segmentId: item.segmentIndex, + title: `分镜 ${shotId}`, + x: 0, + y: 0, + cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })), + }); + added.push({ id: shotId, segmentIndex: item.segmentIndex }); + } + + const addedInfo = added.map((a) => `分镜${a.id}(片段${a.segmentIndex})`).join(", "); + this.log("添加分镜", `新增: [${addedInfo}], 跳过片段: [${skipped.join(", ")}]`); + this.emit("shotsUpdated", this.shots); + + if (skipped.length) { + return `已添加${addedInfo}；片段 ${skipped.join(", ")} 已存在分镜被跳过。当前共 ${this.shots.length} 个分镜`; + } + return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`; + }, + { + name: "addShots", + description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过", + schema: z.object({ + shots: z + .array( + z.object({ + segmentIndex: z.number().describe("对应的片段序号"), + prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"), + }), + ) + .describe("要添加的分镜数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 更新指定分镜（供 shotAgent 调用） + * 保留原有 cells 的 id 和 src 字段，只更新 prompt + */ + updateShots = tool( + async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => { + const existingIndex = this.shots.findIndex((item) => item.id === shotId); + + if (existingIndex === -1) { + return `分镜 ${shotId} 不存在，请检查分镜ID是否正确`; + } + + const existingCells = this.shots[existingIndex].cells; + + // 更新 cells，保留原有的 id 和 src 字段 + this.shots[existingIndex].cells = prompts.map((prompt, i) => { + const existingCell = existingCells[i]; + if (existingCell) { + // 保留原有 cell 的 id 和 src，只更新 prompt + return { ...existingCell, prompt }; + } else { + // 新增的 cell + return { id: u.uuid(), prompt }; + } + }); + + this.log("更新分镜", `分镜 ${shotId}`); + this.emit("shotsUpdated", this.shots); + + return `已更新分镜 ${shotId}`; + }, + { + name: "updateShots", + description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜", + schema: z.object({ + shotId: z.number().describe("要更新的分镜ID"), + prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 删除指定分镜（供 shotAgent 调用） + */ + deleteShots = tool( + async ({ shotIds }: { shotIds: number[] }) => { + const deleted: number[] = []; + const notFound: number[] = []; + + for (const shotId of shotIds) { + const idx = this.shots.findIndex((item) => item.id === shotId); + if (idx === -1) { + notFound.push(shotId); + } else { + this.shots.splice(idx, 1); + deleted.push(shotId); + } + } + + this.log("删除分镜", `删除: [分镜${deleted.join(", 分镜")}], 未找到: [分镜${notFound.join(", 分镜")}]`); + this.emit("shotsUpdated", this.shots); + + if (notFound.length) { + return `已删除分镜 ${deleted.join(", ")}；分镜 ${notFound.join(", ")} 不存在。当前共 ${this.shots.length} 个分镜`; + } + return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`; + }, + { + name: "deleteShots", + description: "删除指定的分镜。通过分镜ID指定要删除的分镜", + schema: z.object({ + shotIds: z.array(z.number()).describe("要删除的分镜ID数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 生成分镜图（异步执行，使用 nanoBanana） + */ + generateShotImage = tool( + async ({ shotIds }: { shotIds: number[] }) => { + const toGenerate: number[] = []; + const alreadyGenerating: number[] = []; + const notFound: number[] = []; + + for (const shotId of shotIds) { + const shot = this.shots.find((f) => f.id === shotId); + if (!shot) { + notFound.push(shotId); + continue; + } + if (this.generatingShots.has(shotId)) { + alreadyGenerating.push(shotId); + continue; + } + toGenerate.push(shotId); + } + + if (toGenerate.length === 0) { + if (notFound.length) { + return `分镜 ${notFound.join(", ")} 不存在，请检查分镜ID是否正确`; + } + if (alreadyGenerating.length) { + return `分镜 ${alreadyGenerating.join(", ")} 正在生成中，请稍候`; + } + return "没有需要生成的分镜"; + } + + // 标记为正在生成 + for (const id of toGenerate) { + this.generatingShots.add(id); + } + + // 通知前端开始生成 + this.emit("shotImageGenerateStart", { shotIds: toGenerate }); + this.log("开始生成分镜图", `分镜: [${toGenerate.join(", ")}]`); + + // 异步执行图片生成（不阻塞 Agent 流程） + this.executeShotImageGeneration(toGenerate).catch((err) => { + this.log("分镜图生成错误", err.message); + this.emit("shotImageGenerateError", { shotIds: toGenerate, error: err.message }); + }); + + let result = `已开始为分镜 ${toGenerate.join(", ")} 生成分镜图，生成过程在后台进行`; + if (alreadyGenerating.length) { + result += `；分镜 ${alreadyGenerating.join(", ")} 正在生成中`; + } + if (notFound.length) { + result += `；分镜 ${notFound.join(", ")} 不存在`; + } + return result; + }, + { + name: "generateShotImage", + description: + "为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元", + schema: z.object({ + shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"), + }), + verboseParsingErrors: true, + }, + ); + + /** + * 执行分镜图生成的具体逻辑（异步并发） + * 每个分镜包含多个镜头，所有镜头的提示词合并生成一张宫格图，再分割为单张镜头图片 + */ + async executeShotImageGeneration(shotIds: number[]): Promise { + await Promise.all(shotIds.map((shotId) => this.generateSingleShotImage(shotId))); + } + + /** + * 生成单个分镜的图片 + */ + private async generateSingleShotImage(shotId: number): Promise { + try { + const shot = this.shots.find((f) => f.id === shotId); + if (!shot) return; + + // 提取所有镜头的有效提示词 + const prompts: string[] = shot.cells.map((c) => c.prompt).filter((p): p is string => Boolean(p)); + + if (prompts.length === 0) { + this.log("跳过分镜图生成", `分镜 ${shotId} 没有有效的镜头提示词`); + this.generatingShots.delete(shotId); + return; + } + + // 通知前端正在生成该分镜 + this.emit("shotImageGenerateProgress", { shotId, status: "generating", message: "正在调用 AI 生成宫格图片" }); + + // 根据所有镜头提示词生成宫格图片 + const gridImage = await generateImageTool( + prompts.map((p) => ({ prompt: p })), + this.scriptId, + this.projectId, + ); + + // 通知前端正在分割图片 + this.emit("shotImageGenerateProgress", { shotId, status: "splitting", message: "正在分割宫格图片为单张镜头图" }); + + // 分割宫格图片为单张镜头图片 + const imageBuffers = await imageSplitting(gridImage, prompts.length); + + // 通知前端正在保存图片 + this.emit("shotImageGenerateProgress", { shotId, status: "saving", message: `正在保存 ${imageBuffers.length} 张镜头图片` }); + + // 保存分割后的镜头图片到 OSS，并获取文件路径 + const timestamp = Date.now(); + const imagePaths: string[] = []; + + for (let i = 0; i < imageBuffers.length; i++) { + const fileName = `${this.projectId}/chat/${this.scriptId}/storyboard/shot_${shotId}_take_${i}_${timestamp}.png`; + await u.oss.writeFile(fileName, imageBuffers[i]); + const imageUrl = await u.oss.getFileUrl(fileName); + imagePaths.push(imageUrl); + + // 每保存一张镜头图片通知进度 + this.emit("shotImageGenerateProgress", { + shotId, + status: "saving", + message: `已保存 ${i + 1}/${imageBuffers.length} 张镜头图片`, + progress: Math.round(((i + 1) / imageBuffers.length) * 100), + }); + } + + // 更新每个镜头的 src 字段 + shot.cells = shot.cells.map((cell, i) => ({ + id: u.uuid(), + ...cell, + src: imagePaths[i] || cell.src, + })); + + // 生成完成后更新状态 + this.generatingShots.delete(shotId); + this.emit("shotImageGenerateComplete", { shotId, shot, imagePaths }); + this.emit("shotsUpdated", this.shots); + this.log("分镜图生成完成", `分镜 ${shotId}，共 ${imagePaths.length} 张镜头图片`); + } catch (err: any) { + this.generatingShots.delete(shotId); + this.emit("shotImageGenerateError", { shotId, error: err.message }); + this.log("分镜图生成失败", `分镜 ${shotId}: ${err.message}`); + } + } + + // ==================== 公共访问器 ==================== + + /** + * 获取当前片段数据 + */ + getSegmentsData(): Segment[] { + return this.segments; + } + + /** + * 获取当前分镜数据 + */ + getShotsData(): Shot[] { + return this.shots; + } + + // ==================== 上下文构建 ==================== + + private async buildEnvironmentContext(): Promise { + const projectInfo = await u.db("t_project").where({ id: this.projectId }).first(); + + const row = await u.db("t_outline").where({ id: this.scriptId, projectId: this.projectId }).first(); + const outline: any | null = row?.data ? JSON.parse(row.data) : null; + + // 分类提取资源名称 + const characters = outline?.characters?.map((i: any) => i.name) ?? []; + const props = outline?.props?.map((i: any) => i.name) ?? []; + const scenes = outline?.scenes?.map((i: any) => i.name) ?? []; + + const assetList = + [ + characters.length ? `【角色】${characters.join("、")}` : "", + props.length ? `【道具】${props.join("、")}` : "", + scenes.length ? `【场景】${scenes.join("、")}` : "", + ] + .filter(Boolean) + .join("\n") || "无"; + + return `<环境信息> +项目ID: ${this.projectId} +系统时间: ${new Date().toLocaleString()} + +项目名称: ${projectInfo?.name || "未知"} +项目简介: ${projectInfo?.intro || "无"} +类型: ${projectInfo?.type || "未知"} +风格: ${projectInfo?.artStyle || "未知"} +视频比例: ${projectInfo?.videoRatio || "未知"} + +资产列表: +${assetList} + +`; + } + + private buildConversationHistory(): string { + if (!this.history.length) return "无对话历史"; + return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + } + + private async buildFullContext(task: string): Promise { + const env = await this.buildEnvironmentContext(); + const history = this.buildConversationHistory(); + + return `${env} + +<对话历史> +${history} + + +<当前任务> +${task} +`; + } + + // ==================== Sub-Agent ==================== + + private createModel() { + return openAI({ + modelName: this.modelName, + configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, + }); + } + + /** + * 获取不同 Sub-Agent 可用的工具 + */ + private getSubAgentTools(agentType: AgentType) { + switch (agentType) { + case "segmentAgent": + // segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果 + return [this.getScript, this.getAssets, this.updateSegments]; + case "shotAgent": + // shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图 + return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage]; + default: + return [this.getScript]; + } + } + + /** + * 调用 Sub-Agent（流式传输） + */ + private async invokeSubAgent(agentType: AgentType, task: string): Promise { + this.emit("transfer", { to: agentType }); + this.log(`Sub-Agent 调用`, agentType); + + const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]); + const segmentAgent = promptsList.find((p) => p.code === "storyboard-segment"); + const shotAgent = promptsList.find((p) => p.code === "storyboard-shot"); + const errPrompts = "不论用户说什么，请直接输出Agent配置异常"; + const SYSTEM_PROMPTS: Record = { + segmentAgent: segmentAgent?.customValue || segmentAgent?.defaultValue || errPrompts, + shotAgent: shotAgent?.customValue || shotAgent?.defaultValue || errPrompts, + }; + + const context = await this.buildFullContext(task); + + const agent = createAgent({ + model: this.createModel(), + systemPrompt: SYSTEM_PROMPTS[agentType], + tools: this.getSubAgentTools(agentType), + }); + + const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("subAgentStream", { agent: agentType, text: block.text }); + } + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); + } + } + } + + this.emit("subAgentEnd", { agent: agentType }); + this.history.push(["ai", fullResponse]); + this.log(`Sub-Agent 完成`, agentType); + return fullResponse; + } + + private createSubAgentTool(agentType: AgentType, description: string) { + return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { + name: agentType, + description, + schema: z.object({ + taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"), + }), + }); + } + + // ==================== 主入口 ==================== + + private getAllTools() { + return [ + this.createSubAgentTool( + "segmentAgent", + "调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。", + ), + this.createSubAgentTool( + "shotAgent", + "调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。", + ), + // this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"), + this.getScript, + this.getSegments, + this.generateShotImage, + ...this.getSubAgentTools("segmentAgent"), + ...this.getSubAgentTools("shotAgent"), + ]; + } + + async call(msg: string): Promise { + console.log("模型名称:", this.modelName); + this.history.push(["user", msg]); + + const envContext = await this.buildEnvironmentContext(); + + const prompts = await u.db("t_prompts").where("code", "storyboard-main").first(); + + const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常"; + + const mainAgent = createAgent({ + model: this.createModel(), + tools: this.getAllTools(), + systemPrompt: `${envContext}\n${mainPrompts}`, + }); + const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); + + let fullResponse = ""; + + for await (const [mode, chunk] of stream) { + if (mode !== "messages") continue; + const [token] = chunk as any; + const block = token.contentBlocks?.[0]; + // 处理 AI 文本流 + if (token.type === "ai" && block?.text) { + fullResponse += block.text; + this.emit("data", block.text); + } + + // 处理 tool 调用 + if (token.type === "ai" && token.tool_calls?.length) { + for (const toolCall of token.tool_calls) { + this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); + } + } + } + + this.history.push(["assistant", fullResponse]); + this.emit("response", fullResponse); + + return fullResponse; + } +} diff --git a/src/agents/outlineScript/index.ts b/src/agents/outlineScript/index.ts index 304c7c0..60e919f 100644 --- a/src/agents/outlineScript/index.ts +++ b/src/agents/outlineScript/index.ts @@ -1,10 +1,8 @@ // @/agents/outlineScript.ts import u from "@/utils"; -import { createAgent } from "langchain"; import { EventEmitter } from "events"; -import { openAI } from "@/agents/models"; +import { tool, ModelMessage } from "ai"; import { z } from "zod"; -import { tool } from "@langchain/core/tools"; import type { DB } from "@/types/database"; // ==================== 类型定义 ==================== @@ -75,13 +73,9 @@ const episodeSchema = z.object({ export default class OutlineScript { private readonly projectId: number; readonly emitter = new EventEmitter(); - history: Array<[string, string]> = []; + history: Array = []; novelChapters: DB["t_novel"][] = []; - modelName = "gpt-4.1"; - apiKey = ""; - baseURL = ""; - constructor(projectId: number) { this.projectId = projectId; } @@ -403,123 +397,107 @@ ${formatList(ep.classicQuotes, (q) => q)} // ==================== Tool 定义：故事线 ==================== - getStoryline = tool( - async () => { + getStoryline = tool({ + title: "getStoryline", + description: "Get the weather in a location", + inputSchema: z.object({}), + execute: async () => { this.log("获取故事线"); const storyline = await this.findStoryline(); return storyline?.content ?? "当前项目暂无故事线"; }, - { - name: "getStoryline", - description: "获取当前项目的故事线内容", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); - saveStoryline = tool( - async ({ content }) => { + saveStoryline = tool({ + title: "saveStoryline", + description: "保存或更新当前项目的故事线，会覆盖已有内容", + inputSchema: z.object({ + content: z.string().describe("故事线完整内容"), + }), + execute: async ({ content }) => { this.log("保存故事线"); await this.upsertStorylineContent(content); return "故事线保存成功"; }, - { - name: "saveStoryline", - description: "保存或更新当前项目的故事线，会覆盖已有内容", - schema: z.object({ - content: z.string().describe("故事线完整内容"), - }), - verboseParsingErrors: true, - }, - ); + }); - deleteStoryline = tool( - async () => { + deleteStoryline = tool({ + title: "deleteStoryline", + description: "删除当前项目的故事线", + inputSchema: z.object({}), + execute: async () => { this.log("删除故事线"); const deleted = await this.deleteStorylineContent(); return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线"; }, - { - name: "deleteStoryline", - description: "删除当前项目的故事线", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); // ==================== Tool 定义：大纲 ==================== - getOutline = tool( - async ({ simplified = false }) => { + getOutline = tool({ + title: "getOutline", + description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容", + inputSchema: z.object({ + simplified: z.boolean().default(false).describe("是否返回简化版本"), + }), + execute: async ({ simplified }) => { this.log("获取大纲", `简化模式: ${simplified}`); return this.getOutlineText(simplified); }, - { - name: "getOutline", - description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容", - schema: z.object({ - simplified: z.boolean().default(false).describe("是否返回简化版本"), - }), - verboseParsingErrors: true, - }, - ); + }); - saveOutline = tool( - async ({ episodes, overwrite = true, startEpisode }) => { + saveOutline = tool({ + title: "saveOutline", + description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾", + inputSchema: z.object({ + episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"), + overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"), + startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"), + }), + execute: async ({ episodes, overwrite = true, startEpisode }) => { this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`); const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode); return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`; }, - { - name: "saveOutline", - description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾", - schema: z.object({ - episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"), - overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"), - startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"), - }), - verboseParsingErrors: true, - }, - ); + }); - updateOutline = tool( - async ({ id, data }) => { + updateOutline = tool({ + title: "updateOutline", + description: "更新指定ID的单集大纲内容", + inputSchema: z.object({ + id: z.number().describe("大纲ID"), + data: episodeSchema.describe("更新后的大纲数据"), + }), + execute: async ({ id, data }) => { this.log("更新大纲", `ID: ${id}`); const success = await this.updateOutlineData(id, data as EpisodeData); return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`; }, - { - name: "updateOutline", - description: "更新指定ID的单集大纲内容", - schema: z.object({ - id: z.number().describe("大纲ID"), - data: episodeSchema.describe("更新后的大纲数据"), - }), - verboseParsingErrors: true, - }, - ); + }); - deleteOutline = tool( - async ({ ids }) => { + deleteOutline = tool({ + title: "deleteOutline", + description: "根据大纲ID删除指定大纲及关联数据", + inputSchema: z.object({ + ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"), + }), + execute: async ({ ids }) => { this.log("删除大纲", `IDs: ${ids.join(", ")}`); const results = await this.deleteOutlineData(ids); const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", "); return `删除结果: ${summary}`; }, - { - name: "deleteOutline", - description: "根据大纲ID删除指定大纲及关联数据", - schema: z.object({ - ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"), - }), - verboseParsingErrors: true, - }, - ); + }); // ==================== Tool 定义：章节 ==================== - getChapter = tool( - async ({ chapterNumbers }) => { + getChapter = tool({ + title: "getChapter", + description: "根据章节编号获取小说章节的完整原文内容，支持批量获取", + inputSchema: z.object({ + chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"), + }), + execute: async ({ chapterNumbers }) => { this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`); const results = await Promise.all( @@ -539,36 +517,24 @@ ${formatList(ep.classicQuotes, (q) => q)} return results.join("\n\n---\n"); }, - { - name: "getChapter", - description: "根据章节编号获取小说章节的完整原文内容，支持批量获取", - schema: z.object({ - chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"), - }), - verboseParsingErrors: true, - }, - ); + }); // ==================== Tool 定义：资产 ==================== - generateAssets = tool( - async () => { + generateAssets = tool({ + title: "generateAssets", + description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余", + inputSchema: z.object({}), + execute: async () => { this.log("生成资产"); const stats = await this.generateAssetsFromOutlines(); if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) { return "当前项目没有大纲数据，无法生成资产"; } - return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`; }, - { - name: "generateAssets", - description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余", - schema: z.object({}), - verboseParsingErrors: true, - }, - ); + }); // ==================== 上下文构建 ==================== @@ -606,7 +572,7 @@ ${this.getChapterContext()} private buildConversationHistory(): string { if (!this.history.length) return "无对话历史"; - return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n"); + return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n"); } private async buildFullContext(task: string): Promise { @@ -627,14 +593,14 @@ ${task} // ==================== Sub-Agent ==================== private getSubAgentTools() { - return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline]; - } - - private createModel() { - return openAI({ - modelName: this.modelName, - configuration: { apiKey: this.apiKey, baseURL: this.baseURL }, - }); + return { + getChapter: this.getChapter, + getStoryline: this.getStoryline, + saveStoryline: this.saveStoryline, + getOutline: this.getOutline, + saveOutline: this.saveOutline, + updateOutline: this.updateOutline, + }; } /** @@ -657,74 +623,69 @@ ${task} const context = await this.buildFullContext(task); - const agent = createAgent({ - model: this.createModel(), - systemPrompt: SYSTEM_PROMPTS[agentType], + const { fullStream } = await u.ai.text.stream({ + system: SYSTEM_PROMPTS[agentType], tools: this.getSubAgentTools(), + messages: [{ role: "user", content: context }], + maxStep: 100, }); - const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] }); - let fullResponse = ""; - - for await (const [mode, chunk] of stream) { - if (mode !== "messages") continue; - - const [token] = chunk as any; - const block = token.contentBlocks?.[0]; - - // 处理 AI 文本流 - if (token.type === "ai" && block?.text) { - fullResponse += block.text; - this.emit("subAgentStream", { agent: agentType, text: block.text }); + for await (const item of fullStream) { + if (item.type == "tool-call") { + this.emit("toolCall", { agent: "main", name: item.title, args: null }); } - - // 处理 tool 调用 - if (token.type === "ai" && token.tool_calls?.length) { - for (const toolCall of token.tool_calls) { - this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args }); - } + if (item.type == "text-delta") { + fullResponse += item.text; + this.emit("subAgentStream", { agent: agentType, text: item.text }); } } this.emit("subAgentEnd", { agent: agentType }); - this.history.push(["ai", fullResponse]); + this.history.push({ + role: "assistant", + content: fullResponse, + }); this.log(`Sub-Agent 完成`, agentType); return fullResponse ?? `${agentType}已完成任务`; } private createSubAgentTool(agentType: AgentType, description: string) { - return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), { - name: agentType, + return tool({ + title: agentType, description, - schema: z.object({ + inputSchema: z.object({ taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"), }), + execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), }); } // ==================== 主入口 ==================== private getAllTools() { - return [ - this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"), - this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"), - this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"), - this.getChapter, - this.getStoryline, - this.saveStoryline, - this.deleteStoryline, - this.getOutline, - this.saveOutline, - this.updateOutline, - this.deleteOutline, - this.generateAssets, - ]; + return { + AI1: this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"), + AI2: this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"), + director: this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"), + getChapter: this.getChapter, + getStoryline: this.getStoryline, + saveStoryline: this.saveStoryline, + deleteStoryline: this.deleteStoryline, + getOutline: this.getOutline, + saveOutline: this.saveOutline, + updateOutline: this.updateOutline, + deleteOutline: this.deleteOutline, + generateAssets: this.generateAssets, + }; } async call(msg: string): Promise { - this.history.push(["user", msg]); + this.history.push({ + role: "user", + content: msg, + }); const envContext = await this.buildEnvironmentContext(); @@ -732,36 +693,28 @@ ${task} const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常"; - const mainAgent = createAgent({ - model: this.createModel(), + const { fullStream } = await u.ai.text.stream({ + system: `${envContext}\n${mainPrompts}`, tools: this.getAllTools(), - systemPrompt: `${envContext}\n${mainPrompts}`, + messages: this.history, + maxStep: 100, }); - const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] }); let fullResponse = ""; - - for await (const [mode, chunk] of stream) { - if (mode !== "messages") continue; - - const [token] = chunk as any; - const block = token.contentBlocks?.[0]; - - // 处理 AI 文本流 - if (token.type === "ai" && block?.text) { - fullResponse += block.text; - this.emit("data", block.text); + for await (const item of fullStream) { + if (item.type == "tool-call") { + this.emit("toolCall", { agent: "main", name: item.title, args: null }); } - - // 处理 tool 调用 - if (token.type === "ai" && token.tool_calls?.length) { - for (const toolCall of token.tool_calls) { - this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args }); - } + if (item.type == "text-delta") { + fullResponse += item.text; + this.emit("data", item.text); } } + this.history.push({ + role: "assistant", + content: fullResponse, + }); - this.history.push(["assistant", fullResponse]); this.emit("response", fullResponse); return fullResponse; diff --git a/src/routes/outline/agentsOutline.ts b/src/routes/outline/agentsOutline.ts index 2059272..7ca6032 100644 --- a/src/routes/outline/agentsOutline.ts +++ b/src/routes/outline/agentsOutline.ts @@ -8,7 +8,6 @@ expressWs(router as unknown as Application); router.ws("/", async (ws, req) => { let agent: OutlineScript; - const config = await u.getConfig("language"); const projectId = req.query.projectId; if (!projectId || typeof projectId !== "string") { @@ -19,10 +18,6 @@ router.ws("/", async (ws, req) => { agent = new OutlineScript(Number(projectId)); - agent.modelName = config.model ?? ""; - agent.baseURL = config.baseURL ?? ""; - agent.apiKey = config.apiKey ?? ""; - // const existing = await u // .db("t_chatHistory") // .where({ projectId: Number(projectId) })