import { EventEmitter } from "events"; import { o_novel } from "@/types/database"; import { useSkill } from "@/utils/agent/skillsTools"; import u from "@/utils"; export interface EventType { id: number; event: string; } /* 文本数据清洗 * @param textData 需要清洗的文本 * @param windowSize 每组数量 默认5 * @param overlap 交叠数量 默认1 * @returns {totalCharacter:所有人物角色卡,totalEvent:所有事件} */ class CleanNovel { emitter: EventEmitter; /** 最大并发数 */ concurrency: number; constructor(concurrency: number = 5) { this.emitter = new EventEmitter(); this.concurrency = concurrency; } private async processChapter(novel: o_novel, intansce: ReturnType): Promise { try { const prompt = await u.getPrompts("event"); const data = await u.db("o_prompt").where("name", "eventExtraction").first("prompt"); const resData = await intansce.invoke({ system: data ? JSON.stringify(data.prompt) : (prompt as string), messages: [ { role: "user", content: "请根据以下小说章节数:" + novel.chapterIndex + "小说章节券:" + novel.reel + "小说章节名称:" + novel.chapter + "、小说章节内容生成事件摘要:\n" + novel.chapterData!, }, ], }); const preData = resData.text; this.emitter.emit("item", { id: novel.id, event: preData }); return { id: novel.id!, event: preData }; } catch (e) { this.emitter.emit("item", { id: novel.id, event: null, errorReason: u.error(e).message }); return null; } } async start(allChapters: o_novel[], projectId: number): Promise { const totalEvent: EventType[] = []; const intansce = u.Ai.Text("universalAgent"); // 并发控制:通过信号量限制同时执行的任务数 let running = 0; let index = 0; const results: Promise[] = []; const runNext = (): Promise => { if (index >= allChapters.length) return Promise.resolve(); const novel = allChapters[index++]; running++; return this.processChapter(novel, intansce).then((result) => { if (result) totalEvent.push(result); running--; return runNext(); }); }; // 启动最多 concurrency 个并发任务 const workers = Array.from({ length: Math.min(this.concurrency, allChapters.length) }, () => runNext()); await Promise.all(workers); return totalEvent; } } export default CleanNovel;