import { EventEmitter } from "events"; import { o_novel } from "@/types/database"; import u from "@/utils"; import { stripThink } from "@/utils/stripThink"; export interface EventType { id: number; event: string; } /* 文本数据清洗 * @param textData 需要清洗的文本 * @param windowSize 每组数量 默认5 * @param overlap 交叠数量 默认1 * @returns {totalCharacter:所有人物角色卡,totalEvent:所有事件} */ class CleanNovel { emitter: EventEmitter; /** 最大并发数 */ concurrency: number; constructor(concurrency: number = 5) { this.emitter = new EventEmitter(); this.concurrency = concurrency; } private async processChapter(novel: o_novel): Promise { try { const prompt = await u.getPrompts("event"); const promptData = await u.db("o_prompt").where("type", "eventExtraction").first(); let eventExtraction = "" as string | undefined; if (promptData && promptData.useData) { eventExtraction = promptData.useData; } else { eventExtraction = promptData?.data ?? undefined; } const resData = await u.Ai.Text("universalAi").invoke({ system: eventExtraction ? JSON.stringify(eventExtraction) : (prompt as string), messages: [ { role: "user", content: "请根据以下小说章节数:" + novel.chapterIndex + "小说章节券:" + novel.reel + "小说章节名称:" + novel.chapter + "、小说章节内容生成事件摘要:\n" + novel.chapterData!, }, ], }); const preData = stripThink(resData.text); this.emitter.emit("item", { id: novel.id, event: preData }); return { id: novel.id!, event: preData }; } catch (e) { this.emitter.emit("item", { id: novel.id, event: null, errorReason: u.error(e).message }); return null; } } async start(allChapters: o_novel[], projectId: number): Promise { const totalEvent: EventType[] = []; // 并发控制:通过信号量限制同时执行的任务数 let running = 0; let index = 0; const results: Promise[] = []; const runNext = (): Promise => { if (index >= allChapters.length) return Promise.resolve(); const novel = allChapters[index++]; running++; return this.processChapter(novel).then((result) => { if (result) totalEvent.push(result); running--; return runNext(); }); }; // 启动最多 concurrency 个并发任务 const workers = Array.from({ length: Math.min(this.concurrency, allChapters.length) }, () => runNext()); await Promise.all(workers); return totalEvent; } } export default CleanNovel;