diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index d8a70f6..ada4c87 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -90,7 +90,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => modelName: "", vendorId: null, key: "assetsAi", - name: "资产AI", + name: "资产Agent", desc: "根据角色和场景要素，生成精准的素材提示词", disabled: false, }, @@ -99,7 +99,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => modelName: "", vendorId: null, key: "polishingAi", - name: "润色AI", + name: "润色Agent", desc: "将大纲扩展为完整剧本脚本，包含对话和场景描写", disabled: false, }, @@ -108,7 +108,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => modelName: "", vendorId: null, key: "eventExtractAi", - name: "事件提取AI", + name: "事件提取Agent", desc: "从小说原文中提取事件，生成事件列表和事件关系", disabled: false, }, diff --git a/src/router.ts b/src/router.ts index bd11f73..bddd2d8 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,126 +1,130 @@ -// @routes-hash ce37531033f53b14b8627a6d9bd1314a +// @routes-hash 45e23a536a13265ff62fe92b1cb71fe3 import { Express } from "express"; import route1 from "./routes/agents/clearMemory"; import route2 from "./routes/agents/getMemory"; -import route3 from "./routes/artStyle/getArtStyle"; -import route4 from "./routes/assets/addAssets"; -import route5 from "./routes/assets/batchDelete"; -import route6 from "./routes/assets/batchGenerationData"; -import route7 from "./routes/assets/delAssets"; -import route8 from "./routes/assets/getAssetsApi"; -import route9 from "./routes/assets/getImage"; -import route10 from "./routes/assets/saveAssets"; -import route11 from "./routes/assets/updateAssets"; -import route12 from "./routes/assets/uploadClip"; -import route13 from "./routes/assetsGenerate/generateAssets"; -import route14 from "./routes/assetsGenerate/polishAssetsPrompt"; -import route15 from "./routes/cornerScape/getAllAssets"; -import route16 from "./routes/general/generalStatistics"; -import route17 from "./routes/general/getSingleProject"; -import route18 from "./routes/general/updateProject"; -import route19 from "./routes/login/login"; -import route20 from "./routes/migrate/migrateData"; -import route21 from "./routes/modelSelect/getModelDetail"; -import route22 from "./routes/modelSelect/getModelList"; -import route23 from "./routes/novel/addNovel"; -import route24 from "./routes/novel/batchDeleteNovel"; -import route25 from "./routes/novel/delNovel"; -import route26 from "./routes/novel/event/batchDeleteEvent"; -import route27 from "./routes/novel/event/deletEvent"; -import route28 from "./routes/novel/event/generateEvents"; -import route29 from "./routes/novel/event/getEvent"; -import route30 from "./routes/novel/getNovel"; -import route31 from "./routes/novel/updateNovel"; -import route32 from "./routes/other/deleteAllData"; -import route33 from "./routes/other/getCaptcha"; -import route34 from "./routes/production/getProductionData"; -import route35 from "./routes/project/addProject"; -import route36 from "./routes/project/delProject"; -import route37 from "./routes/project/editProject"; -import route38 from "./routes/project/getProject"; -import route39 from "./routes/script/addScript"; -import route40 from "./routes/script/delScript"; -import route41 from "./routes/script/getScrptApi"; -import route42 from "./routes/script/updateScript"; -import route43 from "./routes/setting/agentDeploy/deployAgentModel"; -import route44 from "./routes/setting/agentDeploy/getAgentDeploy"; -import route45 from "./routes/setting/agentDeploy/updateKey"; -import route46 from "./routes/setting/dbConfig/clearData"; -import route47 from "./routes/setting/getTextModel"; -import route48 from "./routes/setting/loginConfig/getUser"; -import route49 from "./routes/setting/loginConfig/updateUserPwd"; -import route50 from "./routes/setting/memoryConfig/getMemory"; -import route51 from "./routes/setting/memoryConfig/sureMemory"; -import route52 from "./routes/setting/vendorConfig/addVendor"; -import route53 from "./routes/setting/vendorConfig/deleteVendor"; -import route54 from "./routes/setting/vendorConfig/getVendorList"; -import route55 from "./routes/setting/vendorConfig/modelTest"; -import route56 from "./routes/setting/vendorConfig/updateVendor"; -import route57 from "./routes/task/getMyTaskApi"; -import route58 from "./routes/task/getTaskCategories"; -import route59 from "./routes/task/taskDetails"; -import route60 from "./routes/test/test"; +import route3 from "./routes/agents/productionAgent"; +import route4 from "./routes/artStyle/getArtStyle"; +import route5 from "./routes/assets/addAssets"; +import route6 from "./routes/assets/batchDelete"; +import route7 from "./routes/assets/batchGenerationData"; +import route8 from "./routes/assets/delAssets"; +import route9 from "./routes/assets/getAssetsApi"; +import route10 from "./routes/assets/getImage"; +import route11 from "./routes/assets/saveAssets"; +import route12 from "./routes/assets/updateAssets"; +import route13 from "./routes/assets/uploadClip"; +import route14 from "./routes/assetsGenerate/generateAssets"; +import route15 from "./routes/assetsGenerate/polishAssetsPrompt"; +import route16 from "./routes/cornerScape/getAllAssets"; +import route17 from "./routes/general/generalStatistics"; +import route18 from "./routes/general/getSingleProject"; +import route19 from "./routes/general/updateProject"; +import route20 from "./routes/login/login"; +import route21 from "./routes/migrate/migrateData"; +import route22 from "./routes/modelSelect/getModelDetail"; +import route23 from "./routes/modelSelect/getModelList"; +import route24 from "./routes/novel/addNovel"; +import route25 from "./routes/novel/batchDeleteNovel"; +import route26 from "./routes/novel/delNovel"; +import route27 from "./routes/novel/event/batchDeleteEvent"; +import route28 from "./routes/novel/event/deletEvent"; +import route29 from "./routes/novel/event/generateEvents"; +import route30 from "./routes/novel/event/getEvent"; +import route31 from "./routes/novel/getNovel"; +import route32 from "./routes/novel/updateNovel"; +import route33 from "./routes/other/deleteAllData"; +import route34 from "./routes/other/getCaptcha"; +import route35 from "./routes/production/getProductionData"; +import route36 from "./routes/production/workbench/getVideoModelDetail"; +import route37 from "./routes/project/addProject"; +import route38 from "./routes/project/delProject"; +import route39 from "./routes/project/editProject"; +import route40 from "./routes/project/getProject"; +import route41 from "./routes/script/addScript"; +import route42 from "./routes/script/delScript"; +import route43 from "./routes/script/getScrptApi"; +import route44 from "./routes/script/updateScript"; +import route45 from "./routes/setting/agentDeploy/deployAgentModel"; +import route46 from "./routes/setting/agentDeploy/getAgentDeploy"; +import route47 from "./routes/setting/agentDeploy/updateKey"; +import route48 from "./routes/setting/dbConfig/clearData"; +import route49 from "./routes/setting/getTextModel"; +import route50 from "./routes/setting/loginConfig/getUser"; +import route51 from "./routes/setting/loginConfig/updateUserPwd"; +import route52 from "./routes/setting/memoryConfig/getMemory"; +import route53 from "./routes/setting/memoryConfig/sureMemory"; +import route54 from "./routes/setting/vendorConfig/addVendor"; +import route55 from "./routes/setting/vendorConfig/deleteVendor"; +import route56 from "./routes/setting/vendorConfig/getVendorList"; +import route57 from "./routes/setting/vendorConfig/modelTest"; +import route58 from "./routes/setting/vendorConfig/updateVendor"; +import route59 from "./routes/task/getMyTaskApi"; +import route60 from "./routes/task/getTaskCategories"; +import route61 from "./routes/task/taskDetails"; +import route62 from "./routes/test/test"; export default async (app: Express) => { app.use("/api/agents/clearMemory", route1); app.use("/api/agents/getMemory", route2); - app.use("/api/artStyle/getArtStyle", route3); - app.use("/api/assets/addAssets", route4); - app.use("/api/assets/batchDelete", route5); - app.use("/api/assets/batchGenerationData", route6); - app.use("/api/assets/delAssets", route7); - app.use("/api/assets/getAssetsApi", route8); - app.use("/api/assets/getImage", route9); - app.use("/api/assets/saveAssets", route10); - app.use("/api/assets/updateAssets", route11); - app.use("/api/assets/uploadClip", route12); - app.use("/api/assetsGenerate/generateAssets", route13); - app.use("/api/assetsGenerate/polishAssetsPrompt", route14); - app.use("/api/cornerScape/getAllAssets", route15); - app.use("/api/general/generalStatistics", route16); - app.use("/api/general/getSingleProject", route17); - app.use("/api/general/updateProject", route18); - app.use("/api/login/login", route19); - app.use("/api/migrate/migrateData", route20); - app.use("/api/modelSelect/getModelDetail", route21); - app.use("/api/modelSelect/getModelList", route22); - app.use("/api/novel/addNovel", route23); - app.use("/api/novel/batchDeleteNovel", route24); - app.use("/api/novel/delNovel", route25); - app.use("/api/novel/event/batchDeleteEvent", route26); - app.use("/api/novel/event/deletEvent", route27); - app.use("/api/novel/event/generateEvents", route28); - app.use("/api/novel/event/getEvent", route29); - app.use("/api/novel/getNovel", route30); - app.use("/api/novel/updateNovel", route31); - app.use("/api/other/deleteAllData", route32); - app.use("/api/other/getCaptcha", route33); - app.use("/api/production/getProductionData", route34); - app.use("/api/project/addProject", route35); - app.use("/api/project/delProject", route36); - app.use("/api/project/editProject", route37); - app.use("/api/project/getProject", route38); - app.use("/api/script/addScript", route39); - app.use("/api/script/delScript", route40); - app.use("/api/script/getScrptApi", route41); - app.use("/api/script/updateScript", route42); - app.use("/api/setting/agentDeploy/deployAgentModel", route43); - app.use("/api/setting/agentDeploy/getAgentDeploy", route44); - app.use("/api/setting/agentDeploy/updateKey", route45); - app.use("/api/setting/dbConfig/clearData", route46); - app.use("/api/setting/getTextModel", route47); - app.use("/api/setting/loginConfig/getUser", route48); - app.use("/api/setting/loginConfig/updateUserPwd", route49); - app.use("/api/setting/memoryConfig/getMemory", route50); - app.use("/api/setting/memoryConfig/sureMemory", route51); - app.use("/api/setting/vendorConfig/addVendor", route52); - app.use("/api/setting/vendorConfig/deleteVendor", route53); - app.use("/api/setting/vendorConfig/getVendorList", route54); - app.use("/api/setting/vendorConfig/modelTest", route55); - app.use("/api/setting/vendorConfig/updateVendor", route56); - app.use("/api/task/getMyTaskApi", route57); - app.use("/api/task/getTaskCategories", route58); - app.use("/api/task/taskDetails", route59); - app.use("/api/test/test", route60); + app.use("/api/agents/productionAgent", route3); + app.use("/api/artStyle/getArtStyle", route4); + app.use("/api/assets/addAssets", route5); + app.use("/api/assets/batchDelete", route6); + app.use("/api/assets/batchGenerationData", route7); + app.use("/api/assets/delAssets", route8); + app.use("/api/assets/getAssetsApi", route9); + app.use("/api/assets/getImage", route10); + app.use("/api/assets/saveAssets", route11); + app.use("/api/assets/updateAssets", route12); + app.use("/api/assets/uploadClip", route13); + app.use("/api/assetsGenerate/generateAssets", route14); + app.use("/api/assetsGenerate/polishAssetsPrompt", route15); + app.use("/api/cornerScape/getAllAssets", route16); + app.use("/api/general/generalStatistics", route17); + app.use("/api/general/getSingleProject", route18); + app.use("/api/general/updateProject", route19); + app.use("/api/login/login", route20); + app.use("/api/migrate/migrateData", route21); + app.use("/api/modelSelect/getModelDetail", route22); + app.use("/api/modelSelect/getModelList", route23); + app.use("/api/novel/addNovel", route24); + app.use("/api/novel/batchDeleteNovel", route25); + app.use("/api/novel/delNovel", route26); + app.use("/api/novel/event/batchDeleteEvent", route27); + app.use("/api/novel/event/deletEvent", route28); + app.use("/api/novel/event/generateEvents", route29); + app.use("/api/novel/event/getEvent", route30); + app.use("/api/novel/getNovel", route31); + app.use("/api/novel/updateNovel", route32); + app.use("/api/other/deleteAllData", route33); + app.use("/api/other/getCaptcha", route34); + app.use("/api/production/getProductionData", route35); + app.use("/api/production/workbench/getVideoModelDetail", route36); + app.use("/api/project/addProject", route37); + app.use("/api/project/delProject", route38); + app.use("/api/project/editProject", route39); + app.use("/api/project/getProject", route40); + app.use("/api/script/addScript", route41); + app.use("/api/script/delScript", route42); + app.use("/api/script/getScrptApi", route43); + app.use("/api/script/updateScript", route44); + app.use("/api/setting/agentDeploy/deployAgentModel", route45); + app.use("/api/setting/agentDeploy/getAgentDeploy", route46); + app.use("/api/setting/agentDeploy/updateKey", route47); + app.use("/api/setting/dbConfig/clearData", route48); + app.use("/api/setting/getTextModel", route49); + app.use("/api/setting/loginConfig/getUser", route50); + app.use("/api/setting/loginConfig/updateUserPwd", route51); + app.use("/api/setting/memoryConfig/getMemory", route52); + app.use("/api/setting/memoryConfig/sureMemory", route53); + app.use("/api/setting/vendorConfig/addVendor", route54); + app.use("/api/setting/vendorConfig/deleteVendor", route55); + app.use("/api/setting/vendorConfig/getVendorList", route56); + app.use("/api/setting/vendorConfig/modelTest", route57); + app.use("/api/setting/vendorConfig/updateVendor", route58); + app.use("/api/task/getMyTaskApi", route59); + app.use("/api/task/getTaskCategories", route60); + app.use("/api/task/taskDetails", route61); + app.use("/api/test/test", route62); } diff --git a/src/routes/assetsGenerate/generateAssets.ts b/src/routes/assetsGenerate/generateAssets.ts index 6283434..3e1001f 100644 --- a/src/routes/assetsGenerate/generateAssets.ts +++ b/src/routes/assetsGenerate/generateAssets.ts @@ -87,18 +87,38 @@ export default router.post( try { let imagePath; let insertType; + let describe; + let relatedObjects = {}; if (type == "role") { insertType = "role"; imagePath = `/${projectId}/role/${uuidv4()}.jpg`; + describe = `生成角色图，名称：${name}，提示词：${prompt}`; + relatedObjects = { + id: id, + projectId, + type: "角色", + }; } if (type == "scene") { insertType = "scene"; imagePath = `/${projectId}/scene/${uuidv4()}.jpg`; + describe = `生成场景图，名称：${name}，提示词：${prompt}`; + relatedObjects = { + id: id, + projectId, + type: "场景", + }; } if (type == "tool") { insertType = "tool"; imagePath = `/${projectId}/props/${uuidv4()}.jpg`; + describe = `生成道具图，名称：${name}，提示词：${prompt}`; + relatedObjects = { + id: id, + projectId, + type: "道具", + }; } const aiImage = u.Ai.Image(model); @@ -108,6 +128,10 @@ export default router.post( imageBase64: base64 ? [base64] : [], size: resolution, aspectRatio: "16:9", + taskClass, + describe: describe ?? "", // 描述 + projectId, + relatedObjects: JSON.stringify(relatedObjects), // 相关对象信息，便于后续分析和追踪 }); aiImage.save(imagePath!); const imageData = await u.db("o_image").where("id", imageId).select("*").first(); diff --git a/src/routes/production/workbench/getVideoModelDetail.ts b/src/routes/production/workbench/getVideoModelDetail.ts new file mode 100644 index 0000000..fdf11f2 --- /dev/null +++ b/src/routes/production/workbench/getVideoModelDetail.ts @@ -0,0 +1,20 @@ +import express from "express"; +import u from "@/utils"; +import { z } from "zod"; +import { success } from "@/lib/responseFormat"; +import { validateFields } from "@/middleware/middleware"; +const router = express.Router(); + +export default router.post("/", async (req, res) => { + const { type } = req.body; + const vendorData = await u.db("o_vendorConfig").select("id", "models", "name"); + if (!vendorData) { + return res.status(404).send({ error: "模型未找到" }); + } + for (const item of vendorData) { + const modelsData = JSON.parse(item.models! ?? "[]"); + const filterData = modelsData.filter((item: { type: string }) => item.type === type); + if (filterData.length > 0) { + } + } +}); diff --git a/src/utils/ai.ts b/src/utils/ai.ts index a5aa20e..e3be26b 100644 --- a/src/utils/ai.ts +++ b/src/utils/ai.ts @@ -1,29 +1,29 @@ -import { generateText, streamText, stepCountIs, wrapLanguageModel } from "ai"; -import { devToolsMiddleware } from "@ai-sdk/devtools"; +import { generateText, streamText, Output, stepCountIs, ModelMessage, LanguageModel, Tool, GenerateTextResult } from "ai"; import { parse } from "best-effort-json-parser"; import axios from "axios"; import { transform } from "sucrase"; import u from "@/utils"; -type AiType = "scriptAgent" | "productionAgent" | "assetsAi" | "polishingAi" | "ttsDubbing" | "eventExtractAi"; +type AiType = "scriptAgent" | "productionAgent" | "assetsAi" | "polishingAi" | "eventExtractAi" | "ttsDubbing" | "test"; type FnName = "textRequest" | "imageRequest" | "videoRequest" | "ttsRequest"; -const AiTypeValues: AiType[] = ["scriptAgent", "productionAgent", "assetsAi", "polishingAi", "ttsDubbing", "eventExtractAi"]; -async function getVendorTemplateFn(fnName: FnName, value: AiType | `${number}:${string}`) { - let id, modelName; - const isAgent = AiTypeValues.includes(value as AiType); - if (isAgent) { +const AiTypeValues: AiType[] = ["scriptAgent", "productionAgent", "assetsAi", "polishingAi", "eventExtractAi", "ttsDubbing"]; +async function resolveModelName(value: AiType | `${number}:${string}`): Promise<`${number}:${string}`> { + if (AiTypeValues.includes(value as AiType)) { const agentDeployData = await u.db("o_agentDeploy").where("key", value).first(); if (!agentDeployData?.modelName) throw new Error(`${value}模型未配置`); - [id, modelName] = agentDeployData.modelName.split(":"); - } else { - [id, modelName] = value.split(":"); + return agentDeployData.modelName as `${number}:${string}`; } + return value as `${number}:${string}`; +} + +async function getVendorTemplateFn(fnName: FnName, modelName: `${number}:${string}`) { + const [id, name] = modelName.split(":"); const vendorConfigData = await u.db("o_vendorConfig").where("id", id).first(); if (!vendorConfigData) throw new Error(`未找到供应商配置 id=${id}`); const modelList = JSON.parse(vendorConfigData.models ?? "[]"); - const selectedModel = modelList.find((i: any) => i.modelName == modelName); - if (!selectedModel) throw new Error(`未找到模型 ${modelName} id=${id}`); + const selectedModel = modelList.find((i: any) => i.modelName == name); + if (!selectedModel) throw new Error(`未找到模型 ${name} id=${id}`); const jsCode = transform(vendorConfigData.code!, { transforms: ["typescript"] }).code; const fn = u.vm(jsCode)[fnName]; if (!fn) throw new Error(`未找到供应商配置中的函数 ${fnName} id=${id}`); @@ -31,6 +31,27 @@ async function getVendorTemplateFn(fnName: FnName, value: AiType | `${number}:${ else return (input: T) => fn(input, selectedModel); } +async function withTaskRecord( + modelKey: AiType | `${number}:${string}`, + taskClass: string, + describe: string, + relatedObjects: string, + projectId: number, + fn: (modelName: `${number}:${string}`) => Promise, +): Promise { + const modelName = await resolveModelName(modelKey); + const [id, model] = modelName.split(":"); + const taskRecord = await u.task(projectId, taskClass, model, { describe: describe, content: relatedObjects }); + try { + const result = await fn(modelName); + taskRecord(1); + return result; + } catch (e) { + taskRecord(-1, u.error(e).message); + throw e; + } +} + async function urlToBase64(url: string): Promise { const res = await axios.get(url, { responseType: "arraybuffer" }); const base64 = Buffer.from(res.data).toString("base64"); @@ -43,20 +64,19 @@ class AiText { this.AiType = AiType; } async invoke(input: Omit[0], "model">) { + const modelName = await resolveModelName(this.AiType); return generateText({ ...(input.tools && { stopWhen: stepCountIs(Object.keys(input.tools).length * 5) }), ...input, - model: await getVendorTemplateFn("textRequest", this.AiType), + model: await getVendorTemplateFn("textRequest", modelName), } as Parameters[0]); } async stream(input: Omit[0], "model">) { + const modelName = await resolveModelName(this.AiType); return streamText({ ...(input.tools && { stopWhen: stepCountIs(Object.keys(input.tools).length * 5) }), ...input, - model: wrapLanguageModel({ - model: await getVendorTemplateFn("textRequest", this.AiType), - middleware: devToolsMiddleware(), - }), + model: await getVendorTemplateFn("textRequest", modelName), } as Parameters[0]); } } @@ -67,6 +87,10 @@ interface ImageConfig { imageBase64: string[]; //输入的图片提示词 size: "1K" | "2K" | "4K"; // 图片尺寸 aspectRatio: `${number}:${number}`; // 长宽比 + taskClass: string; // 任务分类 + describe: string; // 任务描述 + relatedObjects: string; // 相关对象信息，便于后续分析和追踪 + projectId: number; // 项目ID } class AiImage { @@ -76,11 +100,12 @@ class AiImage { this.key = key; } async run(input: ImageConfig) { - const fn = await getVendorTemplateFn("imageRequest", this.key); - this.result = await fn(input); - if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); - - return this; + return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { + const fn = await getVendorTemplateFn("imageRequest", modelName); + this.result = await fn(input); + if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); + return this; + }); } async save(path: string) { await u.oss.writeFile(path, this.result); @@ -94,10 +119,12 @@ class AiVideo { this.key = key; } async run(input: ImageConfig) { - const fn = await getVendorTemplateFn("videoRequest", this.key); - this.result = await fn(input); - if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); - return this; + return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { + const fn = await getVendorTemplateFn("videoRequest", modelName); + this.result = await fn(input); + if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); + return this; + }); } async save(path: string) { await u.oss.writeFile(path, this.result); @@ -111,10 +138,12 @@ class AiAudio { this.key = key; } async run(input: ImageConfig) { - const fn = await getVendorTemplateFn("ttsRequest", this.key); - this.result = await fn(input); - if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); - return this; + return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { + const fn = await getVendorTemplateFn("ttsRequest", modelName); + this.result = await fn(input); + if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); + return this; + }); } async save(path: string) { await u.oss.writeFile(path, this.result);