diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index 97684c6..635e1b7 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -2,10 +2,12 @@ import { tool, Tool } from "ai"; import { z } from "zod"; import _ from "lodash"; import ResTool from "@/socket/resTool"; - +import u from "@/utils"; +import { useSkill } from "@/utils/agent/skillsTools"; +import { urlToBase64 } from "@/utils/vm"; export const deriveAssetSchema = z.object({ id: z.number().describe("衍生资产ID,如果新增则为空").optional(), - assetsId: z.string().describe("关联的资产ID"), + assetsId: z.number().describe("关联的资产ID"), prompt: z.string().describe("生成提示词"), name: z.string().describe("衍生资产名称"), desc: z.string().describe("衍生资产描述"), @@ -14,14 +16,15 @@ export const deriveAssetSchema = z.object({ type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"), }); export const assetItemSchema = z.object({ - assetsId: z.string().describe("资产唯一标识"), + id: z.number().describe("资产唯一标识"), name: z.string().describe("资产名称"), + type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"), + prompt: z.string().describe("生成提示词"), desc: z.string().describe("资产描述"), - src: z.string().describe("资产资源路径"), derive: z.array(deriveAssetSchema).describe("衍生资产列表"), }); export const storyboardSchema = z.object({ - id: z.number().describe("分镜ID"), + id: z.number().optional().describe("分镜ID,未从工作区获得的分镜列表视为需要新增;如需新增则为空"), title: z.string().describe("分镜标题"), description: z.string().describe("分镜描述"), camera: z.string().describe("镜头信息"), @@ -107,6 +110,35 @@ export default (resTool: ResTool, toolsNames?: string[]) => { execute: async ({ value }) => { console.log("[tools] set_flowData assets", value); resTool.systemMessage("正在保存 衍生资产 数据"); + if (value && Array.isArray(value) && value.length) { + for (const i of value) { + if (!i?.id) { + const [insertedId] = await u.db("o_assets").insert({ + assetsId: null, + name: i.name, + type: i.type, + prompt: i.prompt, + describe: i.desc, + startTime: Date.now(), + }); + i.id = insertedId; + } + if (i.derive && Array.isArray(i.derive) && i.derive.length) { + for (const sub of i.derive) { + if (sub.id) continue; + const [insertedId] = await u.db("o_assets").insert({ + assetsId: +i.id || null, + name: sub.name, + type: sub.type, + prompt: sub.prompt, + describe: sub.desc, + startTime: Date.now(), + }); + sub.id = insertedId; + } + } + } + } socket.emit("setFlowData", { key: "assets", value }); return true; }, @@ -127,6 +159,28 @@ export default (resTool: ResTool, toolsNames?: string[]) => { execute: async ({ value }) => { console.log("[tools] set_flowData storyboard", value); resTool.systemMessage("正在保存 分镜列表 数据..."); + for (const item of value) { + if (!item.id) { + const [insertedId] = await u.db("o_storyboard").insert({ + title: item.title, + prompt: item.prompt, + description: item.description, + filePath: item.src, + frameMode: item.frameMode, + duration: String(item.duration), + camera: item.camera, + sound: item.sound, + lines: item.lines, + state: "未生成", + }); + if (item.associateAssetsIds.length) { + await u.db("o_assets2Storyboard").insert(item.associateAssetsIds.map((i) => ({ storyboardId: insertedId, assetId: i }))); + } + item.id = insertedId; + } + } + console.log("%c Line:181 🍏 value", "background:#93c0a4", value); + socket.emit("setFlowData", { key: "storyboard", value }); return true; }, @@ -175,7 +229,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ images: z.array( z.object({ - id: z.string().describe("图片唯一标识符"), + id: z.number().describe("从工作区获取到的分镜id"), prompt: z.string().describe("图片生成提示词"), referenceIds: z.array(z.string()).describe("依赖的参考图id数组，无依赖填空数组[]"), assetIds: z.array(z.number()).optional().describe("参考的资产图"), @@ -184,18 +238,116 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }), execute: async ({ images }) => { console.log("[tools] generated_assets", images); - return new Promise((resolve) => socket.emit("generatedAssets", { images }, (res: any) => resolve(res))); + + const skill = await useSkill("universal-agent"); + for (const item of images) { + resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`); + await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" }); + u.Ai.Image("1:doubao-seedream-4-5-251128") + .run({ + systemPrompt: skill.prompt, + prompt: item.prompt, + imageBase64: await getAssetsImageBase64(item.assetIds ?? []), + size: "2K", + aspectRatio: "16:9", + taskClass: "资产图片生成", + describe: "生成图片", + relatedObjects: "hhhh", + projectId: resTool.data.projectId, + }) + .then(async (imageCls) => { + const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`; + await imageCls.save(savePath); + const obj = { + ...item, + id: item.id, + src: await u.oss.getFileUrl(savePath), + state: "已完成", + }; + await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath }); + resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`); + socket.emit("setFlowData", { key: "setStoryboardImage", value: obj }); + }); + + socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } }); + } + return "分镜图片生成中"; }, }), generate_assets_images: tool({ - description: "生成分镜图", + description: ` + 生成 资产图片 不区分原资产于衍生资产 + 参数说明： + - images: 图片任务数组 + - assetId: 资产id + - prompt: 图片生成提示词 + 示例： + images:[ + {assetId: 1, prompt: "一张猫的图片"} + ] + `, inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }), execute: async ({ images }) => { + const skill = await useSkill("universal-agent"); + for (const item of images) { + const [imageId] = await u.db("o_image").insert({ + assetsId: item.assetId, + model: "1:doubao-seedream-4-5-251128", + state: "生成中", + resolution: "2K", + }); + u.Ai.Image("1:doubao-seedream-4-5-251128") + .run({ + systemPrompt: skill.prompt, + prompt: item.prompt, + imageBase64: [], + size: "2K", + aspectRatio: "16:9", + taskClass: "资产图片生成", + describe: "生成图片", + relatedObjects: "hhhh", + projectId: resTool.data.projectId, + }) + .then(async (imageCls) => { + const savePath = `/${resTool.data.projectId}/assets/${u.uuid()}.jpg`; + await imageCls.save(savePath); + const obj = { + ...item, + id: item.assetId, + src: await u.oss.getFileUrl(savePath), + state: "生成成功", + }; + await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); + socket.emit("setFlowData", { key: "setAssetsImage", value: obj }); + }); + + socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } }); + } console.log("[tools] generate_assets_images", images); - return new Promise((resolve) => socket.emit("generateAssetsImages", { images }, (res: any) => resolve(res))); + return "资产生成中"; }, }), }; return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools; }; + +async function getAssetsImageBase64(imageIds: number[]) { + if (imageIds.length === 0) return []; + const imagePaths = await u + .db("o_assets") + .leftJoin("o_image", "o_assets.imageId", "o_image.id") + .whereIn("o_assets.id", imageIds) + .select("o_assets.id", "o_image.filePath"); + if (!imagePaths.length) return []; + const imageUrls = await Promise.all( + imagePaths.map(async (i) => { + if (i.filePath) { + return await urlToBase64(await u.oss.getFileUrl(i.filePath)); + } else { + return null; + } + }), + ); + return imageUrls.filter(Boolean) as string[]; +} diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index 84576b5..c4e47e5 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -303,6 +303,8 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => table.text("frameMode"); table.text("camera"); table.text("sound"); + table.text("state"); + table.text("reason"); table.integer("createTime"); table.primary(["id"]); table.unique(["id"]); diff --git a/src/routes/production/getFlowData.ts b/src/routes/production/getFlowData.ts index 3212f8f..2f507e0 100644 --- a/src/routes/production/getFlowData.ts +++ b/src/routes/production/getFlowData.ts @@ -21,17 +21,21 @@ export default router.post( .select("data") .first(); - const scriptData = await u.db("o_script").where("projectId", projectId).first(); - + const scriptData = await u.db("o_script").where("projectId", projectId).where("id", episodesId).first(); + const scriptAssets = await u.db("o_scriptAssets").where("scriptId", episodesId); + const assetIds = scriptAssets.map((i) => i.assetId); const assetsData = await u .db("o_assets") .leftJoin("o_image", "o_assets.imageId", "o_image.id") .select("o_assets.*", "o_image.filePath") + .where("o_assets.id", "in", assetIds) + .whereNull("o_assets.assetsId") .where("o_assets.projectId", projectId); let childAssetsData = await u .db("o_assets") .leftJoin("o_image", "o_assets.imageId", "o_image.id") .select("o_assets.*", "o_image.filePath") + .where("o_assets.id", "in", assetIds) .where("o_assets.projectId", projectId) .whereNotNull("o_assets.assetsId"); @@ -43,6 +47,8 @@ export default router.post( assetsData.map(async (item) => ({ assetsId: item.id, name: item.name ?? "", + type: item.type ?? "", + prompt: item.prompt ?? "", desc: item.describe ?? "", src: item.filePath && (await u.oss.getFileUrl(item.filePath!)), derive: await Promise.all( diff --git a/src/routes/production/workbench/getChatLines.ts b/src/routes/production/workbench/getChatLines.ts index 0f21412..29bc9a9 100644 --- a/src/routes/production/workbench/getChatLines.ts +++ b/src/routes/production/workbench/getChatLines.ts @@ -35,6 +35,7 @@ async function getLines(prompt: string) { const resText = await u.Ai.Text("universalAgent").invoke({ system: skill.prompt, messages: [{ role: "user", content: prompt }], + tools: skill.tools, output: Output.array({ element: z.object({ lines: z.string().describe("台词内容"), diff --git a/src/socket/routes/productionAgent.ts b/src/socket/routes/productionAgent.ts index a7248a6..45b3e2a 100644 --- a/src/socket/routes/productionAgent.ts +++ b/src/socket/routes/productionAgent.ts @@ -35,7 +35,9 @@ export default (nsp: Namespace) => { console.log("[productionAgent] 已连接:", socket.id); - const resTool = new ResTool(socket); + const resTool = new ResTool(socket, { + projectId: socket.handshake.auth.projectId, + }); let abortController: AbortController | null = null; socket.on("message", async (text: string) => { diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 97e7555..efdd9da 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,6 +1,39 @@ -// @db-hash 83c8dadf13c2aee689597b709a690870 +// @db-hash 08cfec7bed045942e10f27f6d3f34623 //该文件由脚本自动生成，请勿手动修改 +export interface _o_storyboard_old_20260324 { + 'camera'?: string | null; + 'createTime'?: number | null; + 'description'?: string | null; + 'duration'?: string | null; + 'filePath'?: string | null; + 'frameMode'?: string | null; + 'id'?: number; + 'mode'?: string | null; + 'model'?: string | null; + 'prompt'?: string | null; + 'resolution'?: string | null; + 'scriptId'?: number | null; + 'sound'?: string | null; + 'title'?: string | null; +} +export interface _o_storyboard_old_20260324_1 { + 'camera'?: string | null; + 'createTime'?: number | null; + 'description'?: string | null; + 'duration'?: string | null; + 'filePath'?: string | null; + 'frameMode'?: string | null; + 'id'?: number; + 'lines'?: string | null; + 'mode'?: string | null; + 'model'?: string | null; + 'prompt'?: string | null; + 'resolution'?: string | null; + 'scriptId'?: number | null; + 'sound'?: string | null; + 'title'?: string | null; +} export interface memories { 'content': string; 'createTime': number; @@ -130,12 +163,15 @@ export interface o_storyboard { 'filePath'?: string | null; 'frameMode'?: string | null; 'id'?: number; + 'lines'?: string | null; 'mode'?: string | null; 'model'?: string | null; 'prompt'?: string | null; + 'reason'?: string | null; 'resolution'?: string | null; 'scriptId'?: number | null; 'sound'?: string | null; + 'state'?: string | null; 'title'?: string | null; } export interface o_storyboardFlow { @@ -195,6 +231,8 @@ export interface o_videoConfig { } export interface DB { + "_o_storyboard_old_20260324": _o_storyboard_old_20260324; + "_o_storyboard_old_20260324_1": _o_storyboard_old_20260324_1; "memories": memories; "o_agentDeploy": o_agentDeploy; "o_agentWorkData": o_agentWorkData;