完善分镜

src/agents/productionAgent/tools.ts

@@ -2,10 +2,12 @@ import { tool, Tool } from "ai";
 import { z } from "zod";
 import _ from "lodash";
 import ResTool from "@/socket/resTool";
-
+import u from "@/utils";
+import { useSkill } from "@/utils/agent/skillsTools";
+import { urlToBase64 } from "@/utils/vm";
 export const deriveAssetSchema = z.object({
   id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
-  assetsId: z.string().describe("关联的资产ID"),
+  assetsId: z.number().describe("关联的资产ID"),
   prompt: z.string().describe("生成提示词"),
   name: z.string().describe("衍生资产名称"),
   desc: z.string().describe("衍生资产描述"),
@@ -14,14 +16,15 @@ export const deriveAssetSchema = z.object({
   type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
 });
 export const assetItemSchema = z.object({
-  assetsId: z.string().describe("资产唯一标识"),
+  id: z.number().describe("资产唯一标识"),
   name: z.string().describe("资产名称"),
+  type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
+  prompt: z.string().describe("生成提示词"),
   desc: z.string().describe("资产描述"),
-  src: z.string().describe("资产资源路径"),
   derive: z.array(deriveAssetSchema).describe("衍生资产列表"),
 });
 export const storyboardSchema = z.object({
-  id: z.number().describe("分镜ID"),
+  id: z.number().optional().describe("分镜ID,未从工作区获得的分镜列表视为需要新增;如需新增则为空"),
   title: z.string().describe("分镜标题"),
   description: z.string().describe("分镜描述"),
   camera: z.string().describe("镜头信息"),
@@ -107,6 +110,35 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
       execute: async ({ value }) => {
         console.log("[tools] set_flowData assets", value);
         resTool.systemMessage("正在保存 衍生资产 数据");
+        if (value && Array.isArray(value) && value.length) {
+          for (const i of value) {
+            if (!i?.id) {
+              const [insertedId] = await u.db("o_assets").insert({
+                assetsId: null,
+                name: i.name,
+                type: i.type,
+                prompt: i.prompt,
+                describe: i.desc,
+                startTime: Date.now(),
+              });
+              i.id = insertedId;
+            }
+            if (i.derive && Array.isArray(i.derive) && i.derive.length) {
+              for (const sub of i.derive) {
+                if (sub.id) continue;
+                const [insertedId] = await u.db("o_assets").insert({
+                  assetsId: +i.id || null,
+                  name: sub.name,
+                  type: sub.type,
+                  prompt: sub.prompt,
+                  describe: sub.desc,
+                  startTime: Date.now(),
+                });
+                sub.id = insertedId;
+              }
+            }
+          }
+        }
         socket.emit("setFlowData", { key: "assets", value });
         return true;
       },
@@ -127,6 +159,28 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
       execute: async ({ value }) => {
         console.log("[tools] set_flowData storyboard", value);
         resTool.systemMessage("正在保存 分镜列表 数据...");
+        for (const item of value) {
+          if (!item.id) {
+            const [insertedId] = await u.db("o_storyboard").insert({
+              title: item.title,
+              prompt: item.prompt,
+              description: item.description,
+              filePath: item.src,
+              frameMode: item.frameMode,
+              duration: String(item.duration),
+              camera: item.camera,
+              sound: item.sound,
+              lines: item.lines,
+              state: "未生成",
+            });
+            if (item.associateAssetsIds.length) {
+              await u.db("o_assets2Storyboard").insert(item.associateAssetsIds.map((i) => ({ storyboardId: insertedId, assetId: i })));
+            }
+            item.id = insertedId;
+          }
+        }
+        console.log("%c Line:181 🍏 value", "background:#93c0a4", value);
+
         socket.emit("setFlowData", { key: "storyboard", value });
         return true;
       },
@@ -175,7 +229,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
       inputSchema: z.object({
         images: z.array(
           z.object({
-            id: z.string().describe("图片唯一标识符"),
+            id: z.number().describe("从工作区获取到的分镜id"),
             prompt: z.string().describe("图片生成提示词"),
             referenceIds: z.array(z.string()).describe("依赖的参考图id数组，无依赖填空数组[]"),
             assetIds: z.array(z.number()).optional().describe("参考的资产图"),
@@ -184,18 +238,116 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
       }),
       execute: async ({ images }) => {
         console.log("[tools] generated_assets", images);
-        return new Promise((resolve) => socket.emit("generatedAssets", { images }, (res: any) => resolve(res)));
+
+        const skill = await useSkill("universal-agent");
+        for (const item of images) {
+          resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`);
+          await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" });
+          u.Ai.Image("1:doubao-seedream-4-5-251128")
+            .run({
+              systemPrompt: skill.prompt,
+              prompt: item.prompt,
+              imageBase64: await getAssetsImageBase64(item.assetIds ?? []),
+              size: "2K",
+              aspectRatio: "16:9",
+              taskClass: "资产图片生成",
+              describe: "生成图片",
+              relatedObjects: "hhhh",
+              projectId: resTool.data.projectId,
+            })
+            .then(async (imageCls) => {
+              const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`;
+              await imageCls.save(savePath);
+              const obj = {
+                ...item,
+                id: item.id,
+                src: await u.oss.getFileUrl(savePath),
+                state: "已完成",
+              };
+              await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath });
+              resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`);
+              socket.emit("setFlowData", { key: "setStoryboardImage", value: obj });
+            });
+
+          socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } });
+        }
+        return "分镜图片生成中";
       },
     }),
     generate_assets_images: tool({
-      description: "生成分镜图",
+      description: `
+      生成 资产图片 不区分原资产于衍生资产
+      参数说明：
+      - images: 图片任务数组
+        - assetId: 资产id
+        - prompt: 图片生成提示词
+      示例：
+      images:[
+        {assetId: 1, prompt: "一张猫的图片"}
+      ]
+      `,
       inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }),
       execute: async ({ images }) => {
+        const skill = await useSkill("universal-agent");
+        for (const item of images) {
+          const [imageId] = await u.db("o_image").insert({
+            assetsId: item.assetId,
+            model: "1:doubao-seedream-4-5-251128",
+            state: "生成中",
+            resolution: "2K",
+          });
+          u.Ai.Image("1:doubao-seedream-4-5-251128")
+            .run({
+              systemPrompt: skill.prompt,
+              prompt: item.prompt,
+              imageBase64: [],
+              size: "2K",
+              aspectRatio: "16:9",
+              taskClass: "资产图片生成",
+              describe: "生成图片",
+              relatedObjects: "hhhh",
+              projectId: resTool.data.projectId,
+            })
+            .then(async (imageCls) => {
+              const savePath = `/${resTool.data.projectId}/assets/${u.uuid()}.jpg`;
+              await imageCls.save(savePath);
+              const obj = {
+                ...item,
+                id: item.assetId,
+                src: await u.oss.getFileUrl(savePath),
+                state: "生成成功",
+              };
+              await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
+              socket.emit("setFlowData", { key: "setAssetsImage", value: obj });
+            });
+
+          socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } });
+        }
         console.log("[tools] generate_assets_images", images);
-        return new Promise((resolve) => socket.emit("generateAssetsImages", { images }, (res: any) => resolve(res)));
+        return "资产生成中";
       },
     }),
   };
 
   return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools;
 };
+
+async function getAssetsImageBase64(imageIds: number[]) {
+  if (imageIds.length === 0) return [];
+  const imagePaths = await u
+    .db("o_assets")
+    .leftJoin("o_image", "o_assets.imageId", "o_image.id")
+    .whereIn("o_assets.id", imageIds)
+    .select("o_assets.id", "o_image.filePath");
+  if (!imagePaths.length) return [];
+  const imageUrls = await Promise.all(
+    imagePaths.map(async (i) => {
+      if (i.filePath) {
+        return await urlToBase64(await u.oss.getFileUrl(i.filePath));
+      } else {
+        return null;
+      }
+    }),
+  );
+  return imageUrls.filter(Boolean) as string[];
+}

src/lib/initDB.ts

@@ -303,6 +303,8 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
         table.text("frameMode");
         table.text("camera");
         table.text("sound");
+        table.text("state");
+        table.text("reason");
         table.integer("createTime");
         table.primary(["id"]);
         table.unique(["id"]);

src/routes/production/getFlowData.ts

@@ -21,17 +21,21 @@ export default router.post(
       .select("data")
       .first();
 
-    const scriptData = await u.db("o_script").where("projectId", projectId).first();
+    const scriptData = await u.db("o_script").where("projectId", projectId).where("id", episodesId).first();
-
+    const scriptAssets = await u.db("o_scriptAssets").where("scriptId", episodesId);
+    const assetIds = scriptAssets.map((i) => i.assetId);
     const assetsData = await u
       .db("o_assets")
       .leftJoin("o_image", "o_assets.imageId", "o_image.id")
       .select("o_assets.*", "o_image.filePath")
+      .where("o_assets.id", "in", assetIds)
+      .whereNull("o_assets.assetsId")
       .where("o_assets.projectId", projectId);
     let childAssetsData = await u
       .db("o_assets")
       .leftJoin("o_image", "o_assets.imageId", "o_image.id")
       .select("o_assets.*", "o_image.filePath")
+      .where("o_assets.id", "in", assetIds)
       .where("o_assets.projectId", projectId)
       .whereNotNull("o_assets.assetsId");
 
@@ -43,6 +47,8 @@ export default router.post(
           assetsData.map(async (item) => ({
             assetsId: item.id,
             name: item.name ?? "",
+            type: item.type ?? "",
+            prompt: item.prompt ?? "",
             desc: item.describe ?? "",
             src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
             derive: await Promise.all(

src/routes/production/workbench/getChatLines.ts

@@ -35,6 +35,7 @@ async function getLines(prompt: string) {
   const resText = await u.Ai.Text("universalAgent").invoke({
     system: skill.prompt,
     messages: [{ role: "user", content: prompt }],
+    tools: skill.tools,
     output: Output.array({
       element: z.object({
         lines: z.string().describe("台词内容"),

src/socket/routes/productionAgent.ts

@@ -35,7 +35,9 @@ export default (nsp: Namespace) => {
 
     console.log("[productionAgent] 已连接:", socket.id);
 
-    const resTool = new ResTool(socket);
+    const resTool = new ResTool(socket, {
+      projectId: socket.handshake.auth.projectId,
+    });
     let abortController: AbortController | null = null;
 
     socket.on("message", async (text: string) => {

src/types/database.d.ts

@@ -1,6 +1,39 @@
-// @db-hash 83c8dadf13c2aee689597b709a690870
+// @db-hash 08cfec7bed045942e10f27f6d3f34623
 //该文件由脚本自动生成，请勿手动修改
 
+export interface _o_storyboard_old_20260324 {
+  'camera'?: string | null;
+  'createTime'?: number | null;
+  'description'?: string | null;
+  'duration'?: string | null;
+  'filePath'?: string | null;
+  'frameMode'?: string | null;
+  'id'?: number;
+  'mode'?: string | null;
+  'model'?: string | null;
+  'prompt'?: string | null;
+  'resolution'?: string | null;
+  'scriptId'?: number | null;
+  'sound'?: string | null;
+  'title'?: string | null;
+}
+export interface _o_storyboard_old_20260324_1 {
+  'camera'?: string | null;
+  'createTime'?: number | null;
+  'description'?: string | null;
+  'duration'?: string | null;
+  'filePath'?: string | null;
+  'frameMode'?: string | null;
+  'id'?: number;
+  'lines'?: string | null;
+  'mode'?: string | null;
+  'model'?: string | null;
+  'prompt'?: string | null;
+  'resolution'?: string | null;
+  'scriptId'?: number | null;
+  'sound'?: string | null;
+  'title'?: string | null;
+}
 export interface memories {
   'content': string;
   'createTime': number;
@@ -130,12 +163,15 @@ export interface o_storyboard {
   'filePath'?: string | null;
   'frameMode'?: string | null;
   'id'?: number;
+  'lines'?: string | null;
   'mode'?: string | null;
   'model'?: string | null;
   'prompt'?: string | null;
+  'reason'?: string | null;
   'resolution'?: string | null;
   'scriptId'?: number | null;
   'sound'?: string | null;
+  'state'?: string | null;
   'title'?: string | null;
 }
 export interface o_storyboardFlow {
@@ -195,6 +231,8 @@ export interface o_videoConfig {
 }
 
 export interface DB {
+  "_o_storyboard_old_20260324": _o_storyboard_old_20260324;
+  "_o_storyboard_old_20260324_1": _o_storyboard_old_20260324_1;
   "memories": memories;
   "o_agentDeploy": o_agentDeploy;
   "o_agentWorkData": o_agentWorkData;