Merge branch 'develop' of https://github.com/HBAI-Ltd/Toonflow-app into develop

src/agents/storyboard/index.ts

@@ -36,8 +36,13 @@ interface Shot {
   x: number;
   y: number;
   cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头
+  fragmentContent: string;
+  assetsTags: AssetsType[];
+}
+interface AssetsType {
+  type: "role" | "props" | "scene";
+  text: string;
 }
-
 // ==================== 主类 ====================
 
 export default class Storyboard {
@@ -220,11 +225,17 @@ ${sections.join("\n\n")}
           z.object({
             segmentIndex: z.number().describe("对应的片段序号"),
             prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
+            assetsTags: z.array(
+              z.object({
+                type: z.enum(["role", "props", "scene"]).describe("资源类型"),
+                text: z.string().describe("资源名称"),
+              }),
+            ),
           }),
         )
         .describe("要添加的分镜数组"),
     }),
-    execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
+    execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[]; assetsTags: AssetsType[] }> }) => {
       const added: { id: number; segmentIndex: number }[] = [];
       const skipped: number[] = [];
 
@@ -244,6 +255,8 @@ ${sections.join("\n\n")}
           x: 0,
           y: 0,
           cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })),
+          fragmentContent: this.segments[item.segmentIndex]?.description,
+          assetsTags: item.assetsTags,
         });
         added.push({ id: shotId, segmentIndex: item.segmentIndex });
       }

src/lib/initDB.ts

@@ -249,41 +249,39 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
           {
             id: 2,
             configId: null,
-
+            name: "分镜Agent图片生成",
-            name: "大纲故事线Agent",
+            key: "storyboardImage",
-            key: "outlineScriptAgent",
           },
           {
             id: 3,
             configId: null,
-
+            name: "大纲故事线Agent",
+            key: "outlineScriptAgent",
+          },
+          {
+            id: 4,
+            configId: null,
             name: "资产提示词润色",
             key: "assetsPrompt",
           },
           {
-            id: 4,
+            id: 5,
             configId: null,
             name: "资产图片生成",
             key: "assetsImage",
           },
           {
-            id: 5,
+            id: 6,
             configId: null,
             name: "剧本生成",
             key: "generateScript",
           },
           {
-            id: 6,
+            id: 7,
             configId: null,
             name: "视频提示词生成",
             key: "videoPrompt",
           },
-          {
-            id: 7,
-            configId: null,
-            name: "分镜图片生成",
-            key: "storyboardImage",
-          },
           {
             id: 8,
             configId: null,

src/routes/setting/delModel.ts

@@ -13,7 +13,7 @@ export default router.post(
   async (req, res) => {
     const { id } = req.body;
     await u.db("t_config").where("id", id).delete();
-    await u.db("t_aiModelMap").where("configId", id).delete();
+    await u.db("t_aiModelMap").where("configId", id).update("configId",null);
     res.status(200).send(success("删除成功"));
   },
 );

src/routes/setting/getAiModelMap.ts

@@ -8,6 +8,6 @@ export default router.post("/", async (req, res) => {
   const configData = await u
     .db("t_aiModelMap")
     .leftJoin("t_config", "t_aiModelMap.configId", "t_config.id")
-    .select("t_aiModelMap.name", "t_config.model", "t_aiModelMap.id");
+    .select("t_aiModelMap.name", "t_config.model", "t_aiModelMap.id", "t_aiModelMap.key");
   res.status(200).send(success(configData));
 });

src/routes/video/generateVideo.ts

@@ -5,6 +5,9 @@ import { v4 as uuidv4 } from "uuid";
 import { error, success } from "@/lib/responseFormat";
 import { validateFields } from "@/middleware/middleware";
 import { t_config } from "@/types/database";
+import sharp from "sharp";
+import fs from "fs";
+import path from "path";
 
 const router = express.Router();
 
@@ -21,32 +24,29 @@ export default router.post(
     filePath: z.array(z.string()),
     duration: z.number(),
     prompt: z.string(),
+    mode: z.enum(["startEnd", "multi", "single", "text"]),
   }),
   async (req, res) => {
-    const { type, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt } = req.body;
+    const { type, mode, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt } = req.body;
 
-    // // 参数校验
+    if (mode == "text") filePath.length = 0;
-    // if (type === "volcengine") {
+    else if (!filePath.length) {
-    //   if (duration < 4 || duration > 12) {
+      return res.status(500).send(error("请先选择图片"));
-    //     return res.status(400).send(error("视频时长需在4-12秒之间"));
+    }
-    //   }
-    //   if (!["480p", "720p", "1080p"].includes(resolution)) {
-    //     return res.status(400).send(error("视频分辨率不正确"));
-    //   }
-    // }
-
-    // if (type === "runninghub") {
-    //   if (duration !== 10 && duration !== 15) {
-    //     return res.status(400).send(error("视频时长只能是10秒或15秒"));
-    //   }
-    //   if (resolution !== "9:16" && resolution !== "16:9") {
-    //     return res.status(400).send(error("视频分辨率不正确"));
-    //   }
-    // }
     const configData = await u.db("t_videoConfig").where("id", configId).first();
+
     if (!configData) {
       return res.status(500).send(error("视频配置不存在"));
     }
+    if (configData.manufacturer == "runninghub") {
+      if (filePath.length > 1) {
+        const gridUrl = await sharpProcessingImage(filePath, projectId);
+        if (gridUrl) {
+          filePath.length = 0;
+          filePath.push(gridUrl);
+        }
+      }
+    }
 
     // 优先使用视频配置中的AI配置ID查询,查不到再使用传入的aiConfigId
     let aiConfigData = null;
@@ -63,12 +63,8 @@ export default router.post(
     // 过滤掉空值
     let fileUrl = filePath.filter((p: string) => p && p.trim() !== "");
 
-    if (fileUrl.length === 0) {
-      return res.status(400).send(error("请至少选择一张图片"));
-    }
-
     // 处理文件路径，如果是 base64 则上传到 OSS
-    if (fileUrl.length === 1) {
+    if (fileUrl.length) {
       const match = fileUrl[0].match(/base64,([A-Za-z0-9+/=]+)/);
       if (match && match.length >= 2) {
         const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
@@ -87,20 +83,21 @@ export default router.post(
       // 否则认为已经是路径
       return url;
     };
+    if (fileUrl.length) {
+      // 校验文件是否存在
+      const fileExistsResults = await Promise.all(
+        fileUrl.map(async (url: string) => {
+          const path = getPathname(url);
+          return u.oss.fileExists(path);
+        }),
+      );
 
-    // 校验文件是否存在
+      if (!fileExistsResults.every(Boolean)) {
-    const fileExistsResults = await Promise.all(
+        return res.status(400).send(error("选择分镜文件不存在"));
-      fileUrl.map(async (url: string) => {
+      }
-        const path = getPathname(url);
-        return u.oss.fileExists(path);
-      }),
-    );
-
-    if (!fileExistsResults.every(Boolean)) {
-      return res.status(400).send(error("选择分镜文件不存在"));
     }
 
-    const firstFrame = getPathname(fileUrl[0]);
+    const firstFrame = fileUrl.length ? getPathname(fileUrl[0]) : "";
     const storyboardImgs = fileUrl.map((path: string) => getPathname(path));
     const savePath = `/${projectId}/video/${uuidv4()}.mp4`;
 
@@ -137,7 +134,7 @@ async function generateVideoAsync(
   aiConfigData: t_config,
 ) {
   try {
-    const projectData = await u.db("t_project").where("id", projectId).select("artStyle").first();
+    const projectData = await u.db("t_project").where("id", projectId).select("artStyle", "videoRatio").first();
 
     // 提取路径名的辅助函数
     const getPathname = (url: string): string => {
@@ -173,7 +170,7 @@ ${prompt}
         savePath,
         prompt: inputPrompt,
         duration: duration as any,
-        aspectRatio: resolution as any,
+        aspectRatio: projectData?.videoRatio as any,
         resolution: resolution as any,
       },
       {
@@ -195,7 +192,142 @@ ${prompt}
       await u.db("t_video").where("id", videoId).update({ state: -1 });
     }
   } catch (err) {
-    console.error(`视频生成失败 videoId=${videoId}:`, err);
+    console.error(`视频生成失败 videoId=${videoId}:`, u.error(err).message);
     await u.db("t_video").where("id", videoId).update({ state: -1 });
   }
 }
+
+/**
+ * 使用sharp把图片拼接为宫格图，最多3x3，图片数量为1-9不等
+ * @param imageList - 图片路径或base64数组
+ * @returns 拼接后的图片Buffer
+ */
+async function sharpProcessingImage(imageList: string[], projectId: number): Promise<string> {
+  if (!imageList || imageList.length === 0) {
+    throw new Error("图片列表不能为空");
+  }
+
+  if (imageList.length > 9) {
+    throw new Error("图片数量不能超过9张");
+  }
+
+  // 计算网格布局：根据图片数量确定行列数
+  const count = imageList.length;
+  let cols: number, rows: number;
+
+  if (count === 1) {
+    cols = rows = 1;
+  } else if (count === 2) {
+    cols = 2;
+    rows = 1;
+  } else if (count <= 4) {
+    cols = rows = 2;
+  } else if (count <= 6) {
+    cols = 3;
+    rows = 2;
+  } else {
+    cols = rows = 3;
+  }
+
+  // 第一步：加载所有图片并获取原始尺寸
+  const loadedImages = await Promise.all(
+    imageList.map(async (imagePath) => {
+      let imageBuffer: Buffer;
+
+      // 判断是base64、URL还是文件路径
+      if (imagePath.startsWith("data:image") || imagePath.match(/^[A-Za-z0-9+/=]+$/)) {
+        // Base64格式
+        const base64Data = imagePath.replace(/^data:image\/\w+;base64,/, "");
+        imageBuffer = Buffer.from(base64Data, "base64");
+      } else if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
+        // URL格式，提取pathname后从OSS读取
+        const pathname = new URL(imagePath).pathname;
+        imageBuffer = await u.oss.getFile(pathname);
+      } else {
+        // 文件路径，直接从OSS读取
+        imageBuffer = await u.oss.getFile(imagePath);
+      }
+
+      const metadata = await sharp(imageBuffer).metadata();
+      return {
+        buffer: imageBuffer,
+        width: metadata.width || 0,
+        height: metadata.height || 0,
+      };
+    }),
+  );
+
+  // 第二步：找出所有图片中的最大宽度和高度
+  const maxWidth = Math.max(...loadedImages.map((img) => img.width));
+  const maxHeight = Math.max(...loadedImages.map((img) => img.height));
+
+  // 第三步：将所有图片调整为统一尺寸（使用contain模式保持比例，填充背景色）
+  const imageData = await Promise.all(
+    loadedImages.map(async (img) => {
+      const resizedBuffer = await sharp(img.buffer)
+        .resize(maxWidth, maxHeight, {
+          fit: "contain",
+          background: { r: 0, g: 0, b: 0, alpha: 1 }, // 黑色背景填充
+        })
+        .png()
+        .toBuffer();
+
+      return {
+        buffer: resizedBuffer,
+        width: maxWidth,
+        height: maxHeight,
+      };
+    }),
+  );
+
+  // 所有图片都是相同尺寸，直接计算画布大小
+  const cellWidth = maxWidth;
+  const cellHeight = maxHeight;
+  const canvasWidth = cols * cellWidth;
+  const canvasHeight = rows * cellHeight;
+
+  // 创建空白画布
+  const canvas = sharp({
+    create: {
+      width: canvasWidth,
+      height: canvasHeight,
+      channels: 4,
+      background: { r: 255, g: 255, b: 255, alpha: 1 },
+    },
+  });
+
+  // 准备合成操作
+  const compositeOperations = imageData.map((data, index) => {
+    const row = Math.floor(index / cols);
+    const col = index % cols;
+
+    // 计算当前图片的位置（无偏移，紧密排列）
+    const left = col * cellWidth;
+    const top = row * cellHeight;
+
+    return {
+      input: data.buffer,
+      top: top,
+      left: left,
+    };
+  });
+
+  // 合成所有图片
+  const result = await canvas.composite(compositeOperations).png().toBuffer();
+
+  // 保存图片到当前文件夹，方便查看测试效果
+  const timestamp = new Date().getTime();
+  const outputFileName = `merged_image_${timestamp}.png`;
+  const outputPath = path.join(__dirname, outputFileName);
+
+  try {
+    await fs.promises.writeFile(outputPath, result);
+  } catch (err) {
+    console.error(`❌ 保存图片失败:`, err);
+  }
+  const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
+  const buffer = Buffer.from(result, "base64");
+  await u.oss.writeFile(imagePath, buffer);
+
+  return await u.oss.getFileUrl(imagePath);
+}

src/utils/ai/image/owned/other.ts

@@ -40,7 +40,6 @@ export default async (input: ImageConfig, config: AIConfig): Promise<string> =>
     } else {
       promptData = fullPrompt + `请直接输出图片`;
     }
-    console.log("%c Line:31 🍅 promptData", "background:#2eafb0", promptData);
 
     const result = await generateText({
       model: otherProvider.languageModel(model),

src/utils/ai/text/index.ts

@@ -23,7 +23,7 @@ interface AIConfig {
 }
 
 const buildOptions = async (input: AIInput<any>, config: AIConfig = {}) => {
-  if (!config || !config?.model || !config?.apiKey || !config?.baseURL || !config?.manufacturer) throw new Error("请检查模型配置是否正确");
+  if (!config || !config?.model || !config?.apiKey || !config?.manufacturer) throw new Error("请检查模型配置是否正确");
   const { model, apiKey, baseURL, manufacturer } = { ...config };
   let owned;
   if (manufacturer == "other") {

src/utils/ai/text/modelList.ts

@@ -48,7 +48,7 @@ const modelList: Owned[] = [
   // 豆包
   {
     manufacturer: "doubao",
-    model: "doubao-seed-1-8",
+    model: "doubao-seed-1-8-251228",
     responseFormat: "schema",
     image: true,
     think: true,
@@ -57,7 +57,7 @@ const modelList: Owned[] = [
   },
   {
     manufacturer: "doubao",
-    model: "doubao-seed-1-6",
+    model: "doubao-seed-1-6-251015",
     responseFormat: "schema",
     image: true,
     think: true,
@@ -66,7 +66,7 @@ const modelList: Owned[] = [
   },
   {
     manufacturer: "doubao",
-    model: "doubao-seed-1-6-lite",
+    model: "doubao-seed-1-6-lite-251015",
     responseFormat: "schema",
     image: true,
     think: true,
@@ -75,7 +75,7 @@ const modelList: Owned[] = [
   },
   {
     manufacturer: "doubao",
-    model: "doubao-seed-1-6-flash",
+    model: "doubao-seed-1-6-flash-250828",
     responseFormat: "schema",
     image: true,
     think: true,
@@ -286,7 +286,7 @@ const modelList: Owned[] = [
 
   // Gemini
   {
-    manufacturer: "google",
+    manufacturer: "gemini",
     model: "gemini-2.5-pro",
     responseFormat: "schema",
     image: true,
@@ -295,7 +295,7 @@ const modelList: Owned[] = [
     tool: true,
   },
   {
-    manufacturer: "google",
+    manufacturer: "gemini",
     model: "gemini-2.5-flash",
     responseFormat: "schema",
     image: true,
@@ -304,7 +304,7 @@ const modelList: Owned[] = [
     tool: true,
   },
   {
-    manufacturer: "google",
+    manufacturer: "gemini",
     model: "gemini-2.0-flash",
     responseFormat: "schema",
     image: true,
@@ -313,7 +313,7 @@ const modelList: Owned[] = [
     tool: true,
   },
   {
-    manufacturer: "google",
+    manufacturer: "gemini",
     model: "gemini-2.0-flash-lite",
     responseFormat: "schema",
     image: true,
@@ -322,7 +322,7 @@ const modelList: Owned[] = [
     tool: true,
   },
   {
-    manufacturer: "google",
+    manufacturer: "gemini",
     model: "gemini-1.5-pro",
     responseFormat: "schema",
     image: true,
@@ -331,7 +331,7 @@ const modelList: Owned[] = [
     tool: true,
   },
   {
-    manufacturer: "google",
+    manufacturer: "gemini",
     model: "gemini-1.5-flash",
     responseFormat: "schema",
     image: true,

src/utils/ai/utils.ts

@@ -37,9 +37,15 @@ export const validateVideoConfig = (input: VideoConfig, config: AIConfig, custom
     throw new Error(`模型 ${config.model} 不支持多图模式`);
   }
   // 校验duration和resolution是否在支持范围内
-  const validDurationResolution = owned.durationResolutionMap.some(
+  const validDurationResolution = owned.durationResolutionMap.some((map) => {
-    (map) => map.duration.includes(input.duration) && map.resolution.includes(input.resolution as typeof map.resolution[number]),
+    const durationMatch = map.duration.includes(input.duration);
-  );
+    const resolutionMatch =
+      // 若 map.resolution 和 input.resolution 均为空，视为匹配
+      (!input.resolution && map.resolution.length === 0) ||
+      // 否则匹配 includes
+      map.resolution.includes(input.resolution as (typeof map.resolution)[number]);
+    return durationMatch && resolutionMatch;
+  });
   if (!validDurationResolution) {
     const supportedDurations = [...new Set(owned.durationResolutionMap.flatMap((m) => m.duration))].sort((a, b) => a - b);
     const supportedResolutions = [...new Set(owned.durationResolutionMap.flatMap((m) => m.resolution))];

src/utils/ai/video/index.ts

@@ -10,6 +10,7 @@ import wan from "./owned/wan";
 import runninghub from "./owned/runninghub";
 import gemini from "./owned/gemini";
 import apimart from "./owned/apimart";
+import other from "./owned/other";
 
 const modelInstance = {
   volcengine: volcengine,
@@ -19,6 +20,7 @@ const modelInstance = {
   gemini: gemini,
   runninghub: runninghub,
   apimart: apimart,
+  // other: other,
 } as const;
 
 export default async (input: VideoConfig, config?: AIConfig) => {

src/utils/ai/video/owned/other.ts

@@ -0,0 +1,59 @@
+import "../type";
+import axios from "axios";
+import sharp from "sharp";
+import FormData from "form-data";
+import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
+import { createOpenAI } from "@ai-sdk/openai";
+import { experimental_generateVideo as generateVideo } from "ai";
+export default async (input: VideoConfig, config: AIConfig) => {
+  console.log("%c Line:9 🌰 config", "background:#fca650", config);
+  console.log("%c Line:9 🍒 input", "background:#33a5ff", input);
+  if (!config.apiKey) throw new Error("缺少API Key");
+  if (!config.baseURL) throw new Error("缺少baseURL");
+  // const { owned, images, hasTextType } = validateVideoConfig(input, config);
+  const [requestUrl, queryUrl] = config.baseURL.split("|");
+
+  const authorization = `Bearer ${config.apiKey}`;
+
+  const formData = new FormData();
+  formData.append("model", config.model);
+  formData.append("prompt", input.prompt);
+  formData.append("seconds", String(input.duration));
+
+  // 根据 aspectRatio 设置 size
+  const sizeMap: Record<string, string> = {
+    "16:9": "1280x720",
+    "9:16": "720x1280",
+  };
+  formData.append("size", sizeMap[input.aspectRatio] || "1920x1080");
+  console.log("%c Line:30 🍇 sizeMap[input.aspectRatio]", "background:#93c0a4", sizeMap[input.aspectRatio]);
+  if (input.imageBase64 && input.imageBase64.length) {
+    const base64Data = input.imageBase64[0]!.replace(/^data:image\/\w+;base64,/, "");
+    const buffer = Buffer.from(base64Data, "base64");
+    formData.append("input_reference", buffer, { filename: "image.jpg", contentType: "image/jpeg" });
+  }
+
+  const body = {
+    model: "sora-2-all",
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "text",
+            text: input.prompt,
+          },
+        ],
+      },
+    ],
+  };
+  const { data } = await axios.post(
+    "https://api2.aigcbest.top/v1/chat/completions",
+    { ...body },
+    {
+      headers: { "Content-Type": "application/json", Authorization: authorization },
+    },
+  );
+  console.log("%c Line:62 🍩 data", "background:#465975", data);
+  if (data.status === "FAILED") throw new Error(`任务提交失败: ${data.errorMessage || "未知错误"}`);
+};

src/utils/ai/video/owned/runninghub.ts

@@ -14,7 +14,7 @@ export default async (input: VideoConfig, config: AIConfig) => {
     "https://www.runninghub.cn/openapi/v2/rhart-video-s/image-to-video-pro",
     "https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video",
     "https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video-pro",
-    "https://www.runninghub.cn/openapi/v2/rhart-video-s/{taskId}",
+    "https://www.runninghub.cn/openapi/v2/query",
     "https://www.runninghub.cn/openapi/v2/media/upload/binary",
   ].join("|");
 
@@ -78,9 +78,17 @@ export default async (input: VideoConfig, config: AIConfig) => {
   const { taskId } = await submitTask(submitUrl, requestBody);
 
   return await pollTask(async () => {
-    const { data } = await axios.get(queryUrl.replace("{taskId}", taskId), {
+
-      headers: { Authorization: authorization },
+    const { data } = await axios.post(
-    });
+      queryUrl,
+      {
+        taskId,
+      },
+      {
+        headers: { Authorization: authorization },
+      },
+    );
+
     if (data.status === "SUCCESS") {
       return data.results?.length ? { completed: true, url: data.results[0].url } : { completed: false, error: "任务成功但未返回视频链接" };
     }