zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

视频生成 sora多图自动转换宫格，其他厂商接入视频(待测试)。/(ㄒoㄒ)/~~

Browse Source
This commit is contained in:
zhishi 2026-02-08 18:07:03 +08:00
parent 32ac82b08b
commit 12fdb3f0d1
8 changed files with 267 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/agents/outlineScript/index.ts
View File

@ -700,7 +700,6 @@ ${task}
const envContext = await this.buildEnvironmentContext();
const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first();
console.log("%c Line:703 🍭 prompts", "background:#f5ce50", prompts);
const promptConfig = await u.getPromptAi("outlineScriptAgent");
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";

17
src/agents/storyboard/index.ts
View File

@ -36,8 +36,13 @@ interface Shot {
x: number;
y: number;
cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头
fragmentContent: string;
assetsTags: AssetsType[];
}
interface AssetsType {
type: "role" | "props" | "scene";
text: string;
}
// ==================== 主类 ====================
export default class Storyboard {
@ -220,11 +225,17 @@ ${sections.join("\n\n")}
z.object({
segmentIndex: z.number().describe("对应的片段序号"),
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
assetsTags: z.array(
z.object({
type: z.enum(["role", "props", "scene"]).describe("资源类型"),
text: z.string().describe("资源名称"),
}),
),
}),
)
.describe("要添加的分镜数组"),
}),
execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[]; assetsTags: AssetsType[] }> }) => {
const added: { id: number; segmentIndex: number }[] = [];
const skipped: number[] = [];
@ -244,6 +255,8 @@ ${sections.join("\n\n")}
x: 0,
y: 0,
cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })),
fragmentContent: this.segments[item.segmentIndex]?.description,
assetsTags: item.assetsTags,
});
added.push({ id: shotId, segmentIndex: item.segmentIndex });
}

208
src/routes/video/generateVideo.ts
View File

@ -5,6 +5,9 @@ import { v4 as uuidv4 } from "uuid";
import { error, success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { t_config } from "@/types/database";
import sharp from "sharp";
import fs from "fs";
import path from "path";
const router = express.Router();
@ -21,32 +24,29 @@ export default router.post(
filePath: z.array(z.string()),
duration: z.number(),
prompt: z.string(),
mode: z.enum(["startEnd", "multi", "single", "text"]),
}),
async (req, res) => {
const { type, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt } = req.body;
const { type, mode, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt } = req.body;
// // 参数校验
// if (type === "volcengine") {
// if (duration < 4 || duration > 12) {
// return res.status(400).send(error("视频时长需在4-12秒之间"));
// }
// if (!["480p", "720p", "1080p"].includes(resolution)) {
// return res.status(400).send(error("视频分辨率不正确"));
// }
// }
// if (type === "runninghub") {
// if (duration !== 10 && duration !== 15) {
// return res.status(400).send(error("视频时长只能是10秒或15秒"));
// }
// if (resolution !== "9:16" && resolution !== "16:9") {
// return res.status(400).send(error("视频分辨率不正确"));
// }
// }
if (mode == "text") filePath.length = 0;
else if (!filePath.length) {
return res.status(500).send(error("请先选择图片"));
}
const configData = await u.db("t_videoConfig").where("id", configId).first();
if (!configData) {
return res.status(500).send(error("视频配置不存在"));
}
if (configData.manufacturer == "runninghub") {
if (filePath.length > 1) {
const gridUrl = await sharpProcessingImage(filePath, projectId);
if (gridUrl) {
filePath.length = 0;
filePath.push(gridUrl);
}
}
}
// 优先使用视频配置中的AI配置ID查询,查不到再使用传入的aiConfigId
let aiConfigData = null;
@ -63,12 +63,8 @@ export default router.post(
// 过滤掉空值
let fileUrl = filePath.filter((p: string) => p && p.trim() !== "");
if (fileUrl.length === 0) {
return res.status(400).send(error("请至少选择一张图片"));
}
// 处理文件路径，如果是 base64 则上传到 OSS
if (fileUrl.length === 1) {
if (fileUrl.length) {
const match = fileUrl[0].match(/base64,([A-Za-z0-9+/=]+)/);
if (match && match.length >= 2) {
const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
@ -87,20 +83,21 @@ export default router.post(
// 否则认为已经是路径
return url;
};
if (fileUrl.length) {
// 校验文件是否存在
const fileExistsResults = await Promise.all(
fileUrl.map(async (url: string) => {
const path = getPathname(url);
return u.oss.fileExists(path);
}),
);
// 校验文件是否存在
const fileExistsResults = await Promise.all(
fileUrl.map(async (url: string) => {
const path = getPathname(url);
return u.oss.fileExists(path);
}),
);
if (!fileExistsResults.every(Boolean)) {
return res.status(400).send(error("选择分镜文件不存在"));
if (!fileExistsResults.every(Boolean)) {
return res.status(400).send(error("选择分镜文件不存在"));
}
}
const firstFrame = getPathname(fileUrl[0]);
const firstFrame = fileUrl.length ? getPathname(fileUrl[0]) : "";
const storyboardImgs = fileUrl.map((path: string) => getPathname(path));
const savePath = `/${projectId}/video/${uuidv4()}.mp4`;
@ -137,7 +134,7 @@ async function generateVideoAsync(
aiConfigData: t_config,
) {
try {
const projectData = await u.db("t_project").where("id", projectId).select("artStyle").first();
const projectData = await u.db("t_project").where("id", projectId).select("artStyle", "videoRatio").first();
// 提取路径名的辅助函数
const getPathname = (url: string): string => {
@ -173,7 +170,7 @@ ${prompt}
savePath,
prompt: inputPrompt,
duration: duration as any,
aspectRatio: resolution as any,
aspectRatio: projectData?.videoRatio as any,
resolution: resolution as any,
},
{
@ -195,7 +192,142 @@ ${prompt}
await u.db("t_video").where("id", videoId).update({ state: -1 });
}
} catch (err) {
console.error(`视频生成失败 videoId=${videoId}:`, err);
console.error(`视频生成失败 videoId=${videoId}:`, u.error(err).message);
await u.db("t_video").where("id", videoId).update({ state: -1 });
}
}
/**
* 使用sharp把图片拼接为宫格图，最多3x3，图片数量为1-9不等
* @param imageList - 图片路径或base64数组
* @returns 拼接后的图片Buffer
*/
async function sharpProcessingImage(imageList: string[], projectId: number): Promise<string> {
if (!imageList || imageList.length === 0) {
throw new Error("图片列表不能为空");
}
if (imageList.length > 9) {
throw new Error("图片数量不能超过9张");
}
// 计算网格布局：根据图片数量确定行列数
const count = imageList.length;
let cols: number, rows: number;
if (count === 1) {
cols = rows = 1;
} else if (count === 2) {
cols = 2;
rows = 1;
} else if (count <= 4) {
cols = rows = 2;
} else if (count <= 6) {
cols = 3;
rows = 2;
} else {
cols = rows = 3;
}
// 第一步：加载所有图片并获取原始尺寸
const loadedImages = await Promise.all(
imageList.map(async (imagePath) => {
let imageBuffer: Buffer;
// 判断是base64、URL还是文件路径
if (imagePath.startsWith("data:image") || imagePath.match(/^[A-Za-z0-9+/=]+$/)) {
// Base64格式
const base64Data = imagePath.replace(/^data:image\/\w+;base64,/, "");
imageBuffer = Buffer.from(base64Data, "base64");
} else if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
// URL格式，提取pathname后从OSS读取
const pathname = new URL(imagePath).pathname;
imageBuffer = await u.oss.getFile(pathname);
} else {
// 文件路径，直接从OSS读取
imageBuffer = await u.oss.getFile(imagePath);
}
const metadata = await sharp(imageBuffer).metadata();
return {
buffer: imageBuffer,
width: metadata.width || 0,
height: metadata.height || 0,
};
}),
);
// 第二步：找出所有图片中的最大宽度和高度
const maxWidth = Math.max(...loadedImages.map((img) => img.width));
const maxHeight = Math.max(...loadedImages.map((img) => img.height));
// 第三步：将所有图片调整为统一尺寸（使用contain模式保持比例，填充背景色）
const imageData = await Promise.all(
loadedImages.map(async (img) => {
const resizedBuffer = await sharp(img.buffer)
.resize(maxWidth, maxHeight, {
fit: "contain",
background: { r: 0, g: 0, b: 0, alpha: 1 }, // 黑色背景填充
})
.png()
.toBuffer();
return {
buffer: resizedBuffer,
width: maxWidth,
height: maxHeight,
};
}),
);
// 所有图片都是相同尺寸，直接计算画布大小
const cellWidth = maxWidth;
const cellHeight = maxHeight;
const canvasWidth = cols * cellWidth;
const canvasHeight = rows * cellHeight;
// 创建空白画布
const canvas = sharp({
create: {
width: canvasWidth,
height: canvasHeight,
channels: 4,
background: { r: 255, g: 255, b: 255, alpha: 1 },
},
});
// 准备合成操作
const compositeOperations = imageData.map((data, index) => {
const row = Math.floor(index / cols);
const col = index % cols;
// 计算当前图片的位置（无偏移，紧密排列）
const left = col * cellWidth;
const top = row * cellHeight;
return {
input: data.buffer,
top: top,
left: left,
};
});
// 合成所有图片
const result = await canvas.composite(compositeOperations).png().toBuffer();
// 保存图片到当前文件夹，方便查看测试效果
const timestamp = new Date().getTime();
const outputFileName = `merged_image_${timestamp}.png`;
const outputPath = path.join(__dirname, outputFileName);
try {
await fs.promises.writeFile(outputPath, result);
} catch (err) {
console.error(`❌ 保存图片失败:`, err);
}
const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
const buffer = Buffer.from(result, "base64");
await u.oss.writeFile(imagePath, buffer);
return await u.oss.getFileUrl(imagePath);
}

1
src/utils/ai/image/owned/other.ts
View File

@ -40,7 +40,6 @@ export default async (input: ImageConfig, config: AIConfig): Promise<string> =>
} else {
promptData = fullPrompt + `请直接输出图片`;
}
console.log("%c Line:31 🍅 promptData", "background:#2eafb0", promptData);
const result = await generateText({
model: otherProvider.languageModel(model),

12
src/utils/ai/utils.ts
View File

@ -37,9 +37,15 @@ export const validateVideoConfig = (input: VideoConfig, config: AIConfig, custom
throw new Error(`模型 ${config.model} 不支持多图模式`);
}
// 校验duration和resolution是否在支持范围内
const validDurationResolution = owned.durationResolutionMap.some(
(map) => map.duration.includes(input.duration) && map.resolution.includes(input.resolution as typeof map.resolution[number]),
);
const validDurationResolution = owned.durationResolutionMap.some((map) => {
const durationMatch = map.duration.includes(input.duration);
const resolutionMatch =
// 若 map.resolution 和 input.resolution 均为空，视为匹配
(!input.resolution && map.resolution.length === 0) ||
// 否则匹配 includes
map.resolution.includes(input.resolution as (typeof map.resolution)[number]);
return durationMatch && resolutionMatch;
});
if (!validDurationResolution) {
const supportedDurations = [...new Set(owned.durationResolutionMap.flatMap((m) => m.duration))].sort((a, b) => a - b);
const supportedResolutions = [...new Set(owned.durationResolutionMap.flatMap((m) => m.resolution))];

3
src/utils/ai/video/index.ts
View File

@ -10,6 +10,7 @@ import wan from "./owned/wan";
import runninghub from "./owned/runninghub";
import gemini from "./owned/gemini";
import apimart from "./owned/apimart";
import other from "./owned/other";
const modelInstance = {
volcengine: volcengine,
@ -19,10 +20,10 @@ const modelInstance = {
gemini: gemini,
runninghub: runninghub,
apimart: apimart,
// other: other,
} as const;
export default async (input: VideoConfig, config?: AIConfig) => {
console.log("%c Line:25 🥛 config", "background:#2eafb0", config);
const { model, apiKey, baseURL, manufacturer } = { ...config };
if (!config || !config?.model || !config?.apiKey) throw new Error("请检查模型配置是否正确");

59
src/utils/ai/video/owned/other.ts Normal file
View File

@ -0,0 +1,59 @@
import "../type";
import axios from "axios";
import sharp from "sharp";
import FormData from "form-data";
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
import { createOpenAI } from "@ai-sdk/openai";
import { experimental_generateVideo as generateVideo } from "ai";
export default async (input: VideoConfig, config: AIConfig) => {
console.log("%c Line:9 🌰 config", "background:#fca650", config);
console.log("%c Line:9 🍒 input", "background:#33a5ff", input);
if (!config.apiKey) throw new Error("缺少API Key");
if (!config.baseURL) throw new Error("缺少baseURL");
// const { owned, images, hasTextType } = validateVideoConfig(input, config);
const [requestUrl, queryUrl] = config.baseURL.split("|");
const authorization = `Bearer ${config.apiKey}`;
const formData = new FormData();
formData.append("model", config.model);
formData.append("prompt", input.prompt);
formData.append("seconds", String(input.duration));
// 根据 aspectRatio 设置 size
const sizeMap: Record<string, string> = {
"16:9": "1280x720",
"9:16": "720x1280",
};
formData.append("size", sizeMap[input.aspectRatio] || "1920x1080");
console.log("%c Line:30 🍇 sizeMap[input.aspectRatio]", "background:#93c0a4", sizeMap[input.aspectRatio]);
if (input.imageBase64 && input.imageBase64.length) {
const base64Data = input.imageBase64[0]!.replace(/^data:image\/\w+;base64,/, "");
const buffer = Buffer.from(base64Data, "base64");
formData.append("input_reference", buffer, { filename: "image.jpg", contentType: "image/jpeg" });
}
const body = {
model: "sora-2-all",
messages: [
{
role: "user",
content: [
{
type: "text",
text: input.prompt,
},
],
},
],
};
const { data } = await axios.post(
"https://api2.aigcbest.top/v1/chat/completions",
{ ...body },
{
headers: { "Content-Type": "application/json", Authorization: authorization },
},
);
console.log("%c Line:62 🍩 data", "background:#465975", data);
if (data.status === "FAILED") throw new Error(`任务提交失败: ${data.errorMessage || "未知错误"}`);
};

16
src/utils/ai/video/owned/runninghub.ts
View File

@ -14,7 +14,7 @@ export default async (input: VideoConfig, config: AIConfig) => {
"https://www.runninghub.cn/openapi/v2/rhart-video-s/image-to-video-pro",
"https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video",
"https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video-pro",
"https://www.runninghub.cn/openapi/v2/rhart-video-s/{taskId}",
"https://www.runninghub.cn/openapi/v2/query",
"https://www.runninghub.cn/openapi/v2/media/upload/binary",
].join("|");
@ -78,9 +78,17 @@ export default async (input: VideoConfig, config: AIConfig) => {
const { taskId } = await submitTask(submitUrl, requestBody);
return await pollTask(async () => {
const { data } = await axios.get(queryUrl.replace("{taskId}", taskId), {
headers: { Authorization: authorization },
});
const { data } = await axios.post(
queryUrl,
{
taskId,
},
{
headers: { Authorization: authorization },
},
);
if (data.status === "SUCCESS") {
return data.results?.length ? { completed: true, url: data.results[0].url } : { completed: false, error: "任务成功但未返回视频链接" };
}