330 lines
10 KiB
TypeScript
330 lines
10 KiB
TypeScript
import express from "express";
|
||
import u from "@/utils";
|
||
import { z } from "zod";
|
||
import { v4 as uuidv4 } from "uuid";
|
||
import { error, success } from "@/lib/responseFormat";
|
||
import { validateFields } from "@/middleware/middleware";
|
||
import { t_config } from "@/types/database";
|
||
import sharp from "sharp";
|
||
import fs from "fs";
|
||
import path from "path";
|
||
|
||
const router = express.Router();
|
||
|
||
// 生成视频
|
||
export default router.post(
|
||
"/",
|
||
validateFields({
|
||
projectId: z.number(),
|
||
scriptId: z.number(),
|
||
configId: z.number().optional(), // 关联的视频配 置ID
|
||
type: z.string().optional(),
|
||
resolution: z.string(),
|
||
aiConfigId: z.number(),
|
||
filePath: z.array(z.string()),
|
||
duration: z.number(),
|
||
prompt: z.string(),
|
||
mode: z.enum(["startEnd", "multi", "single", "text"]),
|
||
audioEnabled: z.boolean(),
|
||
}),
|
||
async (req, res) => {
|
||
const { type, mode, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt, audioEnabled } = req.body;
|
||
|
||
if (mode == "text") filePath.length = 0;
|
||
else if (!filePath.length) {
|
||
return res.status(500).send(error("请先选择图片"));
|
||
}
|
||
const configData = await u.db("t_videoConfig").where("id", configId).first();
|
||
|
||
if (!configData) {
|
||
return res.status(500).send(error("视频配置不存在"));
|
||
}
|
||
if (configData.manufacturer == "runninghub") {
|
||
if (filePath.length > 1) {
|
||
const gridUrl = await sharpProcessingImage(filePath, projectId);
|
||
if (gridUrl) {
|
||
filePath.length = 0;
|
||
filePath.push(gridUrl);
|
||
}
|
||
}
|
||
}
|
||
|
||
// 优先使用视频配置中的AI配置ID查询,查不到再使用传入的aiConfigId
|
||
let aiConfigData = null;
|
||
if (configData.aiConfigId) {
|
||
aiConfigData = await u.db("t_config").where("id", configData.aiConfigId).first();
|
||
}
|
||
if (!aiConfigData) {
|
||
aiConfigData = await u.db("t_config").where("id", aiConfigId).first();
|
||
}
|
||
|
||
if (!aiConfigData) {
|
||
return res.status(500).send(error("模型配置不存在"));
|
||
}
|
||
// 过滤掉空值
|
||
let fileUrl = filePath.filter((p: string) => p && p.trim() !== "");
|
||
|
||
// 处理文件路径，如果是 base64 则上传到 OSS
|
||
if (fileUrl.length) {
|
||
const match = fileUrl[0].match(/base64,([A-Za-z0-9+/=]+)/);
|
||
if (match && match.length >= 2) {
|
||
const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
|
||
const buffer = Buffer.from(match[1], "base64");
|
||
await u.oss.writeFile(imagePath, buffer);
|
||
fileUrl = [await u.oss.getFileUrl(imagePath)];
|
||
}
|
||
}
|
||
|
||
// 提取路径名的辅助函数
|
||
const getPathname = (url: string): string => {
|
||
// 如果是完整 URL，提取 pathname
|
||
if (url.startsWith("http://") || url.startsWith("https://")) {
|
||
return new URL(url).pathname;
|
||
}
|
||
// 否则认为已经是路径
|
||
return url;
|
||
};
|
||
if (fileUrl.length) {
|
||
// 校验文件是否存在
|
||
const fileExistsResults = await Promise.all(
|
||
fileUrl.map(async (url: string) => {
|
||
const path = getPathname(url);
|
||
return u.oss.fileExists(path);
|
||
}),
|
||
);
|
||
|
||
if (!fileExistsResults.every(Boolean)) {
|
||
return res.status(400).send(error("选择分镜文件不存在"));
|
||
}
|
||
}
|
||
|
||
const firstFrame = fileUrl.length ? getPathname(fileUrl[0]) : "";
|
||
const storyboardImgs = fileUrl.map((path: string) => getPathname(path));
|
||
const savePath = `/${projectId}/video/${uuidv4()}.mp4`;
|
||
|
||
// 先插入记录，state 默认为 0
|
||
const [videoId] = await u.db("t_video").insert({
|
||
scriptId,
|
||
configId: configId || null, // 关联的视频配置ID
|
||
time: duration,
|
||
resolution,
|
||
prompt,
|
||
firstFrame,
|
||
storyboardImgs: JSON.stringify(storyboardImgs),
|
||
filePath: savePath,
|
||
state: 0,
|
||
});
|
||
|
||
// 立即返回，不等待视频生成
|
||
res.status(200).send(success({ id: videoId, configId: configId || null }));
|
||
|
||
// 异步生成视频
|
||
generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, audioEnabled, aiConfigData);
|
||
},
|
||
);
|
||
|
||
// 异步生成视频
|
||
async function generateVideoAsync(
|
||
videoId: number,
|
||
projectId: number,
|
||
fileUrl: string[],
|
||
savePath: string,
|
||
prompt: string,
|
||
duration: number,
|
||
resolution: string,
|
||
audioEnabled: boolean,
|
||
aiConfigData: t_config,
|
||
) {
|
||
try {
|
||
const projectData = await u.db("t_project").where("id", projectId).select("artStyle", "videoRatio").first();
|
||
|
||
// 提取路径名的辅助函数
|
||
const getPathname = (url: string): string => {
|
||
if (url.startsWith("http://") || url.startsWith("https://")) {
|
||
return new URL(url).pathname;
|
||
}
|
||
return url;
|
||
};
|
||
|
||
const imageBase64 = await Promise.all(
|
||
fileUrl.map((path: string) => {
|
||
if (path.startsWith("http://") || path.startsWith("https://")) {
|
||
return u.oss.getImageBase64(getPathname(path));
|
||
}
|
||
// 如果是相对路径，直接获取
|
||
return u.oss.getImageBase64(path);
|
||
}),
|
||
);
|
||
|
||
const inputPrompt = `
|
||
请完全参照以下内容生成视频：
|
||
${prompt}
|
||
重要强调：
|
||
风格高度保持${projectData?.artStyle || "CG"}风格，保证人物一致性
|
||
1. 视频整体风格、色调、光影、人脸五官与参考图片保持高度一致
|
||
2. 保证视频连贯性、前后无矛盾
|
||
3. 关键人物在画面中全部清晰显示，不得被遮挡、缺失或省略
|
||
4. 画面真实、细致，无畸形、无模糊、无杂物、无多余人物、无文字、水印、logo
|
||
`;
|
||
const videoPath = await u.ai.video(
|
||
{
|
||
imageBase64,
|
||
savePath,
|
||
prompt: inputPrompt,
|
||
duration: duration as any,
|
||
aspectRatio: projectData?.videoRatio as any,
|
||
resolution: resolution as any,
|
||
audio: audioEnabled,
|
||
},
|
||
{
|
||
baseURL: aiConfigData?.baseUrl!,
|
||
model: aiConfigData?.model!,
|
||
apiKey: aiConfigData?.apiKey!,
|
||
manufacturer: aiConfigData?.manufacturer!,
|
||
},
|
||
);
|
||
|
||
if (videoPath) {
|
||
// 生成成功，更新状态为 1
|
||
await u.db("t_video").where("id", videoId).update({
|
||
filePath: videoPath,
|
||
state: 1,
|
||
});
|
||
} else {
|
||
// 生成失败，更新状态为 -1
|
||
await u.db("t_video").where("id", videoId).update({ state: -1 });
|
||
}
|
||
} catch (err) {
|
||
console.error(`视频生成失败 videoId=${videoId}:`, err);
|
||
await u
|
||
.db("t_video")
|
||
.where("id", videoId)
|
||
.update({ state: -1, errorReason: u.error(err).message });
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 使用sharp把图片拼接为宫格图，最多3x3，图片数量为1-9不等
|
||
* @param imageList - 图片路径或base64数组
|
||
* @returns 拼接后的图片Buffer
|
||
*/
|
||
async function sharpProcessingImage(imageList: string[], projectId: number): Promise<string> {
|
||
if (!imageList || imageList.length === 0) {
|
||
throw new Error("图片列表不能为空");
|
||
}
|
||
|
||
if (imageList.length > 9) {
|
||
throw new Error("图片数量不能超过9张");
|
||
}
|
||
|
||
// 计算网格布局：根据图片数量确定行列数
|
||
const count = imageList.length;
|
||
let cols: number, rows: number;
|
||
|
||
if (count === 1) {
|
||
cols = rows = 1;
|
||
} else if (count === 2) {
|
||
cols = 2;
|
||
rows = 1;
|
||
} else if (count <= 4) {
|
||
cols = rows = 2;
|
||
} else if (count <= 6) {
|
||
cols = 3;
|
||
rows = 2;
|
||
} else {
|
||
cols = rows = 3;
|
||
}
|
||
|
||
// 第一步：加载所有图片并获取原始尺寸
|
||
const loadedImages = await Promise.all(
|
||
imageList.map(async (imagePath) => {
|
||
let imageBuffer: Buffer;
|
||
|
||
// 判断是base64、URL还是文件路径
|
||
if (imagePath.startsWith("data:image") || imagePath.match(/^[A-Za-z0-9+/=]+$/)) {
|
||
// Base64格式
|
||
const base64Data = imagePath.replace(/^data:image\/\w+;base64,/, "");
|
||
imageBuffer = Buffer.from(base64Data, "base64");
|
||
} else if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
|
||
// URL格式，提取pathname后从OSS读取
|
||
const pathname = new URL(imagePath).pathname;
|
||
imageBuffer = await u.oss.getFile(pathname);
|
||
} else {
|
||
// 文件路径，直接从OSS读取
|
||
imageBuffer = await u.oss.getFile(imagePath);
|
||
}
|
||
|
||
const metadata = await sharp(imageBuffer).metadata();
|
||
return {
|
||
buffer: imageBuffer,
|
||
width: metadata.width || 0,
|
||
height: metadata.height || 0,
|
||
};
|
||
}),
|
||
);
|
||
|
||
// 第二步：找出所有图片中的最大宽度和高度
|
||
const maxWidth = Math.max(...loadedImages.map((img) => img.width));
|
||
const maxHeight = Math.max(...loadedImages.map((img) => img.height));
|
||
|
||
// 第三步：将所有图片调整为统一尺寸（使用contain模式保持比例，填充背景色）
|
||
const imageData = await Promise.all(
|
||
loadedImages.map(async (img) => {
|
||
const resizedBuffer = await sharp(img.buffer)
|
||
.resize(maxWidth, maxHeight, {
|
||
fit: "contain",
|
||
background: { r: 0, g: 0, b: 0, alpha: 1 }, // 黑色背景填充
|
||
})
|
||
.png()
|
||
.toBuffer();
|
||
|
||
return {
|
||
buffer: resizedBuffer,
|
||
width: maxWidth,
|
||
height: maxHeight,
|
||
};
|
||
}),
|
||
);
|
||
|
||
// 所有图片都是相同尺寸，直接计算画布大小
|
||
const cellWidth = maxWidth;
|
||
const cellHeight = maxHeight;
|
||
const canvasWidth = cols * cellWidth;
|
||
const canvasHeight = rows * cellHeight;
|
||
|
||
// 创建空白画布
|
||
const canvas = sharp({
|
||
create: {
|
||
width: canvasWidth,
|
||
height: canvasHeight,
|
||
channels: 4,
|
||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||
},
|
||
});
|
||
|
||
// 准备合成操作
|
||
const compositeOperations = imageData.map((data, index) => {
|
||
const row = Math.floor(index / cols);
|
||
const col = index % cols;
|
||
|
||
// 计算当前图片的位置（无偏移，紧密排列）
|
||
const left = col * cellWidth;
|
||
const top = row * cellHeight;
|
||
|
||
return {
|
||
input: data.buffer,
|
||
top: top,
|
||
left: left,
|
||
};
|
||
});
|
||
|
||
// 合成所有图片
|
||
const result = await canvas.composite(compositeOperations).png().toBuffer();
|
||
|
||
const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
|
||
const buffer = Buffer.from(result as any, "base64");
|
||
await u.oss.writeFile(imagePath, buffer);
|
||
|
||
return await u.oss.getFileUrl(imagePath);
|
||
}
|