zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

所有ai 分模块接入，待测试

Browse Source
This commit is contained in:
zhishi 2026-02-07 17:51:18 +08:00
parent 981b6fe863
commit c064fdb679
35 changed files with 391 additions and 373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/agents/outlineScript/index.ts
View File

@ -611,35 +611,18 @@ ${task}
this.log(`Sub-Agent 调用`, agentType);
const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]);
const promptConfig = await u.getPromptAi(promptsList.map((i) => i.id) as number[]);
const promptConfig = await u.getPromptAi("outlineScriptAgent");
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
const getAiPromptConfig = (code: string) => {
const item = promptsList.find((p) => p.code === code);
const subConfig = promptConfig.find((sub) => sub?.promptsId == item?.id);
if (subConfig) {
return {
prompt: item?.customValue || item?.defaultValue || errPrompts,
apiConfig: { ...subConfig },
};
} else {
return {
prompt: item?.customValue || item?.defaultValue || errPrompts,
apiConfig: {},
};
}
return item?.customValue || item?.defaultValue || errPrompts;
};
const a1Prompt = getAiPromptConfig("outlineScript-a1");
const a2Prompt = getAiPromptConfig("outlineScript-a2");
const directorPrompt = getAiPromptConfig("outlineScript-director");
const SYSTEM_PROMPTS: Record<
AgentType,
{
prompt: string;
apiConfig: Object;
}
> = {
const SYSTEM_PROMPTS = {
AI1: a1Prompt,
AI2: a2Prompt,
director: directorPrompt,
@ -649,12 +632,12 @@ ${task}
const { fullStream } = await u.ai.text.stream(
{
system: SYSTEM_PROMPTS[agentType].prompt,
system: SYSTEM_PROMPTS[agentType],
tools: this.getSubAgentTools(),
messages: [{ role: "user", content: context }],
maxStep: 100,
},
SYSTEM_PROMPTS[agentType].apiConfig,
promptConfig,
);
let fullResponse = "";
@ -717,7 +700,9 @@ ${task}
const envContext = await this.buildEnvironmentContext();
const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first();
const promptConfig = await u.getPromptAi(prompts?.id);
console.log("%c Line:703 🍭 prompts", "background:#f5ce50", prompts);
const promptConfig = await u.getPromptAi("outlineScriptAgent");
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
const { fullStream } = await u.ai.text.stream(

2
src/agents/storyboard/generateImagePromptsTool.ts
View File

@ -98,7 +98,7 @@ async function generateGridPrompt(options: GridPromptOptions): Promise<GridPromp
: "";
const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first();
const promptAiConfig = await u.getPromptAi(promptsData?.id);
const promptAiConfig = await u.getPromptAi("storyboardAgent");
const mainPrompts = promptsData?.customValue || promptsData?.defaultValue;
const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`;

42
src/agents/storyboard/generateImageTool.ts
View File

@ -215,7 +215,9 @@ async function filterRelevantAssets(prompts: string[], allResources: ResourceIte
return availableImages;
}
const { relevantAssets } = await u.ai.text.invoke({
const apiConfig = await u.getPromptAi("storyboardAgent");
const { relevantAssets } = await u.ai.text.invoke(
{
messages: [
{
role: "user",
@ -240,33 +242,9 @@ ${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")}
)
.describe("与分镜内容相关的资产列表"),
},
});
// const result = await chatModel!.invoke({
// messages: [
// {
// role: "user",
// content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。
// 分镜描述：
// ${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")}
// 可用资产列表：
// ${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")}
// 请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`,
// },
// ],
// responseFormat: {
// type: "json_schema",
// jsonSchema: {
// name: "filteredAssets",
// strict: true,
// schema: z.toJSONSchema(filteredAssetsSchema),
// },
// },
// });
// const data = result?.json as z.infer<typeof filteredAssetsSchema>;
},
apiConfig,
);
if (!relevantAssets || relevantAssets.length === 0) {
return availableImages;
@ -342,14 +320,18 @@ export default async (cells: { prompt: string }[], scriptId: number, projectId:
console.log("====润色后：", prompts);
const processedImages = await processImages(filteredImages);
const apiConfig = await u.getPromptAi("storyboardImage");
const contentStr = await u.ai.image({
const contentStr = await u.ai.image(
{
systemPrompt: resourcesMapPrompts,
prompt: prompts,
size: "4K",
aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9",
imageBase64: processedImages.map((buf) => buf.toString("base64")),
});
},
apiConfig,
);
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
const base64Str = match?.[1] ?? contentStr;

32
src/agents/storyboard/index.ts
View File

@ -63,9 +63,6 @@ export default class Storyboard {
// 更新shopts
public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) {
console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId);
console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId);
console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell);
const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId);
if (shotIndex === -1) {
return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`;
@ -594,34 +591,17 @@ ${task}
this.log(`Sub-Agent 调用`, agentType);
const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]);
const promptConfig = await u.getPromptAi(promptsList.map((i) => i.id) as number[]);
const promptConfig = await u.getPromptAi("storyboardAgent");
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
const getAiPromptConfig = (code: string) => {
const item = promptsList.find((p) => p.code === code);
const subConfig = promptConfig.find((sub) => sub?.promptsId == item?.id);
if (subConfig) {
return {
prompt: item?.customValue || item?.defaultValue || errPrompts,
apiConfig: { ...subConfig },
};
} else {
return {
prompt: item?.customValue || item?.defaultValue || errPrompts,
apiConfig: {},
};
}
return item?.customValue || item?.defaultValue || errPrompts;
};
const segmentAgent = getAiPromptConfig("storyboard-segment");
const shotAgent = getAiPromptConfig("storyboard-shot");
const SYSTEM_PROMPTS: Record<
AgentType,
{
prompt: string;
apiConfig: Object;
}
> = {
const SYSTEM_PROMPTS = {
segmentAgent: segmentAgent,
shotAgent: shotAgent,
};
@ -630,12 +610,12 @@ ${task}
const { fullStream } = await u.ai.text.stream(
{
system: SYSTEM_PROMPTS[agentType].prompt,
system: SYSTEM_PROMPTS[agentType],
tools: this.getSubAgentTools(agentType),
messages: [{ role: "user", content: context }],
maxStep: 100,
},
SYSTEM_PROMPTS[agentType].apiConfig,
promptConfig,
);
let fullResponse = "";
@ -700,7 +680,7 @@ ${task}
const envContext = await this.buildEnvironmentContext();
const prompts = await u.db("t_prompts").where("code", "storyboard-main").first();
const promptConfig = await u.getPromptAi(prompts?.id);
const promptConfig = await u.getPromptAi("storyboardAgent");
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";

62
src/lib/initDB.ts
View File

File diff suppressed because one or more lines are too long

8
src/routes/assets/generateAssets.ts
View File

@ -123,14 +123,18 @@ export default router.post(
state: "生成中",
assetsId: id,
});
const apiConfig = await u.getPromptAi("assetsImage");
const contentStr = await u.ai.image({
const contentStr = await u.ai.image(
{
systemPrompt,
prompt: userPrompt,
imageBase64: base64 ? [base64] : [],
size: "2K",
aspectRatio: "16:9",
});
},
apiConfig,
);
let insertType;
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);

37
src/routes/assets/polishPrompt.ts
View File

@ -88,31 +88,16 @@ export default router.post(
const result: ResultItem[] = Object.values(itemMap);
const promptsList = await u.db("t_prompts").where("code", "in", ["role-polish", "scene-polish", "storyboard-polish", "tool-polish"]);
const propmptIds = promptsList.map((i) => i.id);
const mapList = await u
.db("t_aiModelMap")
.leftJoin("t_config", "t_config.id", "t_aiModelMap.configId")
.whereIn("t_aiModelMap.promptsId", propmptIds as number[])
.select("t_config.model", "t_config.apiKey", "t_config.baseUrl", "t_config.manufacturer", "t_aiModelMap.promptsId");
const apiConfigData = await u.getPromptAi("assetsPrompt");
const errPrompts = "不论用户说什么，请直接输出AI配置异常";
const getPromptValue = (code: string) => {
const item = promptsList.find((p) => p.code === code);
if (item) {
const apiData = mapList.find((i) => i.promptsId == item.id);
if (apiData) delete apiData?.promptsId;
return { prompt: item?.customValue ?? item?.defaultValue ?? errPrompts, apiData: { ...(apiData ?? {}) } };
} else {
return {
prompt: errPrompts,
apiData: {},
};
}
return item?.customValue ?? item?.defaultValue ?? errPrompts;
};
const role = getPromptValue("role-polish");
const scene = getPromptValue("scene-polish");
const tool = getPromptValue("tool-polish");
const storyboard = getPromptValue("storyboard-polish");
let apiConfig = {};
let systemPrompt = "";
let userPrompt = "";
if (type == "role") {
@ -120,8 +105,7 @@ export default router.post(
const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange];
const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[];
const results: string = mergeNovelText(novelData);
systemPrompt = role.prompt;
apiConfig = role.apiData;
systemPrompt = role;
userPrompt = `
请根据以下参数生成角色标准四视图提示词：
@ -144,8 +128,7 @@ export default router.post(
const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange];
const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[];
const results: string = mergeNovelText(novelData);
systemPrompt = scene.prompt;
apiConfig = scene.apiData;
systemPrompt = scene;
userPrompt = `
请根据以下参数生成场景图提示词：
@ -168,8 +151,7 @@ export default router.post(
const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange];
const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[];
const results: string = mergeNovelText(novelData);
systemPrompt = tool.prompt;
apiConfig = tool.apiData;
systemPrompt = tool;
userPrompt = `
请根据以下参数生成道具图提示词：
@ -188,8 +170,7 @@ export default router.post(
`;
}
if (type == "storyboard") {
systemPrompt = storyboard.prompt;
apiConfig = storyboard.apiData;
systemPrompt = storyboard;
userPrompt = `
请根据以下参数生成分镜图提示词：
@ -207,7 +188,6 @@ export default router.post(
`;
}
async function generatePrompt() {
apiConfig = {};
const result = await u.ai.text.invoke(
{
messages: [
@ -224,9 +204,7 @@ export default router.post(
prompt: zod.string().describe("提示词"),
},
},
{
...apiConfig,
},
apiConfigData,
);
// const result = await model.invoke({
// messages: [
@ -256,7 +234,6 @@ export default router.post(
res.status(200).send(success({ prompt: prompt, assetsId }));
} catch (e: any) {
console.log("%c Line:235 🥚 e", "background:#33a5ff", e);
return res.status(500).send(error(e?.data?.error?.message ?? e?.message ?? "生成失败"));
}
},

5
src/routes/other/testAI.ts
View File

@ -13,12 +13,12 @@ export default router.post(
modelName: z.string(),
apiKey: z.string(),
baseURL: z.string().optional(),
manufacturer: z.string(),
}),
async (req, res) => {
const { modelName, apiKey, baseURL } = req.body;
const { modelName, apiKey, baseURL, manufacturer } = req.body;
const getWeatherTool = tool({
// strict: true,
description: "Get the weather in a location",
inputSchema: z.object({
location: z.string().describe("The location to get the weather for"),
@ -43,6 +43,7 @@ export default router.post(
model: modelName,
apiKey,
baseURL,
manufacturer,
},
);
res.status(200).send(success(reply));

12
src/routes/other/testImage.ts
View File

@ -17,13 +17,21 @@ export default router.post(
async (req, res) => {
const { modelName, apiKey, baseURL, manufacturer } = req.body;
try {
const image = await u.ai.image({
const image = await u.ai.image(
{
prompt:
"一张16:9比例的图片，完美等分为2x2四宫格布局，各区域无缝衔接：\n左上宫格：一只可爱的猫，毛发蓬松，眼睛明亮，姿态俏皮\n右上宫格：一只友善的狗，金毛犬，表情愉悦，摇着尾巴\n左下宫格：一头健壮的牛，田园背景，目光温和，皮毛光泽\n右下宫格：一匹骏马，姿态优雅，鬃毛飘逸，肌肉健美\n风格要求：四个宫格风格统一，色彩鲜艳饱和，高清画质，细节清晰锐利，专业插画风格，线条干净，统一的左上方光源，柔和阴影，和谐配色，卡通/半写实风格，宫格间用白色或浅灰细线分隔",
imageBase64: [],
aspectRatio: "16:9",
size: "1K",
});
},
{
model: modelName,
apiKey,
baseURL,
manufacturer,
},
);
res.status(200).send(success(image));
} catch (err) {
const msg = u.error(err).message;

14
src/routes/other/testVideo.ts
View File

@ -12,12 +12,13 @@ export default router.post(
modelName: z.string().optional(),
apiKey: z.string(),
baseURL: z.string().optional(),
manufacturer: z.enum(["runninghub", "volcengine", "apimart", "gemini", "openAi"]),
manufacturer: z.string(),
}),
async (req, res) => {
const { modelName, apiKey, baseURL, manufacturer } = req.body;
try {
const videoPath = await u.ai.video({
const videoPath = await u.ai.video(
{
imageBase64: [],
savePath: "test.mp4",
prompt: "stickman Dances",
@ -25,7 +26,14 @@ export default router.post(
resolution: "720p",
aspectRatio: "16:9",
audio: false,
});
},
{
model: modelName,
apiKey,
baseURL,
manufacturer,
},
);
const url = await u.oss.getFileUrl(videoPath);
res.status(200).send(success(url));
} catch (err: any) {

8
src/routes/setting/addModel.ts
View File

@ -8,23 +8,23 @@ const router = express.Router();
export default router.post(
"/",
validateFields({
type: z.string(),
name: z.string(),
type: z.enum(["text", "video", "image"]),
model: z.string(),
baseUrl: z.string(),
apiKey: z.string(),
modelType: z.string(),
manufacturer: z.string(),
}),
async (req, res) => {
const { type, name, model, baseUrl, apiKey, manufacturer } = req.body;
const { type, model, baseUrl, apiKey, manufacturer, modelType } = req.body;
await u.db("t_config").insert({
type,
name,
model,
baseUrl,
apiKey,
manufacturer,
modelType,
createTime: Date.now(),
userId: 1,
});

11
src/routes/setting/configurationModel.ts
View File

@ -8,20 +8,13 @@ const router = express.Router();
export default router.post(
"/",
validateFields({
id: z.number().optional(),
promptsId: z.number(),
id: z.number(),
configId: z.number(),
}),
async (req, res) => {
const { id, promptsId, configId } = req.body;
const { id, configId } = req.body;
if (id) {
await u.db("t_aiModelMap").where("id", id).update({
promptsId,
configId,
});
} else {
await u.db("t_aiModelMap").insert({
promptsId,
configId,
});
}

8
src/routes/setting/getAiModelMap.ts
View File

@ -1,13 +1,13 @@
import express from "express";
import u from "@/utils";
import { success } from "@/lib/responseFormat";
const router = express.Router();
export default router.post("/", async (req, res) => {
const configData = await u
.db("t_prompts")
.leftJoin("t_aiModelMap", "t_prompts.id", "t_aiModelMap.promptsId")
.leftJoin("t_config", "t_config.id", "t_aiModelMap.configId")
.select("t_prompts.id as promptsId", "t_prompts.code", "t_prompts.name", "t_config.model", "t_aiModelMap.id");
.db("t_aiModelMap")
.leftJoin("t_config", "t_aiModelMap.configId", "t_config.id")
.select("t_aiModelMap.name", "t_config.model", "t_aiModelMap.id");
res.status(200).send(success(configData));
});

8
src/routes/setting/updeteModel.ts
View File

@ -9,23 +9,23 @@ export default router.post(
"/",
validateFields({
id: z.number(),
type: z.string(),
name: z.string(),
type: z.enum(["text", "video", "image"]),
model: z.string(),
baseUrl: z.string(),
modelType: z.string(),
apiKey: z.string(),
manufacturer: z.string(),
}),
async (req, res) => {
const { id, type, name, model, baseUrl, apiKey, manufacturer } = req.body;
const { id, type, model, baseUrl, apiKey, manufacturer, modelType } = req.body;
await u.db("t_config").where("id", id).update({
type,
name,
model,
baseUrl,
apiKey,
manufacturer,
modelType,
});
res.status(200).send(success("编辑成功"));
},

32
src/routes/storyboard/batchSuperScoreImage.ts
View File

@ -17,19 +17,19 @@ async function urlToBase64(imageUrl: string): Promise<string> {
}
// 超分并保存到 oss
async function superResolutionAndSave(
src: string,
projectId: number,
videoRatio: string,
): Promise<{ ossPath: string; base64: string }> {
const contentStr = await u.ai.image({
async function superResolutionAndSave(src: string, projectId: number, videoRatio: string): Promise<{ ossPath: string; base64: string }> {
const apiConfig = await u.getPromptAi("storyboardImage");
const contentStr = await u.ai.image(
{
aspectRatio: videoRatio,
size: "1K",
resType: "b64",
systemPrompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率",
prompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率",
imageBase64: [await urlToBase64(src)],
});
},
apiConfig,
);
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
const base64Str = match ? match[1] : contentStr;
const buffer = Buffer.from(base64Str, "base64");
@ -50,9 +50,9 @@ export default router.post(
id: z.string(),
prompt: z.string().optional(),
src: z.string(),
})
}),
),
})
}),
),
}),
async (req, res) => {
@ -63,9 +63,7 @@ export default router.post(
if (!projectData) return res.status(500).send(error("项目不存在"));
// 遍历处理每个分镜段
const processSegment = async (
segment: { cells: { id: string; src: string }[] }
) => {
const processSegment = async (segment: { cells: { id: string; src: string }[] }) => {
// 超分所有 cell
const cellsWithSuperscore = await Promise.all(
segment.cells.map(async (cell) => {
@ -76,9 +74,9 @@ export default router.post(
scriptId,
filePath: ossPath, // oss 路径（未签名）
src: cell.src,
type: "分镜"
type: "分镜",
};
})
}),
);
return cellsWithSuperscore;
};
@ -92,9 +90,9 @@ export default router.post(
(item.value as any[]).map(async (cell) => ({
...cell,
filePath: await u.oss.getFileUrl(cell.filePath ?? ""),
}))
)
})),
),
);
res.status(200).send(success(flatList));
}
},
);

36
src/routes/storyboard/generateVideoPrompt.ts
View File

@ -4,6 +4,7 @@ import { error, success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { z } from "zod";
import path from "path";
import axios from "axios";
const router = express.Router();
@ -103,7 +104,12 @@ const prompt = `
现在请根据我提供的分镜内容，严格按照以上规则输出 Motion Prompt JSON 对象。
`;
async function urlToBase64(imageUrl: string): Promise<string> {
const response = await axios.get(imageUrl, { responseType: "arraybuffer" });
const contentType = response.headers["content-type"] || "image/png";
const base64 = Buffer.from(response.data, "binary").toString("base64");
return `data:${contentType};base64,${base64}`;
}
// 生成单个分镜提示
async function generateSingleVideoPrompt({
scriptText,
@ -114,19 +120,6 @@ async function generateSingleVideoPrompt({
storyboardPrompt: string;
ossPath: string;
}): Promise<{ content: string; time: number; name: string }> {
let rootDir: string;
if (typeof process.versions?.electron !== "undefined") {
const { app } = require("electron");
const userDataDir: string = app.getPath("userData");
rootDir = path.join(userDataDir, "uploads");
} else {
rootDir = path.join(process.cwd(), "uploads");
}
let imagePath = ossPath;
if (ossPath.includes("http")) {
imagePath = new URL(ossPath).pathname;
}
const messages: any[] = [
{
role: "system",
@ -140,24 +133,27 @@ async function generateSingleVideoPrompt({
text: `剧本内容:${scriptText}\n分镜提示词:${storyboardPrompt}`,
},
{
type: "local",
path: path.join(rootDir, imagePath),
type: "image",
image: await urlToBase64(ossPath),
},
],
},
];
try {
const result = await u.ai.text.invoke({
const apiConfig = await u.getPromptAi("videoPrompt");
const result = await u.ai.text.invoke(
{
messages,
output: {
time: z.number().describe("时长,镜头时长 1-15"),
content: z.string().describe("提示词内容"),
name: z.string().describe("分镜名称"),
},
});
console.log("%c Line:156 🍩 result", "background:#33a5ff", result);
},
apiConfig,
);
if (!result) {
console.error("AI 返回结果为空:", result);
throw new Error("AI 返回结果为空");

2
src/routes/video/addVideo.ts
View File

@ -42,5 +42,5 @@ export default router.post(
});
res.status(200).send(success({ message: "新增视频成功" }));
}
},
);

18
src/routes/video/addVideoConfig.ts
View File

@ -1,6 +1,6 @@
import express from "express";
import u from "@/utils";
import { success } from "@/lib/responseFormat";
import { error, success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { z } from "zod";
const router = express.Router();
@ -20,8 +20,8 @@ export default router.post(
validateFields({
scriptId: z.number(),
projectId: z.number(),
manufacturer: z.string(),
mode: z.enum(["startEnd", "multi", "single"]),
configId: z.number(),
mode: z.enum(["startEnd", "multi", "single",'text','']),
startFrame: imageItemSchema.optional(),
endFrame: imageItemSchema.optional(),
images: z
@ -38,19 +38,21 @@ export default router.post(
prompt: z.string().optional(),
}),
async (req, res) => {
const { scriptId, projectId, manufacturer, mode, startFrame, endFrame, images, resolution, duration, prompt } = req.body;
const { scriptId, projectId, configId, mode, startFrame, endFrame, images, resolution, duration, prompt } = req.body;
// 生成新ID
const maxIdResult: any = await u.db("t_videoConfig").max("id as maxId").first();
const newId = (maxIdResult?.maxId || 0) + 1;
const now = Date.now();
const configData = await u.db("t_config").where("id", configId).first();
if (!configData) return res.status(500).send(error("不存在的模型"));
// 插入数据
await u.db("t_videoConfig").insert({
id: newId,
scriptId,
projectId,
manufacturer,
manufacturer: configData.manufacturer,
aiConfigId: configId,
mode,
startFrame: startFrame ? JSON.stringify(startFrame) : null,
endFrame: endFrame ? JSON.stringify(endFrame) : null,
@ -70,7 +72,9 @@ export default router.post(
id: newId,
scriptId,
projectId,
manufacturer,
manufacturer: configData.manufacturer,
aiConfigId: configId,
model: configData.model,
mode,
startFrame,
endFrame,

40
src/routes/video/generatePrompt.ts
View File

@ -1,6 +1,6 @@
import express from "express";
import u from "@/utils";
import { success } from "@/lib/responseFormat";
import { error, success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { z } from "zod";
@ -8,47 +8,26 @@ const router = express.Router();
type GenerateMode = "startEnd" | "multi" | "single";
const getSystemPrompt = async (mode: GenerateMode): Promise<{ prompt: string; apiConfig: Object }> => {
const getSystemPrompt = async (mode: GenerateMode) => {
const promptsList = await u.db("t_prompts").where("code", "in", ["video-startEnd", "video-multi", "video-single", "video-main"]);
const promptAiConfig = await u.getPromptAi(promptsList.map((i) => i.id) as number[]);
const errPrompts = "不论用户说什么，请直接输出AI配置异常";
const getPromptValue = (code: string) => {
const item = promptsList.find((p) => p.code === code);
const subData = promptAiConfig.find((i) => i?.promptsId == item?.id);
const returnData = {
prompt: item?.customValue ?? item?.defaultValue ?? errPrompts,
apiConfig: {},
};
if (subData) {
returnData.apiConfig = { ...subData };
return returnData;
} else {
return returnData;
}
return item?.customValue ?? item?.defaultValue ?? errPrompts;
};
const startEnd = getPromptValue("video-startEnd");
const multi = getPromptValue("video-multi");
const single = getPromptValue("video-single");
const main = getPromptValue("video-main");
const modeDescriptions: Record<
GenerateMode,
{
prompt: string;
apiConfig: Object;
}
> = {
const modeDescriptions = {
startEnd: startEnd,
multi: multi,
single: single,
};
const modeData = modeDescriptions[mode];
return {
prompt: `${main}\n\n${modeData.prompt}`,
apiConfig: modeData.apiConfig,
};
return `${main}\n\n${modeData}`;
};
const getModeDescription = (mode: GenerateMode): string => {
@ -82,12 +61,14 @@ export default router.post(
const shotCount = images.length;
const avgDuration = (parseFloat(duration) / shotCount).toFixed(1);
const promptConfig = await getSystemPrompt(mode);
const promptAiConfig = await u.getPromptAi("videoPrompt");
try {
const result = await u.ai.text.invoke(
{
messages: [
{
role: "system",
content: promptConfig.prompt,
content: promptConfig,
},
{
role: "user",
@ -108,9 +89,12 @@ Generate storyboard prompts:`,
},
],
},
promptConfig.apiConfig,
promptAiConfig,
);
res.status(200).send(success(result.text));
} catch (e) {
return res.status(500).send(error(u.error(e).message));
}
},
);

66
src/routes/video/generateVideo.ts
View File

@ -4,6 +4,7 @@ import { z } from "zod";
import { v4 as uuidv4 } from "uuid";
import { error, success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { t_config } from "@/types/database";
const router = express.Router();
@ -13,35 +14,52 @@ export default router.post(
validateFields({
projectId: z.number(),
scriptId: z.number(),
configId: z.number().optional(), // 关联的视频配置ID
configId: z.number().optional(), // 关联的视频配 置ID
type: z.string().optional(),
resolution: z.string(),
aiConfigId: z.number(),
filePath: z.array(z.string()),
duration: z.number(),
prompt: z.string(),
}),
async (req, res) => {
const { type, scriptId, projectId, configId, resolution, filePath, duration, prompt } = req.body;
const { type, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt } = req.body;
// 参数校验
if (type === "volcengine") {
if (duration < 4 || duration > 12) {
return res.status(400).send(error("视频时长需在4-12秒之间"));
}
if (!["480p", "720p", "1080p"].includes(resolution)) {
return res.status(400).send(error("视频分辨率不正确"));
}
// // 参数校验
// if (type === "volcengine") {
// if (duration < 4 || duration > 12) {
// return res.status(400).send(error("视频时长需在4-12秒之间"));
// }
// if (!["480p", "720p", "1080p"].includes(resolution)) {
// return res.status(400).send(error("视频分辨率不正确"));
// }
// }
// if (type === "runninghub") {
// if (duration !== 10 && duration !== 15) {
// return res.status(400).send(error("视频时长只能是10秒或15秒"));
// }
// if (resolution !== "9:16" && resolution !== "16:9") {
// return res.status(400).send(error("视频分辨率不正确"));
// }
// }
const configData = await u.db("t_videoConfig").where("id", configId).first();
if (!configData) {
return res.status(500).send(error("视频配置不存在"));
}
if (type === "runninghub") {
if (duration !== 10 && duration !== 15) {
return res.status(400).send(error("视频时长只能是10秒或15秒"));
}
if (resolution !== "9:16" && resolution !== "16:9") {
return res.status(400).send(error("视频分辨率不正确"));
// 优先使用视频配置中的AI配置ID查询,查不到再使用传入的aiConfigId
let aiConfigData = null;
if (configData.aiConfigId) {
aiConfigData = await u.db("t_config").where("id", configData.aiConfigId).first();
}
if (!aiConfigData) {
aiConfigData = await u.db("t_config").where("id", aiConfigId).first();
}
if (!aiConfigData) {
return res.status(500).send(error("模型配置不存在"));
}
// 过滤掉空值
let fileUrl = filePath.filter((p: string) => p && p.trim() !== "");
@ -103,7 +121,7 @@ export default router.post(
res.status(200).send(success({ id: videoId, configId: configId || null }));
// 异步生成视频
generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, type);
generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, aiConfigData);
},
);
@ -116,7 +134,7 @@ async function generateVideoAsync(
prompt: string,
duration: number,
resolution: string,
type?: string,
aiConfigData: t_config,
) {
try {
const projectData = await u.db("t_project").where("id", projectId).select("artStyle").first();
@ -149,14 +167,22 @@ ${prompt}
3. 关键人物在画面中全部清晰显示，不得被遮挡、缺失或省略
4. 画面真实、细致，无畸形、无模糊、无杂物、无多余人物、无文字、水印、logo
`;
const videoPath = await u.ai.video({
const videoPath = await u.ai.video(
{
imageBase64,
savePath,
prompt: inputPrompt,
duration: duration as any,
aspectRatio: resolution as any,
resolution: resolution as any,
});
},
{
baseURL: aiConfigData?.baseUrl!,
model: aiConfigData?.model!,
apiKey: aiConfigData?.apiKey!,
manufacturer: aiConfigData?.manufacturer!,
},
);
if (videoPath) {
// 生成成功，更新状态为 1

4
src/routes/video/getManufacturer.ts
View File

@ -14,8 +14,8 @@ export default router.post(
async (req, res) => {
const { userId } = req.body;
const data = await u.db("t_config").where("userId", userId).select("manufacturer", "model");
const data = await u.db("t_config").where("type", "video").where("userId", userId).select("manufacturer", "model", "id");
res.status(200).send(success(data));
}
},
);

10
src/routes/video/getVideoConfigs.ts
View File

@ -15,16 +15,20 @@ export default router.post(
const { scriptId } = req.body;
// 查询该脚本下的所有视频配置
const configs = await u.db("t_videoConfig")
const configs = await u
.db("t_videoConfig")
.leftJoin("t_config", "t_config.id", "t_videoConfig.aiConfigId")
.where({ scriptId })
.orderBy("createTime", "desc");
.orderBy("createTime", "desc")
.select("t_videoConfig.*", "t_config.manufacturer as manufacturer", "t_config.model");
// 解析 JSON 字段
const result = configs.map((config: any) => ({
id: config.id,
scriptId: config.scriptId,
projectId: config.projectId,
aiConfigId: config.aiConfigId,
manufacturer: config.manufacturer,
model: config.model,
mode: config.mode,
startFrame: config.startFrame ? JSON.parse(config.startFrame) : null,
endFrame: config.endFrame ? JSON.parse(config.endFrame) : null,

8
src/types/database.d.ts vendored
View File

@ -1,10 +1,11 @@
// @db-hash e1460b0ace03f6aaed458653a32b6ffb
// @db-hash 4cd44aef6bb6ffb02c4619525966496d
//该文件由脚本自动生成，请勿手动修改
export interface t_aiModelMap {
'configId'?: number | null;
'id'?: number;
'promptsId'?: number | null;
'key'?: string | null;
'name'?: string | null;
}
export interface t_assets {
'duration'?: string | null;
@ -37,7 +38,7 @@ export interface t_config {
'id'?: number;
'manufacturer'?: string | null;
'model'?: string | null;
'name'?: string | null;
'modelType'?: string | null;
'type'?: string | null;
'userId'?: number | null;
}
@ -135,6 +136,7 @@ export interface t_video {
'time'?: number | null;
}
export interface t_videoConfig {
'aiConfigId'?: number | null;
'createTime'?: number | null;
'duration'?: number | null;
'endFrame'?: string | null;

9
src/utils/ai/image/index.ts
View File

@ -28,13 +28,16 @@ const modelInstance = {
other,
} as const;
export default async (input: ImageConfig, config?: AIConfig) => {
const sqlTextModelConfig = await u.getConfig("image");
const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config };
export default async (input: ImageConfig, config: AIConfig) => {
const { model, apiKey, baseURL, manufacturer } = { ...config };
if (!config || !config?.model || !config?.apiKey || !config?.manufacturer) throw new Error("请检查模型配置是否正确");
const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance];
if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商");
if (manufacturer !== "other") {
const owned = modelList.find((m) => m.model === model);
if (!owned) throw new Error("不支持的模型");
}
// 补充图片的 base64 内容类型字符串
if (input.imageBase64 && input.imageBase64.length > 0) {

3
src/utils/ai/image/owned/gemini.ts
View File

@ -3,14 +3,13 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google";
import { generateText } from "ai";
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
console.log("%c Line:6 🌰 config", "background:#ffdd4d", config);
if (!config.model) throw new Error("缺少Model名称");
if (!config.apiKey) throw new Error("缺少API Key");
if (!input.prompt) throw new Error("缺少提示词");
const google = createGoogleGenerativeAI({
apiKey: config.apiKey,
baseURL: config.baseURL,
baseURL: config?.baseURL ?? "https://generativelanguage.googleapis.com/v1beta",
});
// 构建完整的提示词

19
src/utils/ai/image/owned/other.ts
View File

@ -1,5 +1,5 @@
import "../type";
import { generateImage, generateText } from "ai";
import { generateImage, generateText, ModelMessage } from "ai";
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
@ -27,9 +27,24 @@ export default async (input: ImageConfig, config: AIConfig): Promise<string> =>
const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt;
const model = config.model;
if (model.includes("gemini") || model.includes("nano")) {
let promptData;
if (input.imageBase64 && input.imageBase64.length) {
promptData = [{ role: "system", content: fullPrompt + `请直接输出图片` }];
(promptData as ModelMessage[]).push({
role: "user",
content: input.imageBase64.map((i) => ({
type: "image",
image: i,
})),
});
} else {
promptData = fullPrompt + `请直接输出图片`;
}
console.log("%c Line:31 🍅 promptData", "background:#2eafb0", promptData);
const result = await generateText({
model: otherProvider.languageModel(model),
prompt: fullPrompt + `请直接输出图片`,
prompt: promptData as string | ModelMessage[],
providerOptions: {
google: {
imageConfig: {

12
src/utils/ai/image/owned/vidu.ts
View File

@ -20,7 +20,6 @@ function template(replaceObj: Record<string, any>, url: string) {
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
if (!config.model) throw new Error("缺少Model名称");
if (!config.apiKey) throw new Error("缺少API Key");
const apiKey = "Token " + config.apiKey.replace(/Token\s+/g, "").trim();
const viduq2Ratio = ["16:9", "9:16", "1:1", "3:4", "4:3", "21:9", "2:3", "3:2"];
const viduq1Ratio = ["16:9", "9:16", "1:1", "3:4", "4:3"];
@ -60,7 +59,8 @@ export default async (input: ImageConfig, config: AIConfig): Promise<string> =>
...(images.length && { images: images }),
};
const urlObj = getApiUrl(config.baseURL!);
const urlObj = getApiUrl(config.baseURL! ?? "https://api.vidu.cn/ent/v2/reference2image|https://api.vidu.cn/ent/v2/tasks/{id}/creations");
try {
const { data } = await axios.post(urlObj.requestUrl, body, { headers: { Authorization: apiKey } });
@ -69,17 +69,13 @@ export default async (input: ImageConfig, config: AIConfig): Promise<string> =>
return await pollTask(async () => {
const { data: queryData } = await axios.get(queryUrl, { headers: { Authorization: apiKey } });
if (queryData.state !== 0) {
return { completed: false, error: queryData.message || "查询任务失败" };
}
const { state, err_code, creations } = queryData.data || {};
const { state, err_code, creations } = queryData || {};
if (state === "failed") {
return { completed: false, error: err_code || "图片生成失败" };
}
if (state === "succeed") {
if (state === "success") {
return { completed: true, url: creations?.[0]?.url };
}

27
src/utils/ai/text/index.ts
View File

@ -6,7 +6,6 @@ import { parse } from "best-effort-json-parser";
import modelList from "./modelList";
import { z } from "zod";
import { OpenAIProvider } from "@ai-sdk/openai";
interface AIInput<T extends Record<string, z.ZodTypeAny> | undefined = undefined> {
system?: string;
tools?: Record<string, Tool>;
@ -23,10 +22,9 @@ interface AIConfig {
manufacturer?: string;
}
const buildOptions = async (input: AIInput<any>, config: AIConfig) => {
let sqlTextModelConfig = {};
if (!config || !config?.model || !config?.apiKey || !config?.baseURL) sqlTextModelConfig = await u.getConfig("text");
const { model, apiKey, baseURL, manufacturer } = { ...(sqlTextModelConfig as Awaited<ReturnType<typeof u.getConfig>>), ...config };
const buildOptions = async (input: AIInput<any>, config: AIConfig = {}) => {
if (!config || !config?.model || !config?.apiKey || !config?.baseURL || !config?.manufacturer) throw new Error("请检查模型配置是否正确");
const { model, apiKey, baseURL, manufacturer } = { ...config };
let owned;
if (manufacturer == "other") {
owned = modelList.find((m) => m.manufacturer === manufacturer);
@ -39,7 +37,9 @@ const buildOptions = async (input: AIInput<any>, config: AIConfig) => {
const maxStep = input.maxStep ?? (input.tools ? Object.keys(input.tools).length * 5 : undefined);
const outputBuilders: Record<string, (schema: any) => any> = {
schema: (s) => Output.object({ schema: z.object(s) }),
schema: (s) => {
return Output.object({ schema: z.object(s) });
},
object: () => {
const jsonSchemaPrompt = `\n请按照以下 JSON Schema 格式返回结果:\n${JSON.stringify(
z.toJSONSchema(z.object(input.output)),
@ -52,16 +52,11 @@ const buildOptions = async (input: AIInput<any>, config: AIConfig) => {
};
const output = input.output ? (outputBuilders[owned.responseFormat]?.(input.output) ?? null) : null;
const modelFn = owned.manufacturer == "doubao" ? (modelInstance as OpenAIProvider).chat(model!) : modelInstance(model!);
const chatModelManufacturer = ["doubao", "other", "openai"];
const modelFn = chatModelManufacturer.includes(owned.manufacturer) ? (modelInstance as OpenAIProvider).chat(model!) : modelInstance(model!);
return {
config: {
model:
process.env.NODE_ENV === "dev"
? wrapLanguageModel({
model: modelFn as any,
middleware: devToolsMiddleware(),
})
: (modelFn as LanguageModel),
model: modelFn as LanguageModel,
...(input.system && { system: input.system }),
...(input.prompt ? { prompt: input.prompt } : { messages: input.messages! }),
...(input.tools && owned.tool && { tools: input.tools }),
@ -79,7 +74,7 @@ const ai = Object.create({}) as {
stream(input: AIInput, config?: AIConfig): Promise<ReturnType<typeof streamText>>;
};
ai.invoke = async (input: AIInput<any>, config: AIConfig = {}) => {
ai.invoke = async (input: AIInput<any>, config: AIConfig) => {
const options = await buildOptions(input, config);
const result = await generateText(options.config);
if (options.responseFormat === "object" && input.output) {
@ -95,7 +90,7 @@ ai.invoke = async (input: AIInput<any>, config: AIConfig = {}) => {
return result;
};
ai.stream = async (input: AIInput, config: AIConfig = {}) => {
ai.stream = async (input: AIInput, config: AIConfig) => {
const options = await buildOptions(input, config);
return streamText(options.config);
};

2
src/utils/ai/text/modelList.ts
View File

@ -417,7 +417,7 @@ const modelList: Owned[] = [
responseFormat: "schema",
image: true,
think: false,
instance: createOpenAICompatible,
instance: createOpenAI,
tool: true,
},
];

6
src/utils/ai/video/index.ts
View File

@ -22,8 +22,10 @@ const modelInstance = {
} as const;
export default async (input: VideoConfig, config?: AIConfig) => {
const sqlTextModelConfig = await u.getConfig("video");
const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config };
console.log("%c Line:25 🥛 config", "background:#2eafb0", config);
const { model, apiKey, baseURL, manufacturer } = { ...config };
if (!config || !config?.model || !config?.apiKey) throw new Error("请检查模型配置是否正确");
const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance];
if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的视频厂商");
const owned = modelList.find((m) => m.model === model);

1
src/utils/ai/video/owned/volcengine.ts
View File

@ -47,6 +47,7 @@ export default async (input: VideoConfig, config: AIConfig) => {
});
const taskId = createResponse.data.id;
if (!taskId) throw new Error("视频任务创建失败");
// 轮询任务状态

1
src/utils/ai/video/type.ts
View File

@ -12,4 +12,5 @@ interface AIConfig {
model?: string;
apiKey?: string;
baseURL?: string;
manufacturer?: string;
}

9
src/utils/editImage.ts
View File

@ -79,13 +79,18 @@ async function convertDirectiveAndImages(images: Record<string, string>, directi
*/
export default async (images: Record<string, string>, directive: string, projectId: number) => {
const { prompt, images: base64Images } = await convertDirectiveAndImages(images, directive);
const contentStr = await u.ai.image({
const apiConfig = await u.getPromptAi("editImage");
const contentStr = await u.ai.image(
{
systemPrompt: "根据用户提供的具体修改指令，对上传的图片进行智能编辑。",
prompt: prompt,
imageBase64: base64Images,
aspectRatio: "16:9",
size: "1K",
});
},
apiConfig,
);
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
const buffer = Buffer.from(match && match.length >= 1 ? match[1]! : contentStr, "base64");
const filePath = `/${projectId}/storyboard/${uuid()}.jpg`;

2
src/utils/generateScript.ts
View File

@ -127,7 +127,7 @@ ${episodePrompt}
${novelData}`;
const prompts = await u.db("t_prompts").where("code", "script").first();
const promptConfig = await u.getPromptAi(prompts?.id);
const promptConfig = await u.getPromptAi("generateScript");
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出AI配置异常";
const result = await u.ai.text.invoke(

27
src/utils/getPromptAi.ts
View File

@ -1,26 +1,19 @@
import { db } from "./db";
interface AiConfig {
model: string;
model?: string;
apiKey: string;
baseUrl: string;
baseURL?: string;
manufacturer: string;
promptsId: number;
}
export default async function getPromptAi(promptsId: number | undefined): Promise<AiConfig | {}>;
export default async function getPromptAi(promptsId: number[]): Promise<AiConfig[]>;
export default async function getPromptAi(promptsId: number | number[] | undefined): Promise<AiConfig | AiConfig[] | {}> {
if (!promptsId) return {};
const ids = Array.isArray(promptsId) ? promptsId.filter(Boolean) : [promptsId];
const mapList = await db("t_aiModelMap")
export default async function getPromptAi(key: string): Promise<AiConfig | {}> {
const aiConfigData = await db("t_aiModelMap")
.leftJoin("t_config", "t_config.id", "t_aiModelMap.configId")
.whereIn("t_aiModelMap.promptsId", ids)
.select("t_config.model", "t_config.apiKey", "t_config.baseUrl", "t_config.manufacturer", "t_aiModelMap.promptsId");
.where("t_aiModelMap.key", key)
.select("t_config.model", "t_config.apiKey", "t_config.baseUrl as baseURL", "t_config.manufacturer")
.first();
if (Array.isArray(promptsId)) {
return mapList as AiConfig[];
} else {
return mapList[0] ? (mapList[0] as AiConfig) : {};
}
if (aiConfigData) {
return aiConfigData as AiConfig;
} else return {};
}