Merge branch '108' of https://github.com/HBAI-Ltd/Toonflow-app into 108
# Conflicts: # src/types/database.d.ts
This commit is contained in:
zhishi
commit
2f0a064363
@ -527,27 +527,20 @@ description: 专注于从剧本内容中提取所使用的资产（角色、场
|
||||
table.integer("time");
|
||||
table.text("state");
|
||||
table.integer("scriptId");
|
||||
table.integer("storyboardId");
|
||||
table.integer("projectId");
|
||||
table.integer("videoTrackId");
|
||||
table.primary(["id"]);
|
||||
table.unique(["id"]);
|
||||
},
|
||||
},
|
||||
// 视频轨道
|
||||
{
|
||||
name: "o_videoConfig",
|
||||
name: "o_videoTrack",
|
||||
builder: (table) => {
|
||||
table.integer("id").notNullable();
|
||||
table.integer("storyboardId");
|
||||
table.integer("videoId");
|
||||
table.integer("audio"); // 声音
|
||||
table.text("model"); // 模型
|
||||
table.text("mode"); // 模式：
|
||||
table.text("data"); // 所选数据集图片 JSON
|
||||
table.text("resolution"); // 分辨率
|
||||
table.integer("duration"); // 时长
|
||||
table.text("prompt"); // 提示词
|
||||
table.integer("createTime"); // 创建时间
|
||||
table.integer("updateTime"); // 更新时间
|
||||
table.integer("projectId");
|
||||
table.integer("scriptId");
|
||||
table.primary(["id"]);
|
||||
table.unique(["id"]);
|
||||
},
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
// @routes-hash 8b0a09b55cd1b98ff83ad988d077fee3
|
||||
// @routes-hash 4bd2ce809f84bc7049bfcf3accb58805
|
||||
import { Express } from "express";
|
||||
|
||||
import route1 from "./routes/agents/clearMemory";
|
||||
@ -65,11 +65,11 @@ import route61 from "./routes/production/storyboard/getStoryboardData";
|
||||
import route62 from "./routes/production/storyboard/pollingImage";
|
||||
import route63 from "./routes/production/storyboard/previewImage";
|
||||
import route64 from "./routes/production/storyboard/removeFrame";
|
||||
import route65 from "./routes/production/workbench/confirmSelection";
|
||||
import route66 from "./routes/production/workbench/delVideo";
|
||||
import route67 from "./routes/production/workbench/generateVideo";
|
||||
import route68 from "./routes/production/workbench/generateVideoPrompt";
|
||||
import route69 from "./routes/production/workbench/getChatLines";
|
||||
import route65 from "./routes/production/workbench/addTrack";
|
||||
import route66 from "./routes/production/workbench/confirmSelection";
|
||||
import route67 from "./routes/production/workbench/delVideo";
|
||||
import route68 from "./routes/production/workbench/generateVideo";
|
||||
import route69 from "./routes/production/workbench/generateVideoPrompt";
|
||||
import route70 from "./routes/production/workbench/getVideoModelDetail";
|
||||
import route71 from "./routes/production/workbench/videoPolling";
|
||||
import route72 from "./routes/project/addProject";
|
||||
@ -189,11 +189,11 @@ export default async (app: Express) => {
|
||||
app.use("/api/production/storyboard/pollingImage", route62);
|
||||
app.use("/api/production/storyboard/previewImage", route63);
|
||||
app.use("/api/production/storyboard/removeFrame", route64);
|
||||
app.use("/api/production/workbench/confirmSelection", route65);
|
||||
app.use("/api/production/workbench/delVideo", route66);
|
||||
app.use("/api/production/workbench/generateVideo", route67);
|
||||
app.use("/api/production/workbench/generateVideoPrompt", route68);
|
||||
app.use("/api/production/workbench/getChatLines", route69);
|
||||
app.use("/api/production/workbench/addTrack", route65);
|
||||
app.use("/api/production/workbench/confirmSelection", route66);
|
||||
app.use("/api/production/workbench/delVideo", route67);
|
||||
app.use("/api/production/workbench/generateVideo", route68);
|
||||
app.use("/api/production/workbench/generateVideoPrompt", route69);
|
||||
app.use("/api/production/workbench/getVideoModelDetail", route70);
|
||||
app.use("/api/production/workbench/videoPolling", route71);
|
||||
app.use("/api/project/addProject", route72);
|
||||
|
||||
21
src/routes/production/workbench/addTrack.ts
Normal file
21
src/routes/production/workbench/addTrack.ts
Normal file
@ -0,0 +1,21 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
projectId: z.number(),
|
||||
scriptId: z.number(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { projectId, scriptId } = req.body;
|
||||
const [id] = await u.db("o_videoTrack").insert({
|
||||
projectId,
|
||||
scriptId,
|
||||
});
|
||||
res.status(200).send(success(id));
|
||||
},
|
||||
);
|
||||
@ -5,30 +5,26 @@ import { v4 as uuidv4 } from "uuid";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
scriptId: z.number(),
|
||||
projectId: z.number(),
|
||||
storyboardId: z.number(),
|
||||
scriptId: z.number(),
|
||||
uploadData: z.array(
|
||||
z.object({
|
||||
id: z.number(),
|
||||
type: z.string(),
|
||||
}),
|
||||
),
|
||||
prompt: z.string(),
|
||||
data: z
|
||||
.array(
|
||||
z.object({
|
||||
id: z.number(),
|
||||
type: z.string(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
model: z.string(),
|
||||
duration: z.number(),
|
||||
resolution: z.string(),
|
||||
audio: z.boolean().optional(),
|
||||
mode: z.string(),
|
||||
resolution: z.string(),
|
||||
duration: z.number(),
|
||||
audio: z.boolean().optional(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { scriptId, projectId, storyboardId, prompt, data, model, duration, resolution, audio, mode } = req.body;
|
||||
const { scriptId, projectId, prompt, uploadData, model, duration, resolution, audio, mode } = req.body;
|
||||
//获取生成视频比例
|
||||
const ratio = await u.db("o_project").select("videoRatio").where("id", projectId).first();
|
||||
const videoPath = `/${projectId}/video/${uuidv4()}.mp4`; //视频保存路径
|
||||
@ -38,35 +34,12 @@ export default router.post(
|
||||
time: Date.now(),
|
||||
state: "生成中",
|
||||
scriptId,
|
||||
storyboardId,
|
||||
projectId,
|
||||
};
|
||||
const [videoId] = await u.db("o_video").insert(videoData);
|
||||
//查询分镜是否已有配置
|
||||
const config = await u.db("o_videoConfig").where({ storyboardId }).first();
|
||||
//保存配置
|
||||
if (config) {
|
||||
await u
|
||||
.db("o_videoConfig")
|
||||
.update({ audio, model, mode, data: JSON.stringify(data), resolution, duration, prompt, updateTime: Date.now() })
|
||||
.where({ id: config.id });
|
||||
} else {
|
||||
await u.db("o_videoConfig").insert({
|
||||
storyboardId,
|
||||
audio,
|
||||
model,
|
||||
mode,
|
||||
data: JSON.stringify(data),
|
||||
resolution,
|
||||
duration,
|
||||
prompt,
|
||||
createTime: Date.now(),
|
||||
updateTime: Date.now(),
|
||||
});
|
||||
}
|
||||
//查询出图片数据
|
||||
const images = await Promise.all(
|
||||
data.map(async (item: { id: number; type: string }) => {
|
||||
uploadData.map(async (item: { id: number; type: string }) => {
|
||||
if (item.type === "storyboard") {
|
||||
const filePath = await u.db("o_storyboard").where("id", item.id).select("filePath").first();
|
||||
return filePath?.filePath;
|
||||
@ -93,17 +66,11 @@ export default router.post(
|
||||
(async () => {
|
||||
try {
|
||||
const relatedObjects = {
|
||||
id: storyboardId,
|
||||
projectId,
|
||||
videoId,
|
||||
scriptId,
|
||||
type: "视频",
|
||||
};
|
||||
const systemPrompt = `你是一个专业的视频生成引擎，能够根据用户提供的提示词、图片和参数生成高质量的视频内容。请严格按照用户的需求进行视频创作，确保输出的视频符合以下要求：
|
||||
1. 视频内容必须与用户提供的提示词和图片相关联，准确反映用户的创意意图。
|
||||
2. 视频质量应达到专业水平，画面清晰、流畅，符合用户指定的分辨率和时长要求。
|
||||
3. 视频风格应与用户指定的模式数据相匹配，包括色彩、音乐、特效等元素。
|
||||
4. 视频中应包含用户提供的图片，并在视频中适当展示，以增强视频的视觉效果。
|
||||
5. 如果用户指定了音频，请确保视频中的音频与视频内容相匹配，符合用户的创意意图。`;
|
||||
|
||||
const aiVideo = u.Ai.Video(model);
|
||||
await aiVideo.run(
|
||||
{
|
||||
@ -124,7 +91,6 @@ export default router.post(
|
||||
);
|
||||
await aiVideo.save(videoPath);
|
||||
await u.db("o_video").where("id", videoId).update({ state: "生成成功" });
|
||||
// await u.db("o_videoConfig").where("storyboardId", storyboardId).update({ videoId, updateTime: Date.now() });
|
||||
} catch (error: any) {
|
||||
await u
|
||||
.db("o_video")
|
||||
|
||||
@ -9,49 +9,24 @@ export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
projectId: z.number(),
|
||||
storyboardId: z.number(),
|
||||
prompt: z.array(z.string()),
|
||||
model: z.string(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { projectId, storyboardId } = req.body;
|
||||
|
||||
// 查询分镜及其关联的资产提示词
|
||||
const data = await u
|
||||
.db("o_storyboard")
|
||||
.leftJoin("o_assets2Storyboard", "o_storyboard.id", "o_assets2Storyboard.storyboardId")
|
||||
.leftJoin("o_assets", "o_assets2Storyboard.assetId", "o_assets.id")
|
||||
.leftJoin("o_videoConfig", "o_storyboard.id", "o_videoConfig.storyboardId")
|
||||
.where("o_storyboard.id", storyboardId)
|
||||
.select("o_storyboard.id", "o_storyboard.prompt", "o_assets.prompt as assetPrompt", "o_videoConfig.model as videoModel");
|
||||
|
||||
if (data.length === 0) {
|
||||
return res.status(200).send(success({ data: null }));
|
||||
}
|
||||
|
||||
// 聚合资产提示词
|
||||
const prompt = data[0].prompt || "";
|
||||
const videoModel = data[0].videoModel || "";
|
||||
const assetPrompts = data.map((row) => row.assetPrompt).filter(Boolean) as string[];
|
||||
|
||||
// 确定视频模型
|
||||
let model = videoModel;
|
||||
if (!model) {
|
||||
const project = await u.db("o_project").where("id", projectId).select("videoModel").first();
|
||||
model = project?.videoModel || "";
|
||||
}
|
||||
if (!model) {
|
||||
return res.status(200).send(success({ data: null }));
|
||||
}
|
||||
|
||||
const systemPrompt = `你是一个专业的${model}视频生成助手。请根据分镜提示词和关联资产提示词，生成一段完整的、可直接用于视频生成模型的中文提示词。`;
|
||||
const userContent = `分镜提示词：${prompt || "无"}\n资产提示词：${assetPrompts.length > 0 ? assetPrompts.join("\n") : "无"}`;
|
||||
|
||||
const { projectId, prompt, model } = req.body;
|
||||
const [id, modelData] = model.split(":");
|
||||
const projectData = await u.db("o_project").select("*").where({ id: projectId }).first();
|
||||
const artStyle = projectData?.artStyle || "无";
|
||||
const visualManual = await u.getArtPrompt(artStyle, "art_storyboard_video");
|
||||
const { text } = await u.Ai.Text("universalAi").invoke({
|
||||
system: systemPrompt,
|
||||
messages: [{ role: "user", content: userContent }],
|
||||
system: visualManual,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: `你是一个专业的${modelData}视频生成助手。请根据以下提示词，生成一段完整的、可直接用于视频生成模型的中文提示词。${prompt.join(",")}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await u.db("o_storyboard").where("id", storyboardId).update({ videoPrompt: text });
|
||||
|
||||
res.status(200).send(success({ data: { storyboardId, videoPrompt: text } }));
|
||||
res.status(200).send(success(text));
|
||||
},
|
||||
);
|
||||
|
||||
@ -1,50 +0,0 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { useSkill } from "@/utils/agent/skillsTools";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import { Output } from "ai";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
list: z.array(
|
||||
z.object({
|
||||
prompt: z.string(),
|
||||
videoId: z.number(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { list } = req.body;
|
||||
const data = await Promise.all(
|
||||
list.map(async (item: any) => {
|
||||
const output = await getLines(item.prompt);
|
||||
return { ...item, prompt: output };
|
||||
}),
|
||||
);
|
||||
res.status(200).send(success(data));
|
||||
},
|
||||
);
|
||||
|
||||
async function getLines(prompt: string) {
|
||||
const skill = await useSkill("universal_agent.md");//todo：改为AI
|
||||
|
||||
const resText = await u.Ai.Text("universalAi").invoke({
|
||||
system: skill.prompt,
|
||||
messages: [{ role: "user", content: prompt }],
|
||||
tools: skill.tools,
|
||||
output: Output.array({
|
||||
element: z.object({
|
||||
lines: z.string().describe("台词内容"),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
console.log("%c Line:36 🍉 resText", "background:#e41a6a", resText);
|
||||
|
||||
const parseLines = JSON.parse(resText.text);
|
||||
const chatLines = parseLines.elements.map((i: any) => i.lines);
|
||||
return chatLines;
|
||||
}
|
||||
22
src/types/database.d.ts
vendored
22
src/types/database.d.ts
vendored
@ -1,4 +1,4 @@
|
||||
// @db-hash d1fbadf1691063f5a730337685fd85b1
|
||||
// @db-hash c2029b55b7dcdcf64788dafc34799fea
|
||||
//该文件由脚本自动生成，请勿手动修改
|
||||
|
||||
export interface memories {
|
||||
@ -114,6 +114,7 @@ export interface o_project {
|
||||
'imageModel'?: string | null;
|
||||
'imageQuality'?: string | null;
|
||||
'intro'?: string | null;
|
||||
'mode'?: string | null;
|
||||
'name'?: string | null;
|
||||
'projectType'?: string | null;
|
||||
'type'?: string | null;
|
||||
@ -167,6 +168,7 @@ export interface o_storyboard {
|
||||
'duration'?: string | null;
|
||||
'filePath'?: string | null;
|
||||
'frameMode'?: string | null;
|
||||
'group'?: string | null;
|
||||
'id'?: number;
|
||||
'index'?: number | null;
|
||||
'lines'?: string | null;
|
||||
@ -216,21 +218,13 @@ export interface o_video {
|
||||
'projectId'?: number | null;
|
||||
'scriptId'?: number | null;
|
||||
'state'?: string | null;
|
||||
'storyboardId'?: number | null;
|
||||
'time'?: number | null;
|
||||
'videoTrackId'?: number | null;
|
||||
}
|
||||
export interface o_videoConfig {
|
||||
'audio'?: number | null;
|
||||
'createTime'?: number | null;
|
||||
'data'?: string | null;
|
||||
'duration'?: number | null;
|
||||
export interface o_videoTrack {
|
||||
'id'?: number;
|
||||
'mode'?: string | null;
|
||||
'model'?: string | null;
|
||||
'prompt'?: string | null;
|
||||
'resolution'?: string | null;
|
||||
'storyboardId'?: number | null;
|
||||
'updateTime'?: number | null;
|
||||
'projectId'?: number | null;
|
||||
'scriptId'?: number | null;
|
||||
'videoId'?: number | null;
|
||||
}
|
||||
|
||||
@ -260,5 +254,5 @@ export interface DB {
|
||||
"o_user": o_user;
|
||||
"o_vendorConfig": o_vendorConfig;
|
||||
"o_video": o_video;
|
||||
"o_videoConfig": o_videoConfig;
|
||||
"o_videoTrack": o_videoTrack;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user