zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改 生成分镜

Browse Source
This commit is contained in:
zhishi 2026-03-30 21:50:24 +08:00
parent c4dea916c0
commit 3f597232cd
9 changed files with 425 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/agents/productionAgent/tools.ts
View File

@ -14,7 +14,7 @@ const deriveAssetSchema = z.object({
state: z.enum(["未生成", "生成中", "已完成", "生成失败"]).describe("衍生资产生成状态"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
});
const assetItemSchema = z.object({
export const assetItemSchema = z.object({
id: z.number().describe("资产唯一标识"),
name: z.string().describe("资产名称"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
@ -100,18 +100,20 @@ export default (toolCpnfig: ToolConfig) => {
id: z.number().nullable().describe("衍生资产ID,如果新增则为空"),
name: z.string().describe("衍生资产名称"),
desc: z.string().describe("衍生资产描述"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
}),
execute: async (deriveAsset) => {
const thinking = msg.thinking("正在操作资产...");
const { projectId, scriptId } = resTool.data;
const startTime = Date.now();
const parentAssets = await u.db("o_assets").where("id", deriveAsset.assetsId).select("id", "type").first();
if (!parentAssets) return "关联的资产不存在";
const data = {
id: deriveAsset.id ?? undefined,
assetsId: deriveAsset.assetsId,
projectId,
name: deriveAsset.name,
type: deriveAsset.type,
type: parentAssets.type,
describe: deriveAsset.desc,
startTime,
};
@ -149,41 +151,6 @@ export default (toolCpnfig: ToolConfig) => {
},
}),
add_storyboard: tool({
description: "新增或更新分镜面板",
inputSchema: z.object({
id: z.number().nullable().describe("分镜面板ID,如果新增则为空"),
title: z.string().describe("分镜面板名称"),
desc: z.string().describe("分镜面板描述"),
group: z.number().describe("分镜面板分组,根据这个字段 对分镜图片，进行同时生成视频，例如 同一分组的两张图片会被用于首尾帧生成视频"),
}),
execute: async (storyboard) => {
const thinking = msg.thinking("正在操作资产...");
const { projectId, scriptId } = resTool.data;
const createTime = Date.now();
console.log("%c Line:161 🍤 storyboard", "background:#e41a6a", storyboard);
const data = {
id: storyboard.id ?? undefined,
title: storyboard.title,
description: storyboard.desc,
createTime,
scriptId,
};
if (storyboard.id) {
await u.db("o_storyboard").where("id", storyboard.id).update(data);
thinking.appendText(`已更新分镜面板，ID: ${storyboard.id}\n`);
} else {
const [insertedId] = await u.db("o_storyboard").insert(data);
data.id = insertedId;
thinking.appendText(`已新增分镜面板，ID: ${insertedId}\n`);
}
const res = await new Promise((resolve) => socket.emit("addStoryboard", data, (res: any) => resolve(res)));
thinking.updateTitle("分镜面板操作完成");
thinking.complete();
return res ?? "操作成功";
},
}),
generate_deriveAsset: tool({
description: "生成衍生资产",
inputSchema: z.object({
@ -191,25 +158,42 @@ export default (toolCpnfig: ToolConfig) => {
}),
execute: async ({ id }) => {
const thinking = msg.thinking("正在生成衍生资产...");
const res = await new Promise((resolve) => socket.emit("generateDeriveAsset", { id }, (res: any) => resolve(res)));
thinking.appendText(`已生成衍生资产，ID: ${id}\n`);
thinking.updateTitle("衍生资产生成完成");
new Promise((resolve) => socket.emit("generateDeriveAsset", { id }, (res: any) => resolve(res)))
.then((res) => {
thinking.appendText(`已生成衍生资产，ID: ${JSON.stringify(res, null, 2)}\n`);
thinking.updateTitle("衍生资产开始完成");
thinking.complete();
return res ?? "生成失败";
})
.catch((e) => {
thinking.appendText("衍生资产生成失败:\n" + u.error(e).message);
thinking.updateTitle("衍生资产生成失败");
thinking.complete();
});
return "开始生成衍生资产";
},
}),
generate_storyboard: tool({
description: "生成分镜图片",
inputSchema: z.object({
storyboardIds: z.array(z.number()).describe("分镜ID列表"),
ids: z.array(z.number()).describe("分镜面板中需要更新的分镜 ID 列表,传入id仅作对应分镜更新用，不传入则全部生成"),
}),
execute: async ({ storyboardIds }) => {
execute: async ({ ids }) => {
console.log("%c Line:176 🍒 ids", "background:#ea7e5c", ids);
const thinking = msg.thinking("正在生成分镜...");
const res = await new Promise((resolve) => socket.emit("generateStoryboard", { storyboardIds }, (res: any) => resolve(res)));
new Promise((resolve) => socket.emit("generateStoryboard", { ids }, (res: any) => resolve(res)))
.then((res) => {
thinking.appendText("生成的分镜数据:\n" + JSON.stringify(res, null, 2));
thinking.updateTitle("分镜生成完成");
thinking.complete();
return res;
})
.catch((e) => {
thinking.appendText("分镜生成失败:\n" + u.error(e).message);
thinking.updateTitle("分镜生成失败");
thinking.complete();
});
return "开始生成分镜";
},
}),
};

1
src/lib/initDB.ts
View File

@ -469,6 +469,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
table.text("model");
table.text("resolution");
table.text("state");
table.text("reason");
table.primary(["id"]);
table.unique(["id"]);
},

244
src/router.ts
View File

@ -1,4 +1,4 @@
// @routes-hash 845d6aff66aab1f458a9f08f4f2eed34
// @routes-hash 8d0bd75a9e06280f64490cd32d7a5b2e
import { Express } from "express";
import route1 from "./routes/agents/clearMemory";
@ -59,66 +59,67 @@ import route55 from "./routes/production/getStoryboardData";
import route56 from "./routes/production/saveFlowData";
import route57 from "./routes/production/storyboard/batchGenerateImage";
import route58 from "./routes/production/storyboard/downPreviewImage";
import route59 from "./routes/production/storyboard/getStoryboardData";
import route60 from "./routes/production/storyboard/pollingImage";
import route61 from "./routes/production/storyboard/previewImage";
import route62 from "./routes/production/workbench/confirmSelection";
import route63 from "./routes/production/workbench/delVideo";
import route64 from "./routes/production/workbench/generateVideo";
import route65 from "./routes/production/workbench/generateVideoPrompt";
import route66 from "./routes/production/workbench/getChatLines";
import route67 from "./routes/production/workbench/getVideoModelDetail";
import route68 from "./routes/production/workbench/videoPolling";
import route69 from "./routes/project/addProject";
import route70 from "./routes/project/addVisual";
import route71 from "./routes/project/addVisualManual";
import route72 from "./routes/project/deleteVisualManual";
import route73 from "./routes/project/delProject";
import route74 from "./routes/project/editProject";
import route75 from "./routes/project/editVisualManual";
import route76 from "./routes/project/getProject";
import route77 from "./routes/project/getVisualManual";
import route78 from "./routes/project/visualManual";
import route79 from "./routes/script/addScript";
import route80 from "./routes/script/delScript";
import route81 from "./routes/script/exportScript";
import route82 from "./routes/script/extractAssets";
import route83 from "./routes/script/getScrptApi";
import route84 from "./routes/script/pollScriptAssets";
import route85 from "./routes/script/updateScript";
import route86 from "./routes/scriptAgent/getPlanData";
import route87 from "./routes/scriptAgent/setPlanData";
import route88 from "./routes/setting/about/checkUpdate";
import route89 from "./routes/setting/about/downloadApp";
import route90 from "./routes/setting/agentDeploy/agentSetKey";
import route91 from "./routes/setting/agentDeploy/deployAgentModel";
import route92 from "./routes/setting/agentDeploy/getAgentDeploy";
import route93 from "./routes/setting/dbConfig/clearData";
import route94 from "./routes/setting/dev/getSwitchAiDevTool";
import route95 from "./routes/setting/dev/updateSwitchAiDevTool";
import route96 from "./routes/setting/fileManagement/openFolder";
import route97 from "./routes/setting/getTextModel";
import route98 from "./routes/setting/loginConfig/getUser";
import route99 from "./routes/setting/loginConfig/updateUserPwd";
import route100 from "./routes/setting/memoryConfig/delAllMemory";
import route101 from "./routes/setting/memoryConfig/getMemory";
import route102 from "./routes/setting/memoryConfig/sureMemory";
import route103 from "./routes/setting/promptManage/getPrompt";
import route104 from "./routes/setting/promptManage/updatePrompt";
import route105 from "./routes/setting/skillManagement/getSkillContent";
import route106 from "./routes/setting/skillManagement/getSkillList";
import route107 from "./routes/setting/skillManagement/saveSkillContent";
import route108 from "./routes/setting/vendorConfig/addVendor";
import route109 from "./routes/setting/vendorConfig/deleteVendor";
import route110 from "./routes/setting/vendorConfig/getVendorList";
import route111 from "./routes/setting/vendorConfig/modelTest";
import route112 from "./routes/setting/vendorConfig/updateCode";
import route113 from "./routes/setting/vendorConfig/updateVendor";
import route114 from "./routes/task/getProject";
import route115 from "./routes/task/getTaskApi";
import route116 from "./routes/task/getTaskCategories";
import route117 from "./routes/task/taskDetails";
import route118 from "./routes/test/test";
import route59 from "./routes/production/storyboard/generatestory2Image";
import route60 from "./routes/production/storyboard/getStoryboardData";
import route61 from "./routes/production/storyboard/pollingImage";
import route62 from "./routes/production/storyboard/previewImage";
import route63 from "./routes/production/workbench/confirmSelection";
import route64 from "./routes/production/workbench/delVideo";
import route65 from "./routes/production/workbench/generateVideo";
import route66 from "./routes/production/workbench/generateVideoPrompt";
import route67 from "./routes/production/workbench/getChatLines";
import route68 from "./routes/production/workbench/getVideoModelDetail";
import route69 from "./routes/production/workbench/videoPolling";
import route70 from "./routes/project/addProject";
import route71 from "./routes/project/addVisual";
import route72 from "./routes/project/addVisualManual";
import route73 from "./routes/project/deleteVisualManual";
import route74 from "./routes/project/delProject";
import route75 from "./routes/project/editProject";
import route76 from "./routes/project/editVisualManual";
import route77 from "./routes/project/getProject";
import route78 from "./routes/project/getVisualManual";
import route79 from "./routes/project/visualManual";
import route80 from "./routes/script/addScript";
import route81 from "./routes/script/delScript";
import route82 from "./routes/script/exportScript";
import route83 from "./routes/script/extractAssets";
import route84 from "./routes/script/getScrptApi";
import route85 from "./routes/script/pollScriptAssets";
import route86 from "./routes/script/updateScript";
import route87 from "./routes/scriptAgent/getPlanData";
import route88 from "./routes/scriptAgent/setPlanData";
import route89 from "./routes/setting/about/checkUpdate";
import route90 from "./routes/setting/about/downloadApp";
import route91 from "./routes/setting/agentDeploy/agentSetKey";
import route92 from "./routes/setting/agentDeploy/deployAgentModel";
import route93 from "./routes/setting/agentDeploy/getAgentDeploy";
import route94 from "./routes/setting/dbConfig/clearData";
import route95 from "./routes/setting/dev/getSwitchAiDevTool";
import route96 from "./routes/setting/dev/updateSwitchAiDevTool";
import route97 from "./routes/setting/fileManagement/openFolder";
import route98 from "./routes/setting/getTextModel";
import route99 from "./routes/setting/loginConfig/getUser";
import route100 from "./routes/setting/loginConfig/updateUserPwd";
import route101 from "./routes/setting/memoryConfig/delAllMemory";
import route102 from "./routes/setting/memoryConfig/getMemory";
import route103 from "./routes/setting/memoryConfig/sureMemory";
import route104 from "./routes/setting/promptManage/getPrompt";
import route105 from "./routes/setting/promptManage/updatePrompt";
import route106 from "./routes/setting/skillManagement/getSkillContent";
import route107 from "./routes/setting/skillManagement/getSkillList";
import route108 from "./routes/setting/skillManagement/saveSkillContent";
import route109 from "./routes/setting/vendorConfig/addVendor";
import route110 from "./routes/setting/vendorConfig/deleteVendor";
import route111 from "./routes/setting/vendorConfig/getVendorList";
import route112 from "./routes/setting/vendorConfig/modelTest";
import route113 from "./routes/setting/vendorConfig/updateCode";
import route114 from "./routes/setting/vendorConfig/updateVendor";
import route115 from "./routes/task/getProject";
import route116 from "./routes/task/getTaskApi";
import route117 from "./routes/task/getTaskCategories";
import route118 from "./routes/task/taskDetails";
import route119 from "./routes/test/test";
export default async (app: Express) => {
app.use("/api/agents/clearMemory", route1);
@ -179,64 +180,65 @@ export default async (app: Express) => {
app.use("/api/production/saveFlowData", route56);
app.use("/api/production/storyboard/batchGenerateImage", route57);
app.use("/api/production/storyboard/downPreviewImage", route58);
app.use("/api/production/storyboard/getStoryboardData", route59);
app.use("/api/production/storyboard/pollingImage", route60);
app.use("/api/production/storyboard/previewImage", route61);
app.use("/api/production/workbench/confirmSelection", route62);
app.use("/api/production/workbench/delVideo", route63);
app.use("/api/production/workbench/generateVideo", route64);
app.use("/api/production/workbench/generateVideoPrompt", route65);
app.use("/api/production/workbench/getChatLines", route66);
app.use("/api/production/workbench/getVideoModelDetail", route67);
app.use("/api/production/workbench/videoPolling", route68);
app.use("/api/project/addProject", route69);
app.use("/api/project/addVisual", route70);
app.use("/api/project/addVisualManual", route71);
app.use("/api/project/deleteVisualManual", route72);
app.use("/api/project/delProject", route73);
app.use("/api/project/editProject", route74);
app.use("/api/project/editVisualManual", route75);
app.use("/api/project/getProject", route76);
app.use("/api/project/getVisualManual", route77);
app.use("/api/project/visualManual", route78);
app.use("/api/script/addScript", route79);
app.use("/api/script/delScript", route80);
app.use("/api/script/exportScript", route81);
app.use("/api/script/extractAssets", route82);
app.use("/api/script/getScrptApi", route83);
app.use("/api/script/pollScriptAssets", route84);
app.use("/api/script/updateScript", route85);
app.use("/api/scriptAgent/getPlanData", route86);
app.use("/api/scriptAgent/setPlanData", route87);
app.use("/api/setting/about/checkUpdate", route88);
app.use("/api/setting/about/downloadApp", route89);
app.use("/api/setting/agentDeploy/agentSetKey", route90);
app.use("/api/setting/agentDeploy/deployAgentModel", route91);
app.use("/api/setting/agentDeploy/getAgentDeploy", route92);
app.use("/api/setting/dbConfig/clearData", route93);
app.use("/api/setting/dev/getSwitchAiDevTool", route94);
app.use("/api/setting/dev/updateSwitchAiDevTool", route95);
app.use("/api/setting/fileManagement/openFolder", route96);
app.use("/api/setting/getTextModel", route97);
app.use("/api/setting/loginConfig/getUser", route98);
app.use("/api/setting/loginConfig/updateUserPwd", route99);
app.use("/api/setting/memoryConfig/delAllMemory", route100);
app.use("/api/setting/memoryConfig/getMemory", route101);
app.use("/api/setting/memoryConfig/sureMemory", route102);
app.use("/api/setting/promptManage/getPrompt", route103);
app.use("/api/setting/promptManage/updatePrompt", route104);
app.use("/api/setting/skillManagement/getSkillContent", route105);
app.use("/api/setting/skillManagement/getSkillList", route106);
app.use("/api/setting/skillManagement/saveSkillContent", route107);
app.use("/api/setting/vendorConfig/addVendor", route108);
app.use("/api/setting/vendorConfig/deleteVendor", route109);
app.use("/api/setting/vendorConfig/getVendorList", route110);
app.use("/api/setting/vendorConfig/modelTest", route111);
app.use("/api/setting/vendorConfig/updateCode", route112);
app.use("/api/setting/vendorConfig/updateVendor", route113);
app.use("/api/task/getProject", route114);
app.use("/api/task/getTaskApi", route115);
app.use("/api/task/getTaskCategories", route116);
app.use("/api/task/taskDetails", route117);
app.use("/api/test/test", route118);
app.use("/api/production/storyboard/generatestory2Image", route59);
app.use("/api/production/storyboard/getStoryboardData", route60);
app.use("/api/production/storyboard/pollingImage", route61);
app.use("/api/production/storyboard/previewImage", route62);
app.use("/api/production/workbench/confirmSelection", route63);
app.use("/api/production/workbench/delVideo", route64);
app.use("/api/production/workbench/generateVideo", route65);
app.use("/api/production/workbench/generateVideoPrompt", route66);
app.use("/api/production/workbench/getChatLines", route67);
app.use("/api/production/workbench/getVideoModelDetail", route68);
app.use("/api/production/workbench/videoPolling", route69);
app.use("/api/project/addProject", route70);
app.use("/api/project/addVisual", route71);
app.use("/api/project/addVisualManual", route72);
app.use("/api/project/deleteVisualManual", route73);
app.use("/api/project/delProject", route74);
app.use("/api/project/editProject", route75);
app.use("/api/project/editVisualManual", route76);
app.use("/api/project/getProject", route77);
app.use("/api/project/getVisualManual", route78);
app.use("/api/project/visualManual", route79);
app.use("/api/script/addScript", route80);
app.use("/api/script/delScript", route81);
app.use("/api/script/exportScript", route82);
app.use("/api/script/extractAssets", route83);
app.use("/api/script/getScrptApi", route84);
app.use("/api/script/pollScriptAssets", route85);
app.use("/api/script/updateScript", route86);
app.use("/api/scriptAgent/getPlanData", route87);
app.use("/api/scriptAgent/setPlanData", route88);
app.use("/api/setting/about/checkUpdate", route89);
app.use("/api/setting/about/downloadApp", route90);
app.use("/api/setting/agentDeploy/agentSetKey", route91);
app.use("/api/setting/agentDeploy/deployAgentModel", route92);
app.use("/api/setting/agentDeploy/getAgentDeploy", route93);
app.use("/api/setting/dbConfig/clearData", route94);
app.use("/api/setting/dev/getSwitchAiDevTool", route95);
app.use("/api/setting/dev/updateSwitchAiDevTool", route96);
app.use("/api/setting/fileManagement/openFolder", route97);
app.use("/api/setting/getTextModel", route98);
app.use("/api/setting/loginConfig/getUser", route99);
app.use("/api/setting/loginConfig/updateUserPwd", route100);
app.use("/api/setting/memoryConfig/delAllMemory", route101);
app.use("/api/setting/memoryConfig/getMemory", route102);
app.use("/api/setting/memoryConfig/sureMemory", route103);
app.use("/api/setting/promptManage/getPrompt", route104);
app.use("/api/setting/promptManage/updatePrompt", route105);
app.use("/api/setting/skillManagement/getSkillContent", route106);
app.use("/api/setting/skillManagement/getSkillList", route107);
app.use("/api/setting/skillManagement/saveSkillContent", route108);
app.use("/api/setting/vendorConfig/addVendor", route109);
app.use("/api/setting/vendorConfig/deleteVendor", route110);
app.use("/api/setting/vendorConfig/getVendorList", route111);
app.use("/api/setting/vendorConfig/modelTest", route112);
app.use("/api/setting/vendorConfig/updateCode", route113);
app.use("/api/setting/vendorConfig/updateVendor", route114);
app.use("/api/task/getProject", route115);
app.use("/api/task/getTaskApi", route116);
app.use("/api/task/getTaskCategories", route117);
app.use("/api/task/taskDetails", route118);
app.use("/api/test/test", route119);
}

8
src/routes/assets/batchGenerationData.ts
View File

@ -6,7 +6,8 @@ import { validateFields } from "@/middleware/middleware";
const router = express.Router();
// 获取资产
export default router.post("/",
export default router.post(
"/",
validateFields({
projectId: z.number(),
type: z.string(),
@ -15,7 +16,7 @@ export default router.post("/",
limit: z.number(),
}),
async (req, res) => {
const { projectId, type, name, page = 1, limit = 10, } = req.body;
const { projectId, type, name, page = 1, limit = 10 } = req.body;
const offset = (page - 1) * limit;
let query = u.db("o_assets").select("*").where("projectId", projectId).andWhere("type", type);
if (name) {
@ -37,4 +38,5 @@ export default router.post("/",
.count("* as total")
.first()) as any;
res.status(200).send(success({ data: parentAssets, total: totalQuery?.total }));
});
},
);

51
src/routes/production/assets/batchGenerateAssetsImage.ts
View File

@ -5,6 +5,7 @@ import sharp from "sharp";
import { success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { Output } from "ai";
import { urlToBase64 } from "@/utils/vm";
const router = express.Router();
export default router.post(
@ -19,7 +20,25 @@ export default router.post(
const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first();
const assetsDataArr = await u.db("o_assets").whereIn("id", assetIds).select("id", "describe", "name", "type");
const assetsDataArr = await u.db("o_assets").whereIn("id", assetIds).select("id", "describe", "name", "type", "assetsId");
const parentIds = assetsDataArr.map((item) => item.assetsId).filter((id) => id !== null);
const parentAssetsData = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.whereIn("o_assets.id", parentIds as number[])
.select("o_assets.id", "o_image.filePath");
const assetsSrcArr = await Promise.all(
parentAssetsData.map(async (item) => {
return {
src: await u.oss.getFileUrl(item.filePath),
id: item.id,
};
}),
);
const imageUrlRecord: Record<number, string> = {};
assetsSrcArr.forEach((item) => {
imageUrlRecord[item.id] = item.src;
});
const rolePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_character_derivative");
const toolPrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_prop_derivative");
const scenePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_scene_derivative");
@ -28,7 +47,7 @@ export default router.post(
tool: toolPrompt,
scene: scenePrompt,
};
const imageData = [];
for (const item of assetsDataArr) {
const { text } = await u.Ai.Text("universalAi").invoke({
system: `
@ -55,26 +74,42 @@ export default router.post(
resolution: projectSettingData?.imageQuality,
model: projectSettingData?.imageModel,
});
u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`)
.run({
const imageBase64 = imageUrlRecord[item.assetsId!] ? await urlToBase64(imageUrlRecord[item.assetsId!]) : null;
try {
const imageCls = await u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`).run({
prompt: text,
imageBase64: [],
imageBase64: imageBase64 ? [imageBase64] : [],
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
aspectRatio: "16:9",
taskClass: "生成图片",
describe: "资产图片生成",
relatedObjects: JSON.stringify(repeloadObj),
projectId: projectId,
})
.then(async (imageCls) => {
});
const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`;
await imageCls.save(savePath);
// 更新对应数据库
await u.db("o_assets").where("id", item.id).update({ imageId: imageId });
await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
imageData.push({
id: item.id,
state: "已完成",
src: await u.oss.getFileUrl(savePath),
});
} catch (e) {
console.log("%c Line:95 🥛 e", "background:#fca650", e);
await u
.db("o_image")
.where({ id: imageId })
.update({ state: "生成失败", reason: u.error(e).message });
imageData.push({
id: item.id,
state: "生成失败",
src: "",
});
}
}
return res.status(200).send(success());
return res.status(200).send(success(imageData));
},
);

150
src/routes/production/storyboard/batchGenerateImage.ts
View File

@ -4,28 +4,137 @@ import { z } from "zod";
import sharp from "sharp";
import { success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { Output } from "ai";
import { Output, tool } from "ai";
import { urlToBase64 } from "@/utils/vm";
import { assetItemSchema } from "@/agents/productionAgent/tools";
const router = express.Router();
export type AssetData = z.infer<typeof assetItemSchema>;
export default router.post(
"/",
validateFields({
storyboardIds: z.array(z.number()),
storyboardIds: z.array(z.number()).optional(),
projectId: z.number(),
scriptId: z.number(),
script: z.string(),
scriptPlan: z.string(),
storyboardTable: z.string(),
assets: z.array(assetItemSchema),
}),
async (req, res) => {
const { storyboardIds, projectId, scriptId } = req.body;
const {
storyboardIds,
projectId,
scriptId,
script,
scriptPlan,
storyboardTable,
assets,
}: {
storyboardIds: number[];
projectId: number;
scriptId: number;
script: string;
scriptPlan: string;
storyboardTable: string;
assets: AssetData[];
} = req.body;
// 当没有 storyboardIds 时，通过 AI 生成新的分镜面板数据
let finalStoryboardIds: number[] = storyboardIds || [];
if (!storyboardIds || storyboardIds.length === 0) {
const createdIds: number[] = [];
const resultTools = tool({
description: "结果输出工具（必须调用）",
inputSchema: z.object({
items: z.array(
z.object({
title: z.string().describe("分镜名称"),
description: z.string().describe("分镜详细描述"),
relatedAssets: z.array(z.number()).describe("关联衍生资产id数组"),
}),
),
}),
execute: async (resData) => {
console.log("%c Line:46 🌰 resData", "background:#93c0a4", resData.items);
for (const item of resData.items) {
const [id] = await u.db("o_storyboard").insert({
title: item.title,
description: item.description,
scriptId: scriptId,
});
createdIds.push(id);
if (item.relatedAssets.length === 0) continue;
await u.db("o_assets2Storyboard").insert(item.relatedAssets.map((i) => ({ storyboardId: id, assetId: i })));
console.log("%c Line:68 🍷 createdIds", "background:#33a5ff", createdIds);
}
return true;
},
});
const { text } = await u.Ai.Text("universalAi").invoke({
system: `
你需要根据用户提供的剧本、分镜表、拍摄计划和资产列表，来生成一个分镜面板，内容结构为 [{title:"分镜名称",description:"分镜详细描述",relatedAssets:关联衍生资产id}]。
你必须调用 resultTools 来输出结果，传入的参数需要包含 items 字段，items 是一个数组，每个元素包含 title（分镜名称）,description（分镜详细描述）,relatedAssets（关联衍生资产id数组）。请直接输出调用工具的代码，不要做任何多余的描述性文字，必须等待工具调用完成。调用工具后你本身的回复 请保持空白，不要添加任何内容。`,
messages: [
{
role: "user",
content: `
====== 剧本 ======
${script}
====== 分镜表 ======
${storyboardTable}
====== 拍摄计划 ======
${scriptPlan}
====== 资产列表 ======
${assets.map((i) => i.derive.map((t) => `衍生资产名称:${t.name},衍生资产类型:${t.type},关联资产ID:${t.assetsId}`).join("\n")).join("\n")}
`,
},
],
tools: { resultTools },
});
console.log("%c Line:52 🍢 text", "background:#93c0a4", text);
finalStoryboardIds = createdIds;
}
await u.db("o_storyboard").whereIn("id", finalStoryboardIds).where("scriptId", scriptId).update({ state: "生成中" });
console.log("%c Line:98 🍯 finalStoryboardIds", "background:#3f7cff", finalStoryboardIds);
if (finalStoryboardIds.length === 0) {
res.status(200).send(success());
return;
}
const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first();
const sceneArkPrompt = u.getArtPrompt(projectSettingData?.artStyle || "", "art_storyboard");
const storyboardData = await u.db("o_storyboard").whereIn("id", storyboardIds).select("id", "description", "title");
const storyboardData = await u.db("o_storyboard").where("scriptId", scriptId).whereIn("id", finalStoryboardIds);
const assetData = await u
.db("o_assets")
.leftJoin("o_assets2Storyboard", "o_assets.id", "o_assets2Storyboard.assetId")
.whereIn("o_assets2Storyboard.storyboardId", finalStoryboardIds)
.select("o_assets2Storyboard.storyboardId", "o_assets.imageId");
const assetRecord: Record<number, number[]> = {};
assetData.forEach((item: any) => {
if (!assetRecord[item.storyboardId]) {
assetRecord[item.storyboardId] = [];
}
assetRecord[item.storyboardId].push(item.imageId);
});
res.status(200).send(
success(
storyboardData.map((i) => ({
id: i.id,
title: i.title,
description: i.description,
prompt: "",
associateAssetsIds: assetRecord[i.id!],
src: null,
state: i.state,
})),
),
);
for (const item of storyboardData) {
const { text } = await u.Ai.Text("universalAi").invoke({
system: `
你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的分镜图片。请你只优化提示词，不要添加任何额外的描述性文字，请以JSON格式输出: [{id:"对应分镜ID",prompt:"分镜提示词"}]。
你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我生成一段提示词以便生成更符合项目美术风格的分镜图片。直接输出提示词，不做任何解释说明。
美术风格：${sceneArkPrompt}`,
messages: [
{
@ -34,6 +143,8 @@ export default router.post(
},
],
});
console.log("%c Line:27 🍫 text", "background:#ffdd4d", text);
const repeloadObj = {
prompt: text,
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
@ -46,7 +157,7 @@ export default router.post(
u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`)
.run({
prompt: text,
imageBase64: [],
imageBase64: await getAssetsImageBase64(assetRecord[item.id!] || []),
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
aspectRatio: "16:9",
taskClass: "生成图片",
@ -72,7 +183,28 @@ export default router.post(
});
});
}
return res.status(200).send(success());
},
);
async function getAssetsImageBase64(imageIds: number[]) {
if (imageIds.length === 0) return [];
const imagePaths = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.whereIn("o_assets.id", imageIds)
.select("o_assets.id", "o_image.filePath");
if (!imagePaths.length) return [];
const imageUrls = await Promise.all(
imagePaths.map(async (i) => {
if (i.filePath) {
try {
return await urlToBase64(await u.oss.getFileUrl(i.filePath));
} catch {
return null;
}
} else {
return null;
}
}),
);
return imageUrls.filter(Boolean) as string[];
}

2
src/routes/script/extractAssets.ts
View File

@ -63,8 +63,6 @@ export default router.post(
if (!scriptIds.length) return res.status(400).send(error("请先选择剧本"));
const scripts = await u.db("o_script").whereIn("id", scriptIds);
const intansce = u.Ai.Text("universalAi");
const novelData = await u.db("o_novel").where("projectId", projectId).select("chapterData");
if (!novelData || novelData.length === 0) return res.status(400).send(error("请先上传小说"));
await u.db("o_script").whereIn("id", scriptIds).update({
extractState: 0,
});

15
src/types/database.d.ts vendored
View File

@ -1,6 +1,13 @@
// @db-hash 93b2462070c45c2b449e9a18c4e88763
// @db-hash f7bc2fdb80756d5536929eb47155578b
//该文件由脚本自动生成，请勿手动修改
export interface _o_script_old_20260327 {
'content'?: string | null;
'createTime'?: number | null;
'id'?: number;
'name'?: string | null;
'projectId'?: number | null;
}
export interface memories {
'content': string;
'createTime': number;
@ -21,7 +28,7 @@ export interface o_agentDeploy {
'model'?: string | null;
'modelName'?: string | null;
'name'?: string | null;
'vendorId'?: string | null;
'vendorId'?: number | null;
}
export interface o_agentWorkData {
'createTime'?: number | null;
@ -47,7 +54,6 @@ export interface o_assets {
'name'?: string | null;
'projectId'?: number | null;
'prompt'?: string | null;
'promptState'?: string | null;
'remark'?: string | null;
'scriptId'?: number | null;
'startTime'?: number | null;
@ -167,7 +173,7 @@ export interface o_storyboard {
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'index'?: number | null;
'index'?: string | null;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
@ -232,6 +238,7 @@ export interface o_videoConfig {
}
export interface DB {
"_o_script_old_20260327": _o_script_old_20260327;
"memories": memories;
"o_agentDeploy": o_agentDeploy;
"o_agentWorkData": o_agentWorkData;

68
yarn.lock
View File

@ -40,10 +40,10 @@
"@hono/node-server" "^1.13.7"
hono "^4.6.14"
"@ai-sdk/gateway@3.0.82":
version "3.0.82"
resolved "https://registry.npmmirror.com/@ai-sdk/gateway/-/gateway-3.0.82.tgz#904f150486ad3f9456c71ba6732c3f4b518c76d1"
integrity sha512-ddB9FrkHZank1zyx13vypU0RrPjsXWj3NvlsrJ4yFQnrpR+xh48W4wO9ijndUueBsaADjlvMz2Ghv8yq5tZGCQ==
"@ai-sdk/gateway@3.0.83":
version "3.0.83"
resolved "https://registry.npmmirror.com/@ai-sdk/gateway/-/gateway-3.0.83.tgz#214812d3a2f15447adf4ee81d3ed77dd137b41b3"
integrity sha512-LvlWujbSdEkTBXBLFtF7GS6riXdHhH0O+DpDrCaNQvXeHmSF2jKsOg7JWXiCgygAHM5cWFAO3JYmZp83DjiuBQ==
dependencies:
"@ai-sdk/provider" "3.0.8"
"@ai-sdk/provider-utils" "4.0.21"
@ -423,9 +423,9 @@
integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==
"@hono/node-server@^1.13.7":
version "1.19.11"
resolved "https://registry.npmmirror.com/@hono/node-server/-/node-server-1.19.11.tgz#dc419f0826dd2504e9fc86ad289d5636a0444e2f"
integrity sha512-dr8/3zEaB+p0D2n/IUrlPF1HZm586qgJNXK1a9fhg/PzdtkK7Ksd5l312tJX2yBuALqDYBlG20QEbayqPyxn+g==
version "1.19.12"
resolved "https://registry.npmmirror.com/@hono/node-server/-/node-server-1.19.12.tgz#dae075247959b6d7d2dba4c8bdc8c452ca0c7b40"
integrity sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw==
"@huggingface/jinja@^0.5.3":
version "0.5.6"
@ -1042,9 +1042,9 @@
integrity sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w==
"@xmldom/xmldom@^0.8.8":
version "0.8.11"
resolved "https://registry.npmmirror.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608"
integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==
version "0.8.12"
resolved "https://registry.npmmirror.com/@xmldom/xmldom/-/xmldom-0.8.12.tgz#cf488a5435fa06c7374ad1449c69cea0f823624b"
integrity sha512-9k/gHF6n/pAi/9tqr3m3aqkuiNosYTurLLUtc7xQ9sxB/wm7WPygCv8GYa6mS0fLJEHhqMC1ATYhz++U/lRHqg==
abbrev@1, abbrev@^1.0.0:
version "1.1.1"
@ -1117,11 +1117,11 @@ aggregate-error@^3.0.0:
indent-string "^4.0.0"
ai@^6.0.67:
version "6.0.140"
resolved "https://registry.npmmirror.com/ai/-/ai-6.0.140.tgz#1135f3fa05c760e346984f02f55de8e2b9800691"
integrity sha512-+jf6fQDPZe+gQlzPoP5mzy+DdfYOpE0cgUm99U8OxVTIPv19gCDuNzlKTZSntcQzLbo6LFXyNhJdV7XTWQ+5vA==
version "6.0.141"
resolved "https://registry.npmmirror.com/ai/-/ai-6.0.141.tgz#c476280ae69b13ad11f2671e49a24ac0412a0b54"
integrity sha512-+GomGQWaId3xN0wcugUW/H7xMMaFkID2PiS7K/Wugj45G3efv0BXhQ3psRZoQVoRbOpdNoUqcK/KTB+FR4h6qg==
dependencies:
"@ai-sdk/gateway" "3.0.82"
"@ai-sdk/gateway" "3.0.83"
"@ai-sdk/provider" "3.0.8"
"@ai-sdk/provider-utils" "4.0.21"
"@opentelemetry/api" "1.9.0"
@ -1303,13 +1303,13 @@ axios-retry@^4.5.0:
is-retry-allowed "^2.2.0"
axios@^1.13.2:
version "1.13.6"
resolved "https://registry.npmmirror.com/axios/-/axios-1.13.6.tgz#c3f92da917dc209a15dd29936d20d5089b6b6c98"
integrity sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==
version "1.14.0"
resolved "https://registry.npmmirror.com/axios/-/axios-1.14.0.tgz#7c29f4cf2ea91ef05018d5aa5399bf23ed3120eb"
integrity sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ==
dependencies:
follow-redirects "^1.15.11"
form-data "^4.0.5"
proxy-from-env "^1.1.0"
proxy-from-env "^2.1.0"
balanced-match@^1.0.0:
version "1.0.2"
@ -1396,17 +1396,17 @@ boolean@^3.0.1:
integrity sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==
brace-expansion@^1.1.7:
version "1.1.12"
resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.12.tgz#ab9b454466e5a8cc3a187beaad580412a9c5b843"
integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==
version "1.1.13"
resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.13.tgz#d37875c01dc9eff988dd49d112a57cb67b54efe6"
integrity sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==
dependencies:
balanced-match "^1.0.0"
concat-map "0.0.1"
brace-expansion@^2.0.1, brace-expansion@^2.0.2:
version "2.0.2"
resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7"
integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==
version "2.0.3"
resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-2.0.3.tgz#0493338bdd58e319b1039c67cf7ee439892c01d9"
integrity sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==
dependencies:
balanced-match "^1.0.0"
@ -1870,9 +1870,9 @@ cross-spawn@^7.0.1, cross-spawn@^7.0.6:
which "^2.0.1"
custom-electron-titlebar@^4.2.8:
version "4.4.0"
resolved "https://registry.npmmirror.com/custom-electron-titlebar/-/custom-electron-titlebar-4.4.0.tgz#0b6aff12db97babf237d052e0d2ea24623de4783"
integrity sha512-bvYxxDQiG/23eMPDAdhjjOXviEXnShHRo8VCDCouVfVDdG3Xs8SkXISYDZK3pyhINsrrJ21s9tSU7h59WDr5oA==
version "4.4.1"
resolved "https://registry.npmmirror.com/custom-electron-titlebar/-/custom-electron-titlebar-4.4.1.tgz#aea64f009697c9771cb2a67d2eb5ac8059696906"
integrity sha512-I+sOGBdslrGpuCWlhda8P0vtRAZK+W2NzjHLsxTiE2bNmhAIs9YLDe6iRBExwU1xVZt+J1hSXzUT67BlAuMWLA==
debug@2.6.9:
version "2.6.9"
@ -3212,9 +3212,9 @@ keyv@^4.0.0:
json-buffer "3.0.1"
knex@^3.1.0, knex@^3.2.5:
version "3.2.6"
resolved "https://registry.npmmirror.com/knex/-/knex-3.2.6.tgz#d6653ebf7961a83ae0014db1a7c56c75ce1ffa00"
integrity sha512-26I1Tvx0D9SOsFBF9jRWT/JdE3FPI52jAwYpXfREEtnoqgjGvAi8/ux8ktjaTC9IQcAVTCrREkOpa7lrjeAcow==
version "3.2.7"
resolved "https://registry.npmmirror.com/knex/-/knex-3.2.7.tgz#53bc16470217f12fef516a7a649794a405d3473d"
integrity sha512-VxdDE72x7Tc08E5yCu8HqYoeOm0HOjAraOtYiGSAUJTYkydwfSGBOpQqYHrzM5vjLNzw2JDL2vDH8m7DjIjtgA==
dependencies:
colorette "2.0.19"
commander "^10.0.0"
@ -4230,10 +4230,10 @@ proxy-addr@^2.0.7:
forwarded "0.2.0"
ipaddr.js "1.9.1"
proxy-from-env@^1.1.0:
version "1.1.0"
resolved "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
proxy-from-env@^2.1.0:
version "2.1.0"
resolved "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba"
integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==
pstree.remy@^1.1.8:
version "1.1.8"