zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '108' of https://github.com/HBAI-Ltd/Toonflow-app into 108

Browse Source
This commit is contained in:
小帅 2026-03-26 21:06:07 +08:00
commit 248dc724d4
9 changed files with 94 additions and 8359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8246
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

3
package.json
View File

@ -76,6 +76,7 @@
"@types/cors": "^2.8.19",
"@types/express": "^5.0.6",
"@types/express-ws": "^3.0.6",
"@types/graphlib": "^2.1.12",
"@types/jsonwebtoken": "^9.0.10",
"@types/license-checker": "^25.0.6",
"@types/lodash": "^4.17.24",
@ -89,4 +90,4 @@
"tsx": "^4.21.0",
"typescript": "^5.9.3"
}
}
}

4
src/agents/productionAgent/tools.ts
View File

@ -492,7 +492,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
resTool.systemMessage(`图片生成调度计划：共 ${levels.length} 层，${images.length} 张图片`);
// --- 准备公共数据 ---
const skill = await useSkill("universal_agent.md");
const projectData = await u.db("o_project").where("id", resTool.data.projectId).select("videoRatio").first();
const imageModel = resTool.data.imageModel;
@ -514,7 +513,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
]);
const imageCls = await u.Ai.Image(imageModel?.modelId).run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: [...assetsBase64, ...referenceBase64],
size: imageModel?.quality,
@ -595,7 +593,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
),
}),
execute: async ({ images }) => {
const skill = await useSkill("universal_agent.md");
console.log("[tools] generate_assets_images", images);
//先获取到前端资产数据
const flowData: FlowData = await new Promise((resolve) => socket.emit("getFlowData", { key: "assets" }, (res: any) => resolve(res)));
@ -623,7 +620,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
});
u.Ai.Image(imageModel?.modelId)
.run({
// systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: await getAssetsImageBase64(item.id ? [item.id] : []),
size: imageModel?.quality,

44
src/agents/scriptAgent/tools.ts
View File

@ -94,37 +94,37 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
description: "将剧本内容插入sqlite数据库，供后续业务使用",
inputSchema: z.object({
script: ScriptSchema,
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"),
// assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"),
}),
execute: async ({ assetsList, script }) => {
execute: async ({ script }) => {
console.log("%c Line:103 🍷 script", "background:#42b983", script);
console.log("[tools] insert_script_to_sqlite", assetsList);
// console.log("[tools] insert_script_to_sqlite", assetsList);
const [scriptId] = await u.db("o_script").insert({
name: script.name,
content: script.content,
projectId: resTool.data.projectId,
createTime: Date.now(),
});
if (assetsList && assetsList.length) {
const assetId = [];
for (const i of assetsList) {
if (i.id) {
assetId.push(i.id);
continue;
}
const [id] = await u.db("o_assets").insert({
name: i.name,
prompt: i.prompt,
type: i.type,
describe: i.desc,
projectId: resTool.data.projectId,
startTime: Date.now(),
});
assetId.push(id);
}
// if (assetsList && assetsList.length) {
// const assetId = [];
// for (const i of assetsList) {
// if (i.id) {
// assetId.push(i.id);
// continue;
// }
// const [id] = await u.db("o_assets").insert({
// name: i.name,
// prompt: i.prompt,
// type: i.type,
// describe: i.desc,
// projectId: resTool.data.projectId,
// startTime: Date.now(),
// });
// assetId.push(id);
// }
await u.db("o_scriptAssets").insert(assetId.map((i) => ({ scriptId, assetId: i })));
}
// await u.db("o_scriptAssets").insert(assetId.map((i) => ({ scriptId, assetId: i })));
// }
socket.emit("setPlanData", { key: "script", value: scriptId });
return true;
},

68
src/routes/production/editImage/generateFlowImage.ts
View File

@ -5,17 +5,6 @@ import { success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import axios from "axios";
const router = express.Router();
async function getImageBase64ForId(imageId: string | number) {
const imagePath = await u
.db("o_image")
.select("filePath")
.where({ id: Number(imageId) })
.first();
if (!imagePath || !imagePath.filePath) return ""; // 未找到图片路径
const url = await u.oss.getFileUrl(imagePath.filePath);
return await urlToBase64(url);
}
async function urlToBase64(imageUrl: string): Promise<string> {
const response = await axios.get(imageUrl, { responseType: "arraybuffer" });
@ -23,57 +12,11 @@ async function urlToBase64(imageUrl: string): Promise<string> {
const base64 = Buffer.from(response.data, "binary").toString("base64");
return `data:${contentType};base64,${base64}`;
}
// 将图片ID和指令转换为base64数组和替换后的指令
async function convertDirectiveAndImages(images: Record<string, string>, directive: string) {
// step1: 列出所有别名
const aliasList = Object.keys(images);
// step2: 在指令中提取所有 @别名出现
const aliasRegex = /@[\u4e00-\u9fa5\w]+/g;
const referencedAliases = directive.match(aliasRegex) || [];
// step3: 检查别名
for (const alias of referencedAliases) {
if (!(alias in images)) {
throw new Error(`您引用了不存在的图片：${alias}`);
}
}
// step4: 构建别名与顺序编号映射
const aliasToIndex: Record<string, number> = {};
aliasList.forEach((alias, i) => {
aliasToIndex[alias] = i + 1;
});
// step5: 替换指令中的别名为"图N"
let prompt = directive;
for (const [alias, idx] of Object.entries(aliasToIndex)) {
// 转义alias可能含特殊字符
const reg = new RegExp(alias.replace(/([.*+?^${}()|\[\]\/\\])/g, "\\$1"), "g");
prompt = prompt.replace(reg, `图${idx}`);
}
// step6: 依次获取图片 base64 内容（区分id或者本身就是base64）
const base64Images: string[] = [];
for (const imageVal of Object.values(images)) {
// 判断是否为base64串
const isBase64 = typeof imageVal === "string" && /^data:image\//.test(imageVal);
if (isBase64) {
base64Images.push(imageVal);
} else if (typeof imageVal === "number") {
const base64 = await getImageBase64ForId(imageVal);
base64Images.push(base64);
} else if (imageVal.includes("http")) {
const base64 = await urlToBase64(imageVal);
base64Images.push(base64);
}
}
return {
prompt,
images: base64Images,
};
}
export default router.post(
"/",
validateFields({
model: z.string(),
references: z.object().optional(),
references: z.array(z.string()).optional(),
quality: z.string(),
ratio: z.string(),
prompt: z.string(),
@ -81,12 +24,11 @@ export default router.post(
type: z.enum(["role", "scene", "storyboard", "clip", "tool"]),
}),
async (req, res) => {
const { model, references = {}, quality, ratio, prompt, projectId, type } = req.body;
const { prompt: userPrompt, images: base64Images } = await convertDirectiveAndImages(references, prompt);
console.log("%c Line:86 🥒 base64Images", "background:#42b983", base64Images.map((s) => s.slice(0, 4)));
const { model, references = [], quality, ratio, prompt, projectId, type } = req.body;
const imageClass = await u.Ai.Image(model).run({
prompt: userPrompt,
imageBase64: base64Images,
prompt: prompt,
imageBase64: references && references.length ? await Promise.all(references.map((url: string) => urlToBase64(url))) : [],
size: quality,
aspectRatio: ratio,
taskClass: "分镜生成",

2
src/routes/production/editImage/saveImageFlow.ts
View File

@ -62,7 +62,7 @@ export default router.post(
});
insertFlowId = storyboardId;
}
}
} else insertFlowId = id;
await u.db("o_imageFlow").insert({
flowData: JSON.stringify({ edges, nodes }),

34
src/routes/script/extractAssets.ts Normal file
View File

@ -0,0 +1,34 @@
import express from "express";
import u from "@/utils";
import { z } from "zod";
import { error, success } from "@/lib/responseFormat";
import compressing from "compressing";
import { validateFields } from "@/middleware/middleware";
import { useSkill } from "@/utils/agent/skillsTools";
const router = express.Router();
export default router.post(
"/",
validateFields({
scriptIds: z.array(z.number()),
}),
async (req, res) => {
const { scriptIds } = req.body;
if (!scriptIds.length) return res.status(400).send(error("请先选择剧本"));
const scripts = await u.db("o_script").whereIn("id", scriptIds);
const intansce = u.Ai.Text("universalAgent");
const skill = await useSkill("universal_agent.md");
const resData = await intansce.invoke({
system: skill.prompt,
messages: [
{
role: "user",
content: "请根据以下小说章节生成事件摘要：\n",
},
],
tools: skill.tools,
});
},
);

3
src/utils/ai.ts
View File

@ -118,6 +118,7 @@ interface VideoConfig {
projectId: number; // 项目ID
prompt: string; //视频提示词
imageBase64: string[]; //输入的图片提示词
aspectRatio: `${number}:${number}`; // 长宽比
mode: string; //模式
duration: number; // 视频时长，单位秒
resolution: string; // 视频分辨率
@ -136,6 +137,8 @@ class AiVideo {
async run(input: VideoConfig) {
return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => {
const fn = await getVendorTemplateFn("videoRequest", modelName);
console.log("%c Line:142 🎂 input", "background:#42b983", input);
this.result = await fn(input);
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
return this;

49
yarn.lock
View File

@ -32,10 +32,10 @@
"@hono/node-server" "^1.13.7"
hono "^4.6.14"
"@ai-sdk/gateway@3.0.79":
version "3.0.79"
resolved "https://registry.npmmirror.com/@ai-sdk/gateway/-/gateway-3.0.79.tgz#2a800daba54c9a0b332984fb97affa9950e94e1c"
integrity sha512-Wk2QJpqd0em5YcR49uoMCy9msyANAYgjXdlRcqqRt2fz4rNLnMMrKOlLwAXoFzR1ElR3bj4e/k6hscRfjpzSGA==
"@ai-sdk/gateway@3.0.80":
version "3.0.80"
resolved "https://registry.npmmirror.com/@ai-sdk/gateway/-/gateway-3.0.80.tgz#fbd43e14395f013dea49f427020b882be8840af2"
integrity sha512-uM7kpZB5l977lW7+2X1+klBUxIZQ78+1a9jHlaHFEzcOcmmslTl3sdP0QqfuuBcO0YBM2gwOiqVdp8i4TRQYcw==
dependencies:
"@ai-sdk/provider" "3.0.8"
"@ai-sdk/provider-utils" "4.0.21"
@ -812,6 +812,11 @@
dependencies:
"@types/node" "*"
"@types/graphlib@^2.1.12":
version "2.1.12"
resolved "https://registry.npmmirror.com/@types/graphlib/-/graphlib-2.1.12.tgz#089d1218bafe2277c98b12597abb24c3b73965aa"
integrity sha512-abRfQWMphT2qlXwppQa+CTCtUz/GqxBeozQcMjnOFD/WOKD6sRgxkfG8vq1yagV03447BBzCYhuJ0wiNb+/r+Q==
"@types/http-cache-semantics@*":
version "4.2.0"
resolved "https://registry.npmmirror.com/@types/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz#f6a7788f438cbfde15f29acad46512b4c01913b3"
@ -991,11 +996,11 @@ agent-base@^7.1.0, agent-base@^7.1.2:
integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==
ai@^6.0.67:
version "6.0.137"
resolved "https://registry.npmmirror.com/ai/-/ai-6.0.137.tgz#fd57e75092da5c211ad8b8fb0a79a76e4843af5f"
integrity sha512-9e/mNMTmXmMkmldEJPIumy9FFBuUWHUyj2yF8EglC3VVOhVVBwlnpeHYSFKdNc13Jj6Hso3EJcZioMaofgCs4A==
version "6.0.138"
resolved "https://registry.npmmirror.com/ai/-/ai-6.0.138.tgz#96bfa9ebdd9748bc72862d14d6c4fc8e28070f10"
integrity sha512-49OfPe0f5uxJ6jUdA5BBXjIinP6+ZdYfAtpF2aEH64GA5wPcxH2rf/TBUQQ0bbamBz/D+TLMV18xilZqOC+zaA==
dependencies:
"@ai-sdk/gateway" "3.0.79"
"@ai-sdk/gateway" "3.0.80"
"@ai-sdk/provider" "3.0.8"
"@ai-sdk/provider-utils" "4.0.21"
"@opentelemetry/api" "1.9.0"
@ -1272,9 +1277,9 @@ brace-expansion@^2.0.1, brace-expansion@^2.0.2:
balanced-match "^1.0.0"
brace-expansion@^5.0.2:
version "5.0.4"
resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-5.0.4.tgz#614daaecd0a688f660bbbc909a8748c3d80d4336"
integrity sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==
version "5.0.5"
resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-5.0.5.tgz#dcc3a37116b79f3e1b46db994ced5d570e930fdb"
integrity sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==
dependencies:
balanced-match "^4.0.2"
@ -1889,9 +1894,9 @@ electron-publish@26.8.1:
mime "^2.5.2"
electron@^40.0.0:
version "40.8.3"
resolved "https://registry.npmmirror.com/electron/-/electron-40.8.3.tgz#f09a6904e79c72e1894d5a45e560fede50219ca1"
integrity sha512-MH6LK4xM6VVmmtz0nRE0Fe8l2jTKSYTvH1t0ZfbNLw3o6dlBCVTRqQha6uL8ZQVoMy74JyLguGwK7dU7rCKIhw==
version "40.8.4"
resolved "https://registry.npmmirror.com/electron/-/electron-40.8.4.tgz#91c0591e2ee7377e0a5c80e4fc4680cc26d84c33"
integrity sha512-7AoSakFr+g2CTukHDS79cqNiaWPoD8bQ4kIahwUUVv0O5fy4BfZawVCxOFLc61POq8xDvqMSDKPfeFXK/Coc5g==
dependencies:
"@electron/get" "^2.0.0"
"@types/node" "^24.9.0"
@ -2893,9 +2898,9 @@ keyv@^4.0.0:
json-buffer "3.0.1"
knex@^3.1.0, knex@^3.2.5:
version "3.2.5"
resolved "https://registry.npmmirror.com/knex/-/knex-3.2.5.tgz#af4ed3d207581829460b805f49da26882f0f11fc"
integrity sha512-eY21Uwq0l0bFW6/+3ARO+NPjlUmcKsd72/Od+iHVFRhY6xwRVKn/nfeBEErtSnvJRVj0d0Jc20LzNnSgRQwMlQ==
version "3.2.6"
resolved "https://registry.npmmirror.com/knex/-/knex-3.2.6.tgz#d6653ebf7961a83ae0014db1a7c56c75ce1ffa00"
integrity sha512-26I1Tvx0D9SOsFBF9jRWT/JdE3FPI52jAwYpXfREEtnoqgjGvAi8/ux8ktjaTC9IQcAVTCrREkOpa7lrjeAcow==
dependencies:
colorette "2.0.19"
commander "^10.0.0"
@ -3166,11 +3171,11 @@ minipass-fetch@^4.0.0:
encoding "^0.1.13"
minipass-flush@^1.0.5:
version "1.0.5"
resolved "https://registry.npmmirror.com/minipass-flush/-/minipass-flush-1.0.5.tgz#82e7135d7e89a50ffe64610a787953c4c4cbb373"
integrity sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==
version "1.0.6"
resolved "https://registry.npmmirror.com/minipass-flush/-/minipass-flush-1.0.6.tgz#a3f43e5d588f3bd5f1585cc68504a5f05ebd77a8"
integrity sha512-7Uf5gMJZ2kTkFisE3toGxT991s+cg+vMh42nbZGM2bNxfYVpkpqRudf1QrcOy72a3PwcL4JYqL+4NY7t0Hdd0A==
dependencies:
minipass "^3.0.0"
minipass "^7.1.3"
minipass-pipeline@^1.2.4:
version "1.2.4"
@ -3193,7 +3198,7 @@ minipass@^3.0.0:
dependencies:
yallist "^4.0.0"
"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.0.2, minipass@^7.0.3, minipass@^7.0.4, minipass@^7.1.2:
"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.0.2, minipass@^7.0.3, minipass@^7.0.4, minipass@^7.1.2, minipass@^7.1.3:
version "7.1.3"
resolved "https://registry.npmmirror.com/minipass/-/minipass-7.1.3.tgz#79389b4eb1bb2d003a9bba87d492f2bd37bdc65b"
integrity sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==