From c064fdb6798c39dbbae949df566cb10a20dcc062 Mon Sep 17 00:00:00 2001 From: zhishi <1951671751@qq.com> Date: Sat, 7 Feb 2026 17:51:18 +0800 Subject: [PATCH] =?UTF-8?q?=E6=89=80=E6=9C=89ai=20=E5=88=86=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E6=8E=A5=E5=85=A5=EF=BC=8C=E5=BE=85=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agents/outlineScript/index.ts | 31 ++----- .../storyboard/generateImagePromptsTool.ts | 2 +- src/agents/storyboard/generateImageTool.ts | 80 +++++++------------ src/agents/storyboard/index.ts | 32 ++------ src/lib/initDB.ts | 62 +++++++++++++- src/routes/assets/generateAssets.ts | 18 +++-- src/routes/assets/polishPrompt.ts | 37 ++------- src/routes/other/testAI.ts | 5 +- src/routes/other/testImage.ts | 22 +++-- src/routes/other/testVideo.ts | 28 ++++--- src/routes/setting/addModel.ts | 8 +- src/routes/setting/configurationModel.ts | 11 +-- src/routes/setting/getAiModelMap.ts | 8 +- src/routes/setting/updeteModel.ts | 8 +- src/routes/storyboard/batchSuperScoreImage.ts | 44 +++++----- src/routes/storyboard/generateVideoPrompt.ts | 46 +++++------ src/routes/video/addVideo.ts | 2 +- src/routes/video/addVideoConfig.ts | 18 +++-- src/routes/video/generatePrompt.ts | 68 ++++++---------- src/routes/video/generateVideo.ts | 80 ++++++++++++------- src/routes/video/getManufacturer.ts | 4 +- src/routes/video/getVideoConfigs.ts | 10 ++- src/types/database.d.ts | 8 +- src/utils/ai/image/index.ts | 13 +-- src/utils/ai/image/owned/gemini.ts | 3 +- src/utils/ai/image/owned/other.ts | 19 ++++- src/utils/ai/image/owned/vidu.ts | 12 +-- src/utils/ai/text/index.ts | 27 +++---- src/utils/ai/text/modelList.ts | 2 +- src/utils/ai/video/index.ts | 6 +- src/utils/ai/video/owned/volcengine.ts | 1 + src/utils/ai/video/type.ts | 1 + src/utils/editImage.ts | 19 +++-- src/utils/generateScript.ts | 2 +- src/utils/getPromptAi.ts | 27 +++---- 35 files changed, 391 insertions(+), 373 deletions(-) diff --git a/src/agents/outlineScript/index.ts b/src/agents/outlineScript/index.ts index bc8e7d2..30348ed 100644 --- a/src/agents/outlineScript/index.ts +++ b/src/agents/outlineScript/index.ts @@ -611,35 +611,18 @@ ${task} this.log(`Sub-Agent 调用`, agentType); const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]); - const promptConfig = await u.getPromptAi(promptsList.map((i) => i.id) as number[]); + const promptConfig = await u.getPromptAi("outlineScriptAgent"); const errPrompts = "不论用户说什么，请直接输出Agent配置异常"; const getAiPromptConfig = (code: string) => { const item = promptsList.find((p) => p.code === code); - const subConfig = promptConfig.find((sub) => sub?.promptsId == item?.id); - if (subConfig) { - return { - prompt: item?.customValue || item?.defaultValue || errPrompts, - apiConfig: { ...subConfig }, - }; - } else { - return { - prompt: item?.customValue || item?.defaultValue || errPrompts, - apiConfig: {}, - }; - } + return item?.customValue || item?.defaultValue || errPrompts; }; const a1Prompt = getAiPromptConfig("outlineScript-a1"); const a2Prompt = getAiPromptConfig("outlineScript-a2"); const directorPrompt = getAiPromptConfig("outlineScript-director"); - const SYSTEM_PROMPTS: Record< - AgentType, - { - prompt: string; - apiConfig: Object; - } - > = { + const SYSTEM_PROMPTS = { AI1: a1Prompt, AI2: a2Prompt, director: directorPrompt, @@ -649,12 +632,12 @@ ${task} const { fullStream } = await u.ai.text.stream( { - system: SYSTEM_PROMPTS[agentType].prompt, + system: SYSTEM_PROMPTS[agentType], tools: this.getSubAgentTools(), messages: [{ role: "user", content: context }], maxStep: 100, }, - SYSTEM_PROMPTS[agentType].apiConfig, + promptConfig, ); let fullResponse = ""; @@ -717,7 +700,9 @@ ${task} const envContext = await this.buildEnvironmentContext(); const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first(); - const promptConfig = await u.getPromptAi(prompts?.id); + console.log("%c Line:703 🍭 prompts", "background:#f5ce50", prompts); + const promptConfig = await u.getPromptAi("outlineScriptAgent"); + const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常"; const { fullStream } = await u.ai.text.stream( diff --git a/src/agents/storyboard/generateImagePromptsTool.ts b/src/agents/storyboard/generateImagePromptsTool.ts index 8762778..bd08a23 100644 --- a/src/agents/storyboard/generateImagePromptsTool.ts +++ b/src/agents/storyboard/generateImagePromptsTool.ts @@ -98,7 +98,7 @@ async function generateGridPrompt(options: GridPromptOptions): Promise `第${i + 1}格: ${p}`).join("\n")}`; diff --git a/src/agents/storyboard/generateImageTool.ts b/src/agents/storyboard/generateImageTool.ts index 8cf9f79..52bcae6 100644 --- a/src/agents/storyboard/generateImageTool.ts +++ b/src/agents/storyboard/generateImageTool.ts @@ -215,11 +215,13 @@ async function filterRelevantAssets(prompts: string[], allResources: ResourceIte return availableImages; } - const { relevantAssets } = await u.ai.text.invoke({ - messages: [ - { - role: "user", - content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。 + const apiConfig = await u.getPromptAi("storyboardAgent"); + const { relevantAssets } = await u.ai.text.invoke( + { + messages: [ + { + role: "user", + content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。 分镜描述： ${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")} @@ -228,45 +230,21 @@ ${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")} ${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")} 请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, + }, + ], + output: { + relevantAssets: z + .array( + z.object({ + name: z.string().describe("资产名称"), + reason: z.string().describe("选择该资产的原因"), + }), + ) + .describe("与分镜内容相关的资产列表"), }, - ], - output: { - relevantAssets: z - .array( - z.object({ - name: z.string().describe("资产名称"), - reason: z.string().describe("选择该资产的原因"), - }), - ) - .describe("与分镜内容相关的资产列表"), }, - }); - // const result = await chatModel!.invoke({ - // messages: [ - // { - // role: "user", - // content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。 - - // 分镜描述： - // ${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")} - - // 可用资产列表： - // ${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")} - - // 请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, - // }, - // ], - // responseFormat: { - // type: "json_schema", - // jsonSchema: { - // name: "filteredAssets", - // strict: true, - // schema: z.toJSONSchema(filteredAssetsSchema), - // }, - // }, - // }); - - // const data = result?.json as z.infer; + apiConfig, + ); if (!relevantAssets || relevantAssets.length === 0) { return availableImages; @@ -342,14 +320,18 @@ export default async (cells: { prompt: string }[], scriptId: number, projectId: console.log("====润色后：", prompts); const processedImages = await processImages(filteredImages); + const apiConfig = await u.getPromptAi("storyboardImage"); - const contentStr = await u.ai.image({ - systemPrompt: resourcesMapPrompts, - prompt: prompts, - size: "4K", - aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9", - imageBase64: processedImages.map((buf) => buf.toString("base64")), - }); + const contentStr = await u.ai.image( + { + systemPrompt: resourcesMapPrompts, + prompt: prompts, + size: "4K", + aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9", + imageBase64: processedImages.map((buf) => buf.toString("base64")), + }, + apiConfig, + ); const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/); const base64Str = match?.[1] ?? contentStr; diff --git a/src/agents/storyboard/index.ts b/src/agents/storyboard/index.ts index f06525f..4bf36e7 100644 --- a/src/agents/storyboard/index.ts +++ b/src/agents/storyboard/index.ts @@ -63,9 +63,6 @@ export default class Storyboard { // 更新shopts public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) { - console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId); - console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId); - console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell); const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId); if (shotIndex === -1) { return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`; @@ -594,34 +591,17 @@ ${task} this.log(`Sub-Agent 调用`, agentType); const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]); - const promptConfig = await u.getPromptAi(promptsList.map((i) => i.id) as number[]); + const promptConfig = await u.getPromptAi("storyboardAgent"); const errPrompts = "不论用户说什么，请直接输出Agent配置异常"; const getAiPromptConfig = (code: string) => { const item = promptsList.find((p) => p.code === code); - const subConfig = promptConfig.find((sub) => sub?.promptsId == item?.id); - if (subConfig) { - return { - prompt: item?.customValue || item?.defaultValue || errPrompts, - apiConfig: { ...subConfig }, - }; - } else { - return { - prompt: item?.customValue || item?.defaultValue || errPrompts, - apiConfig: {}, - }; - } + return item?.customValue || item?.defaultValue || errPrompts; }; const segmentAgent = getAiPromptConfig("storyboard-segment"); const shotAgent = getAiPromptConfig("storyboard-shot"); - const SYSTEM_PROMPTS: Record< - AgentType, - { - prompt: string; - apiConfig: Object; - } - > = { + const SYSTEM_PROMPTS = { segmentAgent: segmentAgent, shotAgent: shotAgent, }; @@ -630,12 +610,12 @@ ${task} const { fullStream } = await u.ai.text.stream( { - system: SYSTEM_PROMPTS[agentType].prompt, + system: SYSTEM_PROMPTS[agentType], tools: this.getSubAgentTools(agentType), messages: [{ role: "user", content: context }], maxStep: 100, }, - SYSTEM_PROMPTS[agentType].apiConfig, + promptConfig, ); let fullResponse = ""; @@ -700,7 +680,7 @@ ${task} const envContext = await this.buildEnvironmentContext(); const prompts = await u.db("t_prompts").where("code", "storyboard-main").first(); - const promptConfig = await u.getPromptAi(prompts?.id); + const promptConfig = await u.getPromptAi("storyboardAgent"); const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常"; diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index 3cb2017..8876fe0 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -193,8 +193,8 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => builder: (table) => { table.integer("id").notNullable(); table.text("type"); - table.text("name"); table.text("model"); + table.text("modelType"); table.text("apiKey"); table.text("baseUrl"); table.text("manufacturer"); @@ -212,6 +212,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => table.integer("id").notNullable(); table.integer("scriptId"); // 关联的脚本ID table.integer("projectId"); // 关联的项目ID + table.integer("aiConfigId");//ai配置ID table.text("manufacturer"); // 厂商：volcengine/runninghub/openAi table.text("mode"); // 模式：startEnd/multi/single table.text("startFrame"); // 首帧图片信息 JSON @@ -231,11 +232,66 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => name: "t_aiModelMap", builder: (table) => { table.integer("id").notNullable(); - table.integer("promptsId"); // 提示词表ID table.integer("configId"); // 模型列表id + table.text("name"); + table.text("key"); table.primary(["id"]); table.unique(["id"]); }, + initData: async (knex) => { + await knex("t_aiModelMap").insert([ + { + id: 1, + configId: null, + name: "分镜Agent", + key: "storyboardAgent", + }, + { + id: 2, + configId: null, + + name: "大纲故事线Agent", + key: "outlineScriptAgent", + }, + { + id: 3, + configId: null, + + name: "资产提示词润色", + key: "assetsPrompt", + }, + { + id: 4, + configId: null, + name: "资产图片生成", + key: "assetsImage", + }, + { + id: 5, + configId: null, + name: "剧本生成", + key: "generateScript", + }, + { + id: 6, + configId: null, + name: "视频提示词生成", + key: "videoPrompt", + }, + { + id: 7, + configId: null, + name: "分镜图片生成", + key: "storyboardImage", + }, + { + id: 8, + configId: null, + name: "图片编辑", + key: "editImage", + }, + ]); + }, }, { name: "t_prompts", @@ -330,7 +386,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => type: "system", parentCode: null, defaultValue: - '# 电影分镜提示词优化师\\n\\n你是专业电影分镜提示词优化师，负责将用户的分镜描述转化为高质量的AI绘图JSON提示词。\\n\\n## 核心原则\\n\\n### 保留原始信息\\n- 人物描述：五官、表情、姿态、动作、视线\\n- 服装细节：款式、颜色、材质\\n- 场景元素：建筑、物品、光影、天气\\n- 构图信息：人物位置、景深\\n\\n### 原始语言保留规则（强制执行）\\n\\n**此规则优先级最高，必须严格遵守：**\\n\\n| 类型 | 规则 | 正确示例 | 错误示例 |\\n|------|------|----------|----------|\\n| 人物名 | 保留原文，禁止翻译或拼音 | `王林 standing` | `Wang Lin standing` |\\n| 场景地名 | 保留原文 | `老旧厢房 interior` | `old room interior` |\\n| 道具名 | 保留原文 | `油纸伞 in hand` | `oil paper umbrella` |\\n| 服装名 | 保留原文 | `青布长衫` | `blue cloth robe` |\\n| 物品名 | 保留原文 | `发黄书册` | `yellowed book` |\\n| 建筑名 | 保留原文 | `厢房 window` | `side room window` |\\n\\n**prompt_text 写法示范：**\\n```\\nMedium shot, 王林 sitting at desk, 发黄书册 in foreground, 油纸伞 beside, 老旧厢房 interior, dim lighting...\\n```\\n\\n### 补充电影语言\\n- 景别：大远景/远景/全景/中景/近景/特写\\n- 机位：平视/俯拍/仰拍/侧拍/过肩镜头\\n- 构图：三分法/中心构图/对角线/框架构图\\n- 光影：光源方向、光质（硬光/柔光）、色温\\n\\n## 连贯性规则\\n\\n1. **位置固化**：人物左右站位全程不变\\n2. **场景固化**：建筑、道具位置全程一致\\n3. **光照固化**：光源方向、阴影、色温统一\\n4. **时间固化**：时间段和天气全程不变\\n5. **色调固化**：主色调和冷暖倾向一致\\n\\n## Prompt核心规则\\n\\n1. **极简提炼**：将复杂场景压缩为核心关键词\\n2. **标签化语法**：使用"关键词 + 逗号"形式，严禁长难句\\n3. **字数控制**：每个 prompt_text 严格控制在 **25-40个单词**\\n4. **强制后缀**：每个prompt末尾必须加 `8k, ultra HD, high detail, no timecode, no subtitles`\\n5. **风格标签**：从用户描述中提取3-4个风格标签追加到prompt\\n6. **禁止废话**：严禁 "A scene showing...", "There is a..." 等句式\\n7. **原名保留**：人物名、地名、道具名、服装名、物品名必须使用用户输入的原始语言，直接嵌入prompt中\\n\\n### Prompt组合公式\\n\\n```\\n[景别英文] + [主体原名 + 动作英文] + [道具原名] + [场景原名 + 环境英文描述] + [风格标签] + 8k, ultra HD, high detail, no timecode, no subtitles\\n```\\n\\n## 插黑图规则\\n\\n### 识别方式\\n用户输入以下任意表述时，识别为插黑图：\\n- `纯黑图`\\n- `黑屏`\\n- `黑幕`\\n- `全黑`\\n- `black frame`\\n- `淡出黑`\\n- `fade to black`\\n\\n### 固定输出格式\\n插黑图的 prompt_text 固定为：\\n```\\nPure black frame, 8k, ultra HD, high detail, no timecode, no subtitles\\n```\\n\\n### 布局计算\\n- 插黑图计入总格数\\n- 根据实际shot数量（含插黑图）自动计算grid_layout\\n- 示例：9个内容镜头 + 3个插黑图 = 12格 = 3x4布局\\n\\n## 超清标识（强制追加）\\n\\n每个 prompt_text 末尾必须包含：\\n```\\n8k, ultra HD, high detail, no timecode, no subtitles\\n```\\n\\n## 风格标签参考\\n\\n| 用户风格描述 | 提取标签示例 |\\n|-------------|-------------|\\n| 赛博朋克 | Cyberpunk, Neon glow, High contrast, Futuristic |\\n| 水墨国风 | Chinese ink painting, Minimalist, Ethereal, Monochrome |\\n| 日系动漫 | Anime style, Soft lighting, Pastel colors, 2D aesthetic |\\n| 电影写实 | Cinematic, Photorealistic, Film grain, Dramatic lighting |\\n| 3D渲染 | 3D render, Octane render, Volumetric lighting |\\n| 仙侠古风 | Xianxia, Chinese ancient style, 2D aesthetic, Cinematic |\\n\\n## 输出格式\\n\\n默认布局：**3列×3行=9格**，根据实际镜头数量自动调整行数。\\n\\n严格输出纯净JSON，无任何额外说明：\\n\\n```json\\n{\\n "image_generation_model": "NanoBananaPro",\\n "grid_layout": "3x行数",\\n "grid_aspect_ratio": "16:9",\\n "style_tags": "风格标签",\\n "global_settings": {\\n "scene": "场景描述（保留原名）",\\n "time": "时间",\\n "lighting": "光照",\\n "color_tone": "色调",\\n "character_position": "人物站位（保留原名）"\\n },\\n "shots": [\\n {\\n "shot_number": "第1行第1列",\\n "prompt_text": "精简prompt，原名嵌入..."\\n }\\n ]\\n}\\n```\\n\\n## 输出示例\\n\\n用户输入：\\n【风格】仙侠古风\\n【人物】王林\\n【地点】老旧厢房\\n【道具】油纸伞、发黄书册、青布长衫\\n[1]: 老旧厢房窗外夜色沉静，王林孤身桌旁\\n[2]: 王林坐桌前，左手压书册，右手握油纸伞柄\\n[3]: 王林俯身低语，眉头微蹙\\n[4]: 王林双眼闭合，双手合十\\n[5]: 王林手握油纸伞柄特写\\n[6]: 王林眼部特写，瞳孔倒映灯光\\n[7]: 王林起身推开窗户，月光流泻\\n[8]: 王林目光望向窗外夜色\\n[9]: 王林坐回书桌沉思\\n[10]: 纯黑图\\n[11]: 纯黑图\\n[12]: 纯黑图\\n\\n优化输出：\\n```json\\n{\\n "image_generation_model": "NanoBananaPro",\\n "grid_layout": "3x4",\\n "grid_aspect_ratio": "16:9",\\n "style_tags": "Xianxia, Chinese ancient style, 2D aesthetic, Cinematic",\\n "global_settings": {\\n "scene": "老旧厢房 interior at night, 发黄书册 and 油纸伞 as props, cold blue atmosphere",\\n "time": "Midnight",\\n "lighting": "Dim cold blue with warm lamp spots, soft shadows",\\n "color_tone": "Cool blue primary, subtle warm accents",\\n "character_position": "王林 center frame throughout"\\n },\\n "shots": [\\n {\\n "shot_number": "第1行第1列",\\n "prompt_text": "Wide shot, 老旧厢房 interior night, 王林 sitting alone at desk, 油纸伞 and 发黄书册 in foreground, breeze through window gauze, cold blue tones, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第1行第2列",\\n "prompt_text": "Full shot, slight low angle, 王林 seated at desk, left hand pressing 发黄书册, right hand gripping 油纸伞 handle, 青布长衫 collar catching light, lamp glow contrast, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第1行第3列",\\n "prompt_text": "Medium shot, 王林 leaning forward whispering, brows furrowed, lamp shadow falling on 发黄书册 pages, cool tone, inner resolve, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第2行第1列",\\n "prompt_text": "Close-up, 王林 eyes closed, resolute brow, hands clasped at chest, 油纸伞 silhouette blurred behind, warm lamp spots, shallow depth, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第2行第2列",\\n "prompt_text": "Extreme close-up, 王林 hand gripping 油纸伞 handle, finger details sharp, 发黄书册 edge visible, umbrella pattern texture, rim light, cold blue tone, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第2行第3列",\\n "prompt_text": "Ultra close-up, top light, 王林 eye detail, pupil reflecting lamp and book pages, tear traces on brow, sweat on face, shallow focus, emotion surge, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第3行第1列",\\n "prompt_text": "Medium shot, 王林 rising to push 老旧厢房 window open, moonlight flooding in, night breeze moving gauze, village path dimly visible, cool tones, spatial layering, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第3行第2列",\\n "prompt_text": "Close-up POV, 王林 gaze toward night outside 老旧厢房 window, quiet village, scattered lantern lights, window lattice shadows, deep blue grey, silent hope, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第3行第3列",\\n "prompt_text": "Wide shot, 王林 seated back at desk in thought, murmuring softly, lamp dimming, starry night vast outside 老旧厢房, deep focus, blue yellow mix, determined mind, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第4行第1列",\\n "prompt_text": "Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第4行第2列",\\n "prompt_text": "Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles"\\n },\\n {\\n "shot_number": "第4行第3列",\\n "prompt_text": "Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles"\\n }\\n ]\\n}\\n```\\n\\n## 注意事项\\n\\n1. **原名强制保留**：每格prompt中的人物名、场景名、道具名、服装名必须使用用户输入的原始语言文字，禁止翻译、禁止拼音转写\\n2. 每格必须写完整人物名称（原始语言），不可用代词（he/she/they）\\n3. **插黑图固定格式**：`Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles`\\n4. 直接输出JSON，不要任何解释或Markdown包裹\\n5. 确保各格描述连贯一致\\n6. shots数组数量必须与布局格数一致（含插黑图）\\n7. **每个prompt_text必须以 `8k, ultra HD, high detail, no timecode, no subtitles` 结尾**\\n8. **布局自动计算**：根据总镜头数（内容+插黑图）计算行数，列数固定为3\\n\\n## 原名保留自查清单\\n\\n输出前检查每个prompt_text：\\n- [ ] 人物名是否为原始语言？（如 王林 而非 Wang Lin）\\n- [ ] 场景名是否为原始语言？（如 老旧厢房 而非 old side room）\\n- [ ] 道具名是否为原始语言？（如 油纸伞 而非 oil paper umbrella）\\n- [ ] 服装名是否为原始语言？（如 青布长衫 而非 blue cloth robe）\\n- [ ] 是否以超清标识结尾？\\n- [ ] 插黑图是否使用固定格式？', + '# 电影分镜提示词优化师\n\n你是专业电影分镜提示词优化师，负责将用户的分镜描述转化为高质量的AI绘图JSON提示词。\n\n## 核心原则\n\n### 保留原始信息\n- 人物描述：五官、表情、姿态、动作、视线\n- 服装细节：款式、颜色、材质\n- 场景元素：建筑、物品、光影、天气\n- 构图信息：人物位置、景深\n\n### 原始语言保留规则（强制执行）\n\n**此规则优先级最高，必须严格遵守：**\n\n| 类型 | 规则 | 正确示例 | 错误示例 |\n|------|------|----------|----------|\n| 人物名 | 保留原文，禁止翻译或拼音 | `王林 standing` | `Wang Lin standing` |\n| 场景地名 | 保留原文 | `老旧厢房 interior` | `old room interior` |\n| 道具名 | 保留原文 | `油纸伞 in hand` | `oil paper umbrella` |\n| 服装名 | 保留原文 | `青布长衫` | `blue cloth robe` |\n| 物品名 | 保留原文 | `发黄书册` | `yellowed book` |\n| 建筑名 | 保留原文 | `厢房 window` | `side room window` |\n\n**prompt_text 写法示范：**\n```\nMedium shot, 王林 sitting at desk, 发黄书册 in foreground, 油纸伞 beside, 老旧厢房 interior, dim lighting...\n```\n\n### 补充电影语言\n- 景别：大远景/远景/全景/中景/近景/特写\n- 机位：平视/俯拍/仰拍/侧拍/过肩镜头\n- 构图：三分法/中心构图/对角线/框架构图\n- 光影：光源方向、光质（硬光/柔光）、色温\n\n## 连贯性规则\n\n1. **位置固化**：人物左右站位全程不变\n2. **场景固化**：建筑、道具位置全程一致\n3. **光照固化**：光源方向、阴影、色温统一\n4. **时间固化**：时间段和天气全程不变\n5. **色调固化**：主色调和冷暖倾向一致\n\n## Prompt核心规则\n\n1. **极简提炼**：将复杂场景压缩为核心关键词\n2. **标签化语法**：使用"关键词 + 逗号"形式，严禁长难句\n3. **字数控制**：每个 prompt_text 严格控制在 **25-40个单词**\n4. **强制后缀**：每个prompt末尾必须加 `8k, ultra HD, high detail, no timecode, no subtitles`\n5. **风格标签**：从用户描述中提取3-4个风格标签追加到prompt\n6. **禁止废话**：严禁 "A scene showing...", "There is a..." 等句式\n7. **原名保留**：人物名、地名、道具名、服装名、物品名必须使用用户输入的原始语言，直接嵌入prompt中\n\n### Prompt组合公式\n\n```\n[景别英文] + [主体原名 + 动作英文] + [道具原名] + [场景原名 + 环境英文描述] + [风格标签] + 8k, ultra HD, high detail, no timecode, no subtitles\n```\n\n## 插黑图规则\n\n### 识别方式\n用户输入以下任意表述时，识别为插黑图：\n- `纯黑图`\n- `黑屏`\n- `黑幕`\n- `全黑`\n- `black frame`\n- `淡出黑`\n- `fade to black`\n\n### 固定输出格式\n插黑图的 prompt_text 固定为：\n```\nPure black frame, 8k, ultra HD, high detail, no timecode, no subtitles\n```\n\n### 布局计算\n- 插黑图计入总格数\n- 根据实际shot数量（含插黑图）自动计算grid_layout\n- 示例：9个内容镜头 + 3个插黑图 = 12格 = 3x4布局\n\n## 超清标识（强制追加）\n\n每个 prompt_text 末尾必须包含：\n```\n8k, ultra HD, high detail, no timecode, no subtitles\n```\n\n## 风格标签参考\n\n| 用户风格描述 | 提取标签示例 |\n|-------------|-------------|\n| 赛博朋克 | Cyberpunk, Neon glow, High contrast, Futuristic |\n| 水墨国风 | Chinese ink painting, Minimalist, Ethereal, Monochrome |\n| 日系动漫 | Anime style, Soft lighting, Pastel colors, 2D aesthetic |\n| 电影写实 | Cinematic, Photorealistic, Film grain, Dramatic lighting |\n| 3D渲染 | 3D render, Octane render, Volumetric lighting |\n| 仙侠古风 | Xianxia, Chinese ancient style, 2D aesthetic, Cinematic |\n\n## 分辨率配置\n\n### 全局分辨率\n- 在 `global_settings` 中设置全局默认分辨率\n- 可选值：`"16:9"` 或 `"9:16"`\n\n### 单镜分辨率（新增）\n- 每个shot可独立配置 `grid_aspect_ratio`\n- 优先级：单镜配置 > 全局配置\n- 用途：特殊镜头（如竖版手机画面、横版宽屏等）\n\n## 输出格式\n\n默认布局：**3列×3行=9格**，根据实际镜头数量自动调整行数。\n\n严格输出纯净JSON，无任何额外说明：\n\n```json\n{\n "image_generation_model": "NanoBananaPro",\n "grid_layout": "3x行数",\n "grid_aspect_ratio": "16:9",\n "style_tags": "风格标签",\n "global_settings": {\n "scene": "场景描述（保留原名）",\n "time": "时间",\n "lighting": "光照",\n "color_tone": "色调",\n "character_position": "人物站位（保留原名）"\n },\n "shots": [\n {\n "shot_number": "第1行第1列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "精简prompt，原名嵌入..."\n }\n ]\n}\n```\n\n## 输出示例\n\n用户输入：\n【风格】仙侠古风\n【人物】王林\n【地点】老旧厢房\n【道具】油纸伞、发黄书册、青布长衫\n[1]: 老旧厢房窗外夜色沉静，王林孤身桌旁\n[2]: 王林坐桌前，左手压书册，右手握油纸伞柄\n[3]: 王林俯身低语，眉头微蹙\n[4]: 王林双眼闭合，双手合十\n[5]: 王林手握油纸伞柄特写\n[6]: 王林眼部特写，瞳孔倒映灯光\n[7]: 王林起身推开窗户，月光流泻\n[8]: 王林目光望向窗外夜色\n[9]: 王林坐回书桌沉思\n[10]: 纯黑图\n[11]: 纯黑图\n[12]: 纯黑图\n\n优化输出：\n```json\n{\n "image_generation_model": "NanoBananaPro",\n "grid_layout": "3x4",\n "grid_aspect_ratio": "16:9",\n "style_tags": "Xianxia, Chinese ancient style, 2D aesthetic, Cinematic",\n "global_settings": {\n "scene": "老旧厢房 interior at night, 发黄书册 and 油纸伞 as props, cold blue atmosphere",\n "time": "Midnight",\n "lighting": "Dim cold blue with warm lamp spots, soft shadows",\n "color_tone": "Cool blue primary, subtle warm accents",\n "character_position": "王林 center frame throughout"\n },\n "shots": [\n {\n "shot_number": "第1行第1列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Wide shot, 老旧厢房 interior night, 王林 sitting alone at desk, 油纸伞 and 发黄书册 in foreground, breeze through window gauze, cold blue tones, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第1行第2列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Full shot, slight low angle, 王林 seated at desk, left hand pressing 发黄书册, right hand gripping 油纸伞 handle, 青布长衫 collar catching light, lamp glow contrast, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第1行第3列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Medium shot, 王林 leaning forward whispering, brows furrowed, lamp shadow falling on 发黄书册 pages, cool tone, inner resolve, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第2行第1列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Close-up, 王林 eyes closed, resolute brow, hands clasped at chest, 油纸伞 silhouette blurred behind, warm lamp spots, shallow depth, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第2行第2列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Extreme close-up, 王林 hand gripping 油纸伞 handle, finger details sharp, 发黄书册 edge visible, umbrella pattern texture, rim light, cold blue tone, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第2行第3列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Ultra close-up, top light, 王林 eye detail, pupil reflecting lamp and book pages, tear traces on brow, sweat on face, shallow focus, emotion surge, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第3行第1列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Medium shot, 王林 rising to push 老旧厢房 window open, moonlight flooding in, night breeze moving gauze, village path dimly visible, cool tones, spatial layering, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第3行第2列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Close-up POV, 王林 gaze toward night outside 老旧厢房 window, quiet village, scattered lantern lights, window lattice shadows, deep blue grey, silent hope, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第3行第3列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Wide shot, 王林 seated back at desk in thought, murmuring softly, lamp dimming, starry night vast outside 老旧厢房, deep focus, blue yellow mix, determined mind, Xianxia, 2D aesthetic, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第4行第1列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第4行第2列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles"\n },\n {\n "shot_number": "第4行第3列",\n "grid_aspect_ratio": "16:9",\n "prompt_text": "Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles"\n }\n ]\n}\n```\n\n## 注意事项\n\n1. **原名强制保留**：每格prompt中的人物名、场景名、道具名、服装名必须使用用户输入的原始语言文字，禁止翻译、禁止拼音转写\n2. 每格必须写完整人物名称（原始语言），不可用代词（he/she/they）\n3. **插黑图固定格式**：`Pure black frame, 8k, ultra HD, high detail, no timecode, no subtitles`\n4. 直接输出JSON，不要任何解释或Markdown包裹\n5. 确保各格描述连贯一致\n6. shots数组数量必须与布局格数一致（含插黑图）\n7. **每个prompt_text必须以 `8k, ultra HD, high detail, no timecode, no subtitles` 结尾**\n8. **布局自动计算**：根据总镜头数（内容+插黑图）计算行数，列数固定为3\n9. **分辨率配置**：每个shot必须包含 `grid_aspect_ratio` 字段，值为 `"16:9"` 或 `"9:16"`\n\n## 原名保留自查清单\n\n输出前检查每个prompt_text：\n- [ ] 人物名是否为原始语言？（如 王林 而非 Wang Lin）\n- [ ] 场景名是否为原始语言？（如 老旧厢房 而非 old side room）\n- [ ] 道具名是否为原始语言？（如 油纸伞 而非 oil paper umbrella）\n- [ ] 服装名是否为原始语言？（如 青布长衫 而非 blue cloth robe）\n- [ ] 是否以超清标识结尾？\n- [ ] 插黑图是否使用固定格式？\n- [ ] 每个shot是否包含 `grid_aspect_ratio` 字段？\n\n## shot_number计算验证表\n\n**16:9布局（3列）验证：**\n| 镜头索引 | 计算公式 | shot_number |\n|---------|---------|-------------|\n| 0 | (0//3+1, 0%3+1) | 第1行第1列 |\n| 1 | (1//3+1, 1%3+1) | 第1行第2列 |\n| 2 | (2//3+1, 2%3+1) | 第1行第3列 |\n| 3 | (3//3+1, 3%3+1) | 第2行第1列 |\n| 4 | (4//3+1, 4%3+1) | 第2行第2列 |\n| 5 | (5//3+1, 5%3+1) | 第2行第3列 |\n\n**9:16布局（2列）验证：**\n| 镜头索引 | 计算公式 | shot_number |\n|---------|---------|-------------|\n| 0 | (0//2+1, 0%2+1) | 第1行第1列 |\n| 1 | (1//2+1, 1%2+1) | 第1行第2列 |\n| 2 | (2//2+1, 2%2+1) | 第2行第1列 |\n| 3 | (3//2+1, 3%2+1) | 第2行第2列 |\n| 4 | (4//2+1, 4%2+1) | 第3行第1列 |\n| 5 | (5//2+1, 5%2+1) | 第3行第2列 |', customValue: null, }, { diff --git a/src/routes/assets/generateAssets.ts b/src/routes/assets/generateAssets.ts index 53a3310..9d01e9d 100644 --- a/src/routes/assets/generateAssets.ts +++ b/src/routes/assets/generateAssets.ts @@ -123,14 +123,18 @@ export default router.post( state: "生成中", assetsId: id, }); + const apiConfig = await u.getPromptAi("assetsImage"); - const contentStr = await u.ai.image({ - systemPrompt, - prompt: userPrompt, - imageBase64: base64 ? [base64] : [], - size: "2K", - aspectRatio: "16:9", - }); + const contentStr = await u.ai.image( + { + systemPrompt, + prompt: userPrompt, + imageBase64: base64 ? [base64] : [], + size: "2K", + aspectRatio: "16:9", + }, + apiConfig, + ); let insertType; const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/); diff --git a/src/routes/assets/polishPrompt.ts b/src/routes/assets/polishPrompt.ts index 4f65878..3efa125 100644 --- a/src/routes/assets/polishPrompt.ts +++ b/src/routes/assets/polishPrompt.ts @@ -88,31 +88,16 @@ export default router.post( const result: ResultItem[] = Object.values(itemMap); const promptsList = await u.db("t_prompts").where("code", "in", ["role-polish", "scene-polish", "storyboard-polish", "tool-polish"]); - const propmptIds = promptsList.map((i) => i.id); - const mapList = await u - .db("t_aiModelMap") - .leftJoin("t_config", "t_config.id", "t_aiModelMap.configId") - .whereIn("t_aiModelMap.promptsId", propmptIds as number[]) - .select("t_config.model", "t_config.apiKey", "t_config.baseUrl", "t_config.manufacturer", "t_aiModelMap.promptsId"); + const apiConfigData = await u.getPromptAi("assetsPrompt"); const errPrompts = "不论用户说什么，请直接输出AI配置异常"; const getPromptValue = (code: string) => { const item = promptsList.find((p) => p.code === code); - if (item) { - const apiData = mapList.find((i) => i.promptsId == item.id); - if (apiData) delete apiData?.promptsId; - return { prompt: item?.customValue ?? item?.defaultValue ?? errPrompts, apiData: { ...(apiData ?? {}) } }; - } else { - return { - prompt: errPrompts, - apiData: {}, - }; - } + return item?.customValue ?? item?.defaultValue ?? errPrompts; }; const role = getPromptValue("role-polish"); const scene = getPromptValue("scene-polish"); const tool = getPromptValue("tool-polish"); const storyboard = getPromptValue("storyboard-polish"); - let apiConfig = {}; let systemPrompt = ""; let userPrompt = ""; if (type == "role") { @@ -120,8 +105,7 @@ export default router.post( const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange]; const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[]; const results: string = mergeNovelText(novelData); - systemPrompt = role.prompt; - apiConfig = role.apiData; + systemPrompt = role; userPrompt = ` 请根据以下参数生成角色标准四视图提示词： @@ -144,8 +128,7 @@ export default router.post( const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange]; const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[]; const results: string = mergeNovelText(novelData); - systemPrompt = scene.prompt; - apiConfig = scene.apiData; + systemPrompt = scene; userPrompt = ` 请根据以下参数生成场景图提示词： @@ -168,8 +151,7 @@ export default router.post( const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange]; const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[]; const results: string = mergeNovelText(novelData); - systemPrompt = tool.prompt; - apiConfig = tool.apiData; + systemPrompt = tool; userPrompt = ` 请根据以下参数生成道具图提示词： @@ -188,8 +170,7 @@ export default router.post( `; } if (type == "storyboard") { - systemPrompt = storyboard.prompt; - apiConfig = storyboard.apiData; + systemPrompt = storyboard; userPrompt = ` 请根据以下参数生成分镜图提示词： @@ -207,7 +188,6 @@ export default router.post( `; } async function generatePrompt() { - apiConfig = {}; const result = await u.ai.text.invoke( { messages: [ @@ -224,9 +204,7 @@ export default router.post( prompt: zod.string().describe("提示词"), }, }, - { - ...apiConfig, - }, + apiConfigData, ); // const result = await model.invoke({ // messages: [ @@ -256,7 +234,6 @@ export default router.post( res.status(200).send(success({ prompt: prompt, assetsId })); } catch (e: any) { - console.log("%c Line:235 🥚 e", "background:#33a5ff", e); return res.status(500).send(error(e?.data?.error?.message ?? e?.message ?? "生成失败")); } }, diff --git a/src/routes/other/testAI.ts b/src/routes/other/testAI.ts index 3211443..4fe5d85 100644 --- a/src/routes/other/testAI.ts +++ b/src/routes/other/testAI.ts @@ -13,12 +13,12 @@ export default router.post( modelName: z.string(), apiKey: z.string(), baseURL: z.string().optional(), + manufacturer: z.string(), }), async (req, res) => { - const { modelName, apiKey, baseURL } = req.body; + const { modelName, apiKey, baseURL, manufacturer } = req.body; const getWeatherTool = tool({ - // strict: true, description: "Get the weather in a location", inputSchema: z.object({ location: z.string().describe("The location to get the weather for"), @@ -43,6 +43,7 @@ export default router.post( model: modelName, apiKey, baseURL, + manufacturer, }, ); res.status(200).send(success(reply)); diff --git a/src/routes/other/testImage.ts b/src/routes/other/testImage.ts index c4aa78f..5f1714b 100644 --- a/src/routes/other/testImage.ts +++ b/src/routes/other/testImage.ts @@ -17,13 +17,21 @@ export default router.post( async (req, res) => { const { modelName, apiKey, baseURL, manufacturer } = req.body; try { - const image = await u.ai.image({ - prompt: - "一张16:9比例的图片，完美等分为2x2四宫格布局，各区域无缝衔接：\n左上宫格：一只可爱的猫，毛发蓬松，眼睛明亮，姿态俏皮\n右上宫格：一只友善的狗，金毛犬，表情愉悦，摇着尾巴\n左下宫格：一头健壮的牛，田园背景，目光温和，皮毛光泽\n右下宫格：一匹骏马，姿态优雅，鬃毛飘逸，肌肉健美\n风格要求：四个宫格风格统一，色彩鲜艳饱和，高清画质，细节清晰锐利，专业插画风格，线条干净，统一的左上方光源，柔和阴影，和谐配色，卡通/半写实风格，宫格间用白色或浅灰细线分隔", - imageBase64: [], - aspectRatio: "16:9", - size: "1K", - }); + const image = await u.ai.image( + { + prompt: + "一张16:9比例的图片，完美等分为2x2四宫格布局，各区域无缝衔接：\n左上宫格：一只可爱的猫，毛发蓬松，眼睛明亮，姿态俏皮\n右上宫格：一只友善的狗，金毛犬，表情愉悦，摇着尾巴\n左下宫格：一头健壮的牛，田园背景，目光温和，皮毛光泽\n右下宫格：一匹骏马，姿态优雅，鬃毛飘逸，肌肉健美\n风格要求：四个宫格风格统一，色彩鲜艳饱和，高清画质，细节清晰锐利，专业插画风格，线条干净，统一的左上方光源，柔和阴影，和谐配色，卡通/半写实风格，宫格间用白色或浅灰细线分隔", + imageBase64: [], + aspectRatio: "16:9", + size: "1K", + }, + { + model: modelName, + apiKey, + baseURL, + manufacturer, + }, + ); res.status(200).send(success(image)); } catch (err) { const msg = u.error(err).message; diff --git a/src/routes/other/testVideo.ts b/src/routes/other/testVideo.ts index 9ff7139..45c76c6 100644 --- a/src/routes/other/testVideo.ts +++ b/src/routes/other/testVideo.ts @@ -12,20 +12,28 @@ export default router.post( modelName: z.string().optional(), apiKey: z.string(), baseURL: z.string().optional(), - manufacturer: z.enum(["runninghub", "volcengine", "apimart", "gemini", "openAi"]), + manufacturer: z.string(), }), async (req, res) => { const { modelName, apiKey, baseURL, manufacturer } = req.body; try { - const videoPath = await u.ai.video({ - imageBase64: [], - savePath: "test.mp4", - prompt: "stickman Dances", - duration: 4, - resolution: "720p", - aspectRatio: "16:9", - audio: false, - }); + const videoPath = await u.ai.video( + { + imageBase64: [], + savePath: "test.mp4", + prompt: "stickman Dances", + duration: 4, + resolution: "720p", + aspectRatio: "16:9", + audio: false, + }, + { + model: modelName, + apiKey, + baseURL, + manufacturer, + }, + ); const url = await u.oss.getFileUrl(videoPath); res.status(200).send(success(url)); } catch (err: any) { diff --git a/src/routes/setting/addModel.ts b/src/routes/setting/addModel.ts index 715f5f3..0c77bb3 100644 --- a/src/routes/setting/addModel.ts +++ b/src/routes/setting/addModel.ts @@ -8,23 +8,23 @@ const router = express.Router(); export default router.post( "/", validateFields({ - type: z.string(), - name: z.string(), + type: z.enum(["text", "video", "image"]), model: z.string(), baseUrl: z.string(), apiKey: z.string(), + modelType: z.string(), manufacturer: z.string(), }), async (req, res) => { - const { type, name, model, baseUrl, apiKey, manufacturer } = req.body; + const { type, model, baseUrl, apiKey, manufacturer, modelType } = req.body; await u.db("t_config").insert({ type, - name, model, baseUrl, apiKey, manufacturer, + modelType, createTime: Date.now(), userId: 1, }); diff --git a/src/routes/setting/configurationModel.ts b/src/routes/setting/configurationModel.ts index 1a56c47..7d7687e 100644 --- a/src/routes/setting/configurationModel.ts +++ b/src/routes/setting/configurationModel.ts @@ -8,20 +8,13 @@ const router = express.Router(); export default router.post( "/", validateFields({ - id: z.number().optional(), - promptsId: z.number(), + id: z.number(), configId: z.number(), }), async (req, res) => { - const { id, promptsId, configId } = req.body; + const { id, configId } = req.body; if (id) { await u.db("t_aiModelMap").where("id", id).update({ - promptsId, - configId, - }); - } else { - await u.db("t_aiModelMap").insert({ - promptsId, configId, }); } diff --git a/src/routes/setting/getAiModelMap.ts b/src/routes/setting/getAiModelMap.ts index 4e0b056..ebd6d66 100644 --- a/src/routes/setting/getAiModelMap.ts +++ b/src/routes/setting/getAiModelMap.ts @@ -1,13 +1,13 @@ import express from "express"; import u from "@/utils"; import { success } from "@/lib/responseFormat"; + const router = express.Router(); export default router.post("/", async (req, res) => { const configData = await u - .db("t_prompts") - .leftJoin("t_aiModelMap", "t_prompts.id", "t_aiModelMap.promptsId") - .leftJoin("t_config", "t_config.id", "t_aiModelMap.configId") - .select("t_prompts.id as promptsId", "t_prompts.code", "t_prompts.name", "t_config.model", "t_aiModelMap.id"); + .db("t_aiModelMap") + .leftJoin("t_config", "t_aiModelMap.configId", "t_config.id") + .select("t_aiModelMap.name", "t_config.model", "t_aiModelMap.id"); res.status(200).send(success(configData)); }); diff --git a/src/routes/setting/updeteModel.ts b/src/routes/setting/updeteModel.ts index be1ca28..421a06d 100644 --- a/src/routes/setting/updeteModel.ts +++ b/src/routes/setting/updeteModel.ts @@ -9,23 +9,23 @@ export default router.post( "/", validateFields({ id: z.number(), - type: z.string(), - name: z.string(), + type: z.enum(["text", "video", "image"]), model: z.string(), baseUrl: z.string(), + modelType: z.string(), apiKey: z.string(), manufacturer: z.string(), }), async (req, res) => { - const { id, type, name, model, baseUrl, apiKey, manufacturer } = req.body; + const { id, type, model, baseUrl, apiKey, manufacturer, modelType } = req.body; await u.db("t_config").where("id", id).update({ type, - name, model, baseUrl, apiKey, manufacturer, + modelType, }); res.status(200).send(success("编辑成功")); }, diff --git a/src/routes/storyboard/batchSuperScoreImage.ts b/src/routes/storyboard/batchSuperScoreImage.ts index f10f97d..39d1064 100644 --- a/src/routes/storyboard/batchSuperScoreImage.ts +++ b/src/routes/storyboard/batchSuperScoreImage.ts @@ -17,19 +17,19 @@ async function urlToBase64(imageUrl: string): Promise { } // 超分并保存到 oss -async function superResolutionAndSave( - src: string, - projectId: number, - videoRatio: string, -): Promise<{ ossPath: string; base64: string }> { - const contentStr = await u.ai.image({ - aspectRatio: videoRatio, - size: "1K", - resType: "b64", - systemPrompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率", - prompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率", - imageBase64: [await urlToBase64(src)], - }); +async function superResolutionAndSave(src: string, projectId: number, videoRatio: string): Promise<{ ossPath: string; base64: string }> { + const apiConfig = await u.getPromptAi("storyboardImage"); + const contentStr = await u.ai.image( + { + aspectRatio: videoRatio, + size: "1K", + resType: "b64", + systemPrompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率", + prompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率", + imageBase64: [await urlToBase64(src)], + }, + apiConfig, + ); const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/); const base64Str = match ? match[1] : contentStr; const buffer = Buffer.from(base64Str, "base64"); @@ -50,9 +50,9 @@ export default router.post( id: z.string(), prompt: z.string().optional(), src: z.string(), - }) + }), ), - }) + }), ), }), async (req, res) => { @@ -63,9 +63,7 @@ export default router.post( if (!projectData) return res.status(500).send(error("项目不存在")); // 遍历处理每个分镜段 - const processSegment = async ( - segment: { cells: { id: string; src: string }[] } - ) => { + const processSegment = async (segment: { cells: { id: string; src: string }[] }) => { // 超分所有 cell const cellsWithSuperscore = await Promise.all( segment.cells.map(async (cell) => { @@ -76,9 +74,9 @@ export default router.post( scriptId, filePath: ossPath, // oss 路径（未签名） src: cell.src, - type: "分镜" + type: "分镜", }; - }) + }), ); return cellsWithSuperscore; }; @@ -92,9 +90,9 @@ export default router.post( (item.value as any[]).map(async (cell) => ({ ...cell, filePath: await u.oss.getFileUrl(cell.filePath ?? ""), - })) - ) + })), + ), ); res.status(200).send(success(flatList)); - } + }, ); diff --git a/src/routes/storyboard/generateVideoPrompt.ts b/src/routes/storyboard/generateVideoPrompt.ts index 0fa239a..fb97a04 100644 --- a/src/routes/storyboard/generateVideoPrompt.ts +++ b/src/routes/storyboard/generateVideoPrompt.ts @@ -4,6 +4,7 @@ import { error, success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; import { z } from "zod"; import path from "path"; +import axios from "axios"; const router = express.Router(); @@ -103,7 +104,12 @@ const prompt = ` 现在请根据我提供的分镜内容，严格按照以上规则输出 Motion Prompt JSON 对象。 `; - +async function urlToBase64(imageUrl: string): Promise { + const response = await axios.get(imageUrl, { responseType: "arraybuffer" }); + const contentType = response.headers["content-type"] || "image/png"; + const base64 = Buffer.from(response.data, "binary").toString("base64"); + return `data:${contentType};base64,${base64}`; +} // 生成单个分镜提示 async function generateSingleVideoPrompt({ scriptText, @@ -114,19 +120,6 @@ async function generateSingleVideoPrompt({ storyboardPrompt: string; ossPath: string; }): Promise<{ content: string; time: number; name: string }> { - let rootDir: string; - if (typeof process.versions?.electron !== "undefined") { - const { app } = require("electron"); - const userDataDir: string = app.getPath("userData"); - rootDir = path.join(userDataDir, "uploads"); - } else { - rootDir = path.join(process.cwd(), "uploads"); - } - - let imagePath = ossPath; - if (ossPath.includes("http")) { - imagePath = new URL(ossPath).pathname; - } const messages: any[] = [ { role: "system", @@ -140,24 +133,27 @@ async function generateSingleVideoPrompt({ text: `剧本内容:${scriptText}\n分镜提示词:${storyboardPrompt}`, }, { - type: "local", - path: path.join(rootDir, imagePath), + type: "image", + image: await urlToBase64(ossPath), }, ], }, ]; try { - const result = await u.ai.text.invoke({ - messages, - output: { - time: z.number().describe("时长,镜头时长 1-15"), - content: z.string().describe("提示词内容"), - name: z.string().describe("分镜名称"), - }, - }); - console.log("%c Line:156 🍩 result", "background:#33a5ff", result); + const apiConfig = await u.getPromptAi("videoPrompt"); + const result = await u.ai.text.invoke( + { + messages, + output: { + time: z.number().describe("时长,镜头时长 1-15"), + content: z.string().describe("提示词内容"), + name: z.string().describe("分镜名称"), + }, + }, + apiConfig, + ); if (!result) { console.error("AI 返回结果为空:", result); throw new Error("AI 返回结果为空"); diff --git a/src/routes/video/addVideo.ts b/src/routes/video/addVideo.ts index f574a75..bfccd7c 100644 --- a/src/routes/video/addVideo.ts +++ b/src/routes/video/addVideo.ts @@ -42,5 +42,5 @@ export default router.post( }); res.status(200).send(success({ message: "新增视频成功" })); - } + }, ); diff --git a/src/routes/video/addVideoConfig.ts b/src/routes/video/addVideoConfig.ts index e055144..565a55b 100644 --- a/src/routes/video/addVideoConfig.ts +++ b/src/routes/video/addVideoConfig.ts @@ -1,6 +1,6 @@ import express from "express"; import u from "@/utils"; -import { success } from "@/lib/responseFormat"; +import { error, success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; import { z } from "zod"; const router = express.Router(); @@ -20,8 +20,8 @@ export default router.post( validateFields({ scriptId: z.number(), projectId: z.number(), - manufacturer: z.string(), - mode: z.enum(["startEnd", "multi", "single"]), + configId: z.number(), + mode: z.enum(["startEnd", "multi", "single",'text','']), startFrame: imageItemSchema.optional(), endFrame: imageItemSchema.optional(), images: z @@ -38,19 +38,21 @@ export default router.post( prompt: z.string().optional(), }), async (req, res) => { - const { scriptId, projectId, manufacturer, mode, startFrame, endFrame, images, resolution, duration, prompt } = req.body; + const { scriptId, projectId, configId, mode, startFrame, endFrame, images, resolution, duration, prompt } = req.body; // 生成新ID const maxIdResult: any = await u.db("t_videoConfig").max("id as maxId").first(); const newId = (maxIdResult?.maxId || 0) + 1; const now = Date.now(); - + const configData = await u.db("t_config").where("id", configId).first(); + if (!configData) return res.status(500).send(error("不存在的模型")); // 插入数据 await u.db("t_videoConfig").insert({ id: newId, scriptId, projectId, - manufacturer, + manufacturer: configData.manufacturer, + aiConfigId: configId, mode, startFrame: startFrame ? JSON.stringify(startFrame) : null, endFrame: endFrame ? JSON.stringify(endFrame) : null, @@ -70,7 +72,9 @@ export default router.post( id: newId, scriptId, projectId, - manufacturer, + manufacturer: configData.manufacturer, + aiConfigId: configId, + model: configData.model, mode, startFrame, endFrame, diff --git a/src/routes/video/generatePrompt.ts b/src/routes/video/generatePrompt.ts index b32f96c..ea8dd17 100644 --- a/src/routes/video/generatePrompt.ts +++ b/src/routes/video/generatePrompt.ts @@ -1,6 +1,6 @@ import express from "express"; import u from "@/utils"; -import { success } from "@/lib/responseFormat"; +import { error, success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; import { z } from "zod"; @@ -8,47 +8,26 @@ const router = express.Router(); type GenerateMode = "startEnd" | "multi" | "single"; -const getSystemPrompt = async (mode: GenerateMode): Promise<{ prompt: string; apiConfig: Object }> => { +const getSystemPrompt = async (mode: GenerateMode) => { const promptsList = await u.db("t_prompts").where("code", "in", ["video-startEnd", "video-multi", "video-single", "video-main"]); - const promptAiConfig = await u.getPromptAi(promptsList.map((i) => i.id) as number[]); - const errPrompts = "不论用户说什么，请直接输出AI配置异常"; const getPromptValue = (code: string) => { const item = promptsList.find((p) => p.code === code); - const subData = promptAiConfig.find((i) => i?.promptsId == item?.id); - const returnData = { - prompt: item?.customValue ?? item?.defaultValue ?? errPrompts, - apiConfig: {}, - }; - if (subData) { - returnData.apiConfig = { ...subData }; - return returnData; - } else { - return returnData; - } + return item?.customValue ?? item?.defaultValue ?? errPrompts; }; const startEnd = getPromptValue("video-startEnd"); const multi = getPromptValue("video-multi"); const single = getPromptValue("video-single"); const main = getPromptValue("video-main"); - const modeDescriptions: Record< - GenerateMode, - { - prompt: string; - apiConfig: Object; - } - > = { + const modeDescriptions = { startEnd: startEnd, multi: multi, single: single, }; const modeData = modeDescriptions[mode]; - return { - prompt: `${main}\n\n${modeData.prompt}`, - apiConfig: modeData.apiConfig, - }; + return `${main}\n\n${modeData}`; }; const getModeDescription = (mode: GenerateMode): string => { @@ -82,16 +61,18 @@ export default router.post( const shotCount = images.length; const avgDuration = (parseFloat(duration) / shotCount).toFixed(1); const promptConfig = await getSystemPrompt(mode); - const result = await u.ai.text.invoke( - { - messages: [ - { - role: "system", - content: promptConfig.prompt, - }, - { - role: "user", - content: `Mode: ${getModeDescription(mode)} + const promptAiConfig = await u.getPromptAi("videoPrompt"); + try { + const result = await u.ai.text.invoke( + { + messages: [ + { + role: "system", + content: promptConfig, + }, + { + role: "user", + content: `Mode: ${getModeDescription(mode)} Reference Images: ${imagePrompts} @@ -105,12 +86,15 @@ Parameters: - Average Duration: ${avgDuration}s per shot Generate storyboard prompts:`, - }, - ], - }, - promptConfig.apiConfig, - ); + }, + ], + }, + promptAiConfig, + ); - res.status(200).send(success(result.text)); + res.status(200).send(success(result.text)); + } catch (e) { + return res.status(500).send(error(u.error(e).message)); + } }, ); diff --git a/src/routes/video/generateVideo.ts b/src/routes/video/generateVideo.ts index ae18410..14bb5d4 100644 --- a/src/routes/video/generateVideo.ts +++ b/src/routes/video/generateVideo.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { v4 as uuidv4 } from "uuid"; import { error, success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; +import { t_config } from "@/types/database"; const router = express.Router(); @@ -13,35 +14,52 @@ export default router.post( validateFields({ projectId: z.number(), scriptId: z.number(), - configId: z.number().optional(), // 关联的视频配置ID + configId: z.number().optional(), // 关联的视频配 置ID type: z.string().optional(), resolution: z.string(), + aiConfigId: z.number(), filePath: z.array(z.string()), duration: z.number(), prompt: z.string(), }), async (req, res) => { - const { type, scriptId, projectId, configId, resolution, filePath, duration, prompt } = req.body; + const { type, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt } = req.body; - // 参数校验 - if (type === "volcengine") { - if (duration < 4 || duration > 12) { - return res.status(400).send(error("视频时长需在4-12秒之间")); - } - if (!["480p", "720p", "1080p"].includes(resolution)) { - return res.status(400).send(error("视频分辨率不正确")); - } + // // 参数校验 + // if (type === "volcengine") { + // if (duration < 4 || duration > 12) { + // return res.status(400).send(error("视频时长需在4-12秒之间")); + // } + // if (!["480p", "720p", "1080p"].includes(resolution)) { + // return res.status(400).send(error("视频分辨率不正确")); + // } + // } + + // if (type === "runninghub") { + // if (duration !== 10 && duration !== 15) { + // return res.status(400).send(error("视频时长只能是10秒或15秒")); + // } + // if (resolution !== "9:16" && resolution !== "16:9") { + // return res.status(400).send(error("视频分辨率不正确")); + // } + // } + const configData = await u.db("t_videoConfig").where("id", configId).first(); + if (!configData) { + return res.status(500).send(error("视频配置不存在")); } - if (type === "runninghub") { - if (duration !== 10 && duration !== 15) { - return res.status(400).send(error("视频时长只能是10秒或15秒")); - } - if (resolution !== "9:16" && resolution !== "16:9") { - return res.status(400).send(error("视频分辨率不正确")); - } + // 优先使用视频配置中的AI配置ID查询,查不到再使用传入的aiConfigId + let aiConfigData = null; + if (configData.aiConfigId) { + aiConfigData = await u.db("t_config").where("id", configData.aiConfigId).first(); + } + if (!aiConfigData) { + aiConfigData = await u.db("t_config").where("id", aiConfigId).first(); } + if (!aiConfigData) { + return res.status(500).send(error("模型配置不存在")); + } // 过滤掉空值 let fileUrl = filePath.filter((p: string) => p && p.trim() !== ""); @@ -103,7 +121,7 @@ export default router.post( res.status(200).send(success({ id: videoId, configId: configId || null })); // 异步生成视频 - generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, type); + generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, aiConfigData); }, ); @@ -116,7 +134,7 @@ async function generateVideoAsync( prompt: string, duration: number, resolution: string, - type?: string, + aiConfigData: t_config, ) { try { const projectData = await u.db("t_project").where("id", projectId).select("artStyle").first(); @@ -149,14 +167,22 @@ ${prompt} 3. 关键人物在画面中全部清晰显示，不得被遮挡、缺失或省略 4. 画面真实、细致，无畸形、无模糊、无杂物、无多余人物、无文字、水印、logo `; - const videoPath = await u.ai.video({ - imageBase64, - savePath, - prompt: inputPrompt, - duration: duration as any, - aspectRatio: resolution as any, - resolution: resolution as any, - }); + const videoPath = await u.ai.video( + { + imageBase64, + savePath, + prompt: inputPrompt, + duration: duration as any, + aspectRatio: resolution as any, + resolution: resolution as any, + }, + { + baseURL: aiConfigData?.baseUrl!, + model: aiConfigData?.model!, + apiKey: aiConfigData?.apiKey!, + manufacturer: aiConfigData?.manufacturer!, + }, + ); if (videoPath) { // 生成成功，更新状态为 1 diff --git a/src/routes/video/getManufacturer.ts b/src/routes/video/getManufacturer.ts index cf2dcb3..750822b 100644 --- a/src/routes/video/getManufacturer.ts +++ b/src/routes/video/getManufacturer.ts @@ -14,8 +14,8 @@ export default router.post( async (req, res) => { const { userId } = req.body; - const data = await u.db("t_config").where("userId", userId).select("manufacturer", "model"); + const data = await u.db("t_config").where("type", "video").where("userId", userId).select("manufacturer", "model", "id"); res.status(200).send(success(data)); - } + }, ); diff --git a/src/routes/video/getVideoConfigs.ts b/src/routes/video/getVideoConfigs.ts index b15e519..7246212 100644 --- a/src/routes/video/getVideoConfigs.ts +++ b/src/routes/video/getVideoConfigs.ts @@ -15,16 +15,20 @@ export default router.post( const { scriptId } = req.body; // 查询该脚本下的所有视频配置 - const configs = await u.db("t_videoConfig") + const configs = await u + .db("t_videoConfig") + .leftJoin("t_config", "t_config.id", "t_videoConfig.aiConfigId") .where({ scriptId }) - .orderBy("createTime", "desc"); - + .orderBy("createTime", "desc") + .select("t_videoConfig.*", "t_config.manufacturer as manufacturer", "t_config.model"); // 解析 JSON 字段 const result = configs.map((config: any) => ({ id: config.id, scriptId: config.scriptId, projectId: config.projectId, + aiConfigId: config.aiConfigId, manufacturer: config.manufacturer, + model: config.model, mode: config.mode, startFrame: config.startFrame ? JSON.parse(config.startFrame) : null, endFrame: config.endFrame ? JSON.parse(config.endFrame) : null, diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 482bd85..250f5ab 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,10 +1,11 @@ -// @db-hash e1460b0ace03f6aaed458653a32b6ffb +// @db-hash 4cd44aef6bb6ffb02c4619525966496d //该文件由脚本自动生成，请勿手动修改 export interface t_aiModelMap { 'configId'?: number | null; 'id'?: number; - 'promptsId'?: number | null; + 'key'?: string | null; + 'name'?: string | null; } export interface t_assets { 'duration'?: string | null; @@ -37,7 +38,7 @@ export interface t_config { 'id'?: number; 'manufacturer'?: string | null; 'model'?: string | null; - 'name'?: string | null; + 'modelType'?: string | null; 'type'?: string | null; 'userId'?: number | null; } @@ -135,6 +136,7 @@ export interface t_video { 'time'?: number | null; } export interface t_videoConfig { + 'aiConfigId'?: number | null; 'createTime'?: number | null; 'duration'?: number | null; 'endFrame'?: string | null; diff --git a/src/utils/ai/image/index.ts b/src/utils/ai/image/index.ts index 281742a..424b834 100644 --- a/src/utils/ai/image/index.ts +++ b/src/utils/ai/image/index.ts @@ -28,13 +28,16 @@ const modelInstance = { other, } as const; -export default async (input: ImageConfig, config?: AIConfig) => { - const sqlTextModelConfig = await u.getConfig("image"); - const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config }; +export default async (input: ImageConfig, config: AIConfig) => { + const { model, apiKey, baseURL, manufacturer } = { ...config }; + if (!config || !config?.model || !config?.apiKey || !config?.manufacturer) throw new Error("请检查模型配置是否正确"); + const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance]; if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商"); - const owned = modelList.find((m) => m.model === model); - if (!owned) throw new Error("不支持的模型"); + if (manufacturer !== "other") { + const owned = modelList.find((m) => m.model === model); + if (!owned) throw new Error("不支持的模型"); + } // 补充图片的 base64 内容类型字符串 if (input.imageBase64 && input.imageBase64.length > 0) { diff --git a/src/utils/ai/image/owned/gemini.ts b/src/utils/ai/image/owned/gemini.ts index c865394..97aa30e 100644 --- a/src/utils/ai/image/owned/gemini.ts +++ b/src/utils/ai/image/owned/gemini.ts @@ -3,14 +3,13 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { generateText } from "ai"; export default async (input: ImageConfig, config: AIConfig): Promise => { - console.log("%c Line:6 🌰 config", "background:#ffdd4d", config); if (!config.model) throw new Error("缺少Model名称"); if (!config.apiKey) throw new Error("缺少API Key"); if (!input.prompt) throw new Error("缺少提示词"); const google = createGoogleGenerativeAI({ apiKey: config.apiKey, - baseURL: config.baseURL, + baseURL: config?.baseURL ?? "https://generativelanguage.googleapis.com/v1beta", }); // 构建完整的提示词 diff --git a/src/utils/ai/image/owned/other.ts b/src/utils/ai/image/owned/other.ts index 6e72ef5..5683da7 100644 --- a/src/utils/ai/image/owned/other.ts +++ b/src/utils/ai/image/owned/other.ts @@ -1,5 +1,5 @@ import "../type"; -import { generateImage, generateText } from "ai"; +import { generateImage, generateText, ModelMessage } from "ai"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; export default async (input: ImageConfig, config: AIConfig): Promise => { @@ -27,9 +27,24 @@ export default async (input: ImageConfig, config: AIConfig): Promise => const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt; const model = config.model; if (model.includes("gemini") || model.includes("nano")) { + let promptData; + if (input.imageBase64 && input.imageBase64.length) { + promptData = [{ role: "system", content: fullPrompt + `请直接输出图片` }]; + (promptData as ModelMessage[]).push({ + role: "user", + content: input.imageBase64.map((i) => ({ + type: "image", + image: i, + })), + }); + } else { + promptData = fullPrompt + `请直接输出图片`; + } + console.log("%c Line:31 🍅 promptData", "background:#2eafb0", promptData); + const result = await generateText({ model: otherProvider.languageModel(model), - prompt: fullPrompt + `请直接输出图片`, + prompt: promptData as string | ModelMessage[], providerOptions: { google: { imageConfig: { diff --git a/src/utils/ai/image/owned/vidu.ts b/src/utils/ai/image/owned/vidu.ts index 99a8fee..50ea704 100644 --- a/src/utils/ai/image/owned/vidu.ts +++ b/src/utils/ai/image/owned/vidu.ts @@ -20,7 +20,6 @@ function template(replaceObj: Record, url: string) { export default async (input: ImageConfig, config: AIConfig): Promise => { if (!config.model) throw new Error("缺少Model名称"); if (!config.apiKey) throw new Error("缺少API Key"); - const apiKey = "Token " + config.apiKey.replace(/Token\s+/g, "").trim(); const viduq2Ratio = ["16:9", "9:16", "1:1", "3:4", "4:3", "21:9", "2:3", "3:2"]; const viduq1Ratio = ["16:9", "9:16", "1:1", "3:4", "4:3"]; @@ -60,7 +59,8 @@ export default async (input: ImageConfig, config: AIConfig): Promise => ...(images.length && { images: images }), }; - const urlObj = getApiUrl(config.baseURL!); + const urlObj = getApiUrl(config.baseURL! ?? "https://api.vidu.cn/ent/v2/reference2image|https://api.vidu.cn/ent/v2/tasks/{id}/creations"); + try { const { data } = await axios.post(urlObj.requestUrl, body, { headers: { Authorization: apiKey } }); @@ -69,17 +69,13 @@ export default async (input: ImageConfig, config: AIConfig): Promise => return await pollTask(async () => { const { data: queryData } = await axios.get(queryUrl, { headers: { Authorization: apiKey } }); - if (queryData.state !== 0) { - return { completed: false, error: queryData.message || "查询任务失败" }; - } - - const { state, err_code, creations } = queryData.data || {}; + const { state, err_code, creations } = queryData || {}; if (state === "failed") { return { completed: false, error: err_code || "图片生成失败" }; } - if (state === "succeed") { + if (state === "success") { return { completed: true, url: creations?.[0]?.url }; } diff --git a/src/utils/ai/text/index.ts b/src/utils/ai/text/index.ts index 09a987e..a353ab3 100644 --- a/src/utils/ai/text/index.ts +++ b/src/utils/ai/text/index.ts @@ -6,7 +6,6 @@ import { parse } from "best-effort-json-parser"; import modelList from "./modelList"; import { z } from "zod"; import { OpenAIProvider } from "@ai-sdk/openai"; - interface AIInput | undefined = undefined> { system?: string; tools?: Record; @@ -23,10 +22,9 @@ interface AIConfig { manufacturer?: string; } -const buildOptions = async (input: AIInput, config: AIConfig) => { - let sqlTextModelConfig = {}; - if (!config || !config?.model || !config?.apiKey || !config?.baseURL) sqlTextModelConfig = await u.getConfig("text"); - const { model, apiKey, baseURL, manufacturer } = { ...(sqlTextModelConfig as Awaited>), ...config }; +const buildOptions = async (input: AIInput, config: AIConfig = {}) => { + if (!config || !config?.model || !config?.apiKey || !config?.baseURL || !config?.manufacturer) throw new Error("请检查模型配置是否正确"); + const { model, apiKey, baseURL, manufacturer } = { ...config }; let owned; if (manufacturer == "other") { owned = modelList.find((m) => m.manufacturer === manufacturer); @@ -39,7 +37,9 @@ const buildOptions = async (input: AIInput, config: AIConfig) => { const maxStep = input.maxStep ?? (input.tools ? Object.keys(input.tools).length * 5 : undefined); const outputBuilders: Record any> = { - schema: (s) => Output.object({ schema: z.object(s) }), + schema: (s) => { + return Output.object({ schema: z.object(s) }); + }, object: () => { const jsonSchemaPrompt = `\n请按照以下 JSON Schema 格式返回结果:\n${JSON.stringify( z.toJSONSchema(z.object(input.output)), @@ -52,16 +52,11 @@ const buildOptions = async (input: AIInput, config: AIConfig) => { }; const output = input.output ? (outputBuilders[owned.responseFormat]?.(input.output) ?? null) : null; - const modelFn = owned.manufacturer == "doubao" ? (modelInstance as OpenAIProvider).chat(model!) : modelInstance(model!); + const chatModelManufacturer = ["doubao", "other", "openai"]; + const modelFn = chatModelManufacturer.includes(owned.manufacturer) ? (modelInstance as OpenAIProvider).chat(model!) : modelInstance(model!); return { config: { - model: - process.env.NODE_ENV === "dev" - ? wrapLanguageModel({ - model: modelFn as any, - middleware: devToolsMiddleware(), - }) - : (modelFn as LanguageModel), + model: modelFn as LanguageModel, ...(input.system && { system: input.system }), ...(input.prompt ? { prompt: input.prompt } : { messages: input.messages! }), ...(input.tools && owned.tool && { tools: input.tools }), @@ -79,7 +74,7 @@ const ai = Object.create({}) as { stream(input: AIInput, config?: AIConfig): Promise>; }; -ai.invoke = async (input: AIInput, config: AIConfig = {}) => { +ai.invoke = async (input: AIInput, config: AIConfig) => { const options = await buildOptions(input, config); const result = await generateText(options.config); if (options.responseFormat === "object" && input.output) { @@ -95,7 +90,7 @@ ai.invoke = async (input: AIInput, config: AIConfig = {}) => { return result; }; -ai.stream = async (input: AIInput, config: AIConfig = {}) => { +ai.stream = async (input: AIInput, config: AIConfig) => { const options = await buildOptions(input, config); return streamText(options.config); }; diff --git a/src/utils/ai/text/modelList.ts b/src/utils/ai/text/modelList.ts index a37ea8a..e472e78 100644 --- a/src/utils/ai/text/modelList.ts +++ b/src/utils/ai/text/modelList.ts @@ -417,7 +417,7 @@ const modelList: Owned[] = [ responseFormat: "schema", image: true, think: false, - instance: createOpenAICompatible, + instance: createOpenAI, tool: true, }, ]; diff --git a/src/utils/ai/video/index.ts b/src/utils/ai/video/index.ts index 41da25c..f8f7183 100644 --- a/src/utils/ai/video/index.ts +++ b/src/utils/ai/video/index.ts @@ -22,8 +22,10 @@ const modelInstance = { } as const; export default async (input: VideoConfig, config?: AIConfig) => { - const sqlTextModelConfig = await u.getConfig("video"); - const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config }; + console.log("%c Line:25 🥛 config", "background:#2eafb0", config); + const { model, apiKey, baseURL, manufacturer } = { ...config }; + if (!config || !config?.model || !config?.apiKey) throw new Error("请检查模型配置是否正确"); + const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance]; if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的视频厂商"); const owned = modelList.find((m) => m.model === model); diff --git a/src/utils/ai/video/owned/volcengine.ts b/src/utils/ai/video/owned/volcengine.ts index 25e1ec4..328e031 100644 --- a/src/utils/ai/video/owned/volcengine.ts +++ b/src/utils/ai/video/owned/volcengine.ts @@ -47,6 +47,7 @@ export default async (input: VideoConfig, config: AIConfig) => { }); const taskId = createResponse.data.id; + if (!taskId) throw new Error("视频任务创建失败"); // 轮询任务状态 diff --git a/src/utils/ai/video/type.ts b/src/utils/ai/video/type.ts index 1687c0b..4c38699 100644 --- a/src/utils/ai/video/type.ts +++ b/src/utils/ai/video/type.ts @@ -12,4 +12,5 @@ interface AIConfig { model?: string; apiKey?: string; baseURL?: string; + manufacturer?: string; } diff --git a/src/utils/editImage.ts b/src/utils/editImage.ts index 14aaec9..1139621 100644 --- a/src/utils/editImage.ts +++ b/src/utils/editImage.ts @@ -79,13 +79,18 @@ async function convertDirectiveAndImages(images: Record, directi */ export default async (images: Record, directive: string, projectId: number) => { const { prompt, images: base64Images } = await convertDirectiveAndImages(images, directive); - const contentStr = await u.ai.image({ - systemPrompt: "根据用户提供的具体修改指令，对上传的图片进行智能编辑。", - prompt: prompt, - imageBase64: base64Images, - aspectRatio: "16:9", - size: "1K", - }); + const apiConfig = await u.getPromptAi("editImage"); + + const contentStr = await u.ai.image( + { + systemPrompt: "根据用户提供的具体修改指令，对上传的图片进行智能编辑。", + prompt: prompt, + imageBase64: base64Images, + aspectRatio: "16:9", + size: "1K", + }, + apiConfig, + ); const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/); const buffer = Buffer.from(match && match.length >= 1 ? match[1]! : contentStr, "base64"); const filePath = `/${projectId}/storyboard/${uuid()}.jpg`; diff --git a/src/utils/generateScript.ts b/src/utils/generateScript.ts index 77235b5..74e38d6 100644 --- a/src/utils/generateScript.ts +++ b/src/utils/generateScript.ts @@ -127,7 +127,7 @@ ${episodePrompt} ${novelData}`; const prompts = await u.db("t_prompts").where("code", "script").first(); - const promptConfig = await u.getPromptAi(prompts?.id); + const promptConfig = await u.getPromptAi("generateScript"); const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出AI配置异常"; const result = await u.ai.text.invoke( diff --git a/src/utils/getPromptAi.ts b/src/utils/getPromptAi.ts index 1618cba..9f02401 100644 --- a/src/utils/getPromptAi.ts +++ b/src/utils/getPromptAi.ts @@ -1,26 +1,19 @@ import { db } from "./db"; interface AiConfig { - model: string; + model?: string; apiKey: string; - baseUrl: string; + baseURL?: string; manufacturer: string; - promptsId: number; } -export default async function getPromptAi(promptsId: number | undefined): Promise; -export default async function getPromptAi(promptsId: number[]): Promise; - -export default async function getPromptAi(promptsId: number | number[] | undefined): Promise { - if (!promptsId) return {}; - const ids = Array.isArray(promptsId) ? promptsId.filter(Boolean) : [promptsId]; - const mapList = await db("t_aiModelMap") +export default async function getPromptAi(key: string): Promise { + const aiConfigData = await db("t_aiModelMap") .leftJoin("t_config", "t_config.id", "t_aiModelMap.configId") - .whereIn("t_aiModelMap.promptsId", ids) - .select("t_config.model", "t_config.apiKey", "t_config.baseUrl", "t_config.manufacturer", "t_aiModelMap.promptsId"); + .where("t_aiModelMap.key", key) + .select("t_config.model", "t_config.apiKey", "t_config.baseUrl as baseURL", "t_config.manufacturer") + .first(); - if (Array.isArray(promptsId)) { - return mapList as AiConfig[]; - } else { - return mapList[0] ? (mapList[0] as AiConfig) : {}; - } + if (aiConfigData) { + return aiConfigData as AiConfig; + } else return {}; }