Merge branch 'develop'
This commit is contained in:
ACT丶流星雨
commit
546f94c10d
16
.dockerignore
Normal file
16
.dockerignore
Normal file
@ -0,0 +1,16 @@
|
||||
node_modules
|
||||
build
|
||||
dist
|
||||
logs
|
||||
uploads
|
||||
.git
|
||||
.gitignore
|
||||
*.md
|
||||
LICENSE
|
||||
NOTICES.txt
|
||||
electron-builder.yml
|
||||
backup
|
||||
env
|
||||
docs
|
||||
*.log
|
||||
.env*
|
||||
44
.github/workflows/release.yml
vendored
44
.github/workflows/release.yml
vendored
@ -45,8 +45,41 @@ jobs:
|
||||
dist/*.zip
|
||||
retention-days: 30
|
||||
|
||||
build-macos:
|
||||
runs-on: macos-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '24'
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Install dependencies
|
||||
run: yarn install --frozen-lockfile
|
||||
|
||||
- name: Build application
|
||||
run: yarn build
|
||||
|
||||
- name: Build macOS installer
|
||||
run: yarn dist:mac
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: macos-builds
|
||||
path: |
|
||||
dist/*.dmg
|
||||
dist/*.zip
|
||||
retention-days: 30
|
||||
|
||||
release:
|
||||
needs: build-windows
|
||||
needs: [build-windows, build-macos]
|
||||
runs-on: ubuntu-latest
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
|
||||
@ -60,6 +93,12 @@ jobs:
|
||||
name: windows-builds
|
||||
path: dist
|
||||
|
||||
- name: Download macOS artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: macos-builds
|
||||
path: dist
|
||||
|
||||
- name: Create Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
@ -70,5 +109,6 @@ jobs:
|
||||
files: |
|
||||
dist/*.exe
|
||||
dist/*.zip
|
||||
dist/*.dmg
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,3 +1,4 @@
|
||||
.devtools
|
||||
# dependencies (bun install)
|
||||
node_modules
|
||||
|
||||
|
||||
94
NOTICES.txt
94
NOTICES.txt
@ -1,24 +1,42 @@
|
||||
Name: @aigne/core
|
||||
License: Elastic-2.0
|
||||
Repository: https://github.com/AIGNE-io/aigne-framework
|
||||
Name: @ai-sdk/anthropic
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: @aigne/openai
|
||||
License: Elastic-2.0
|
||||
Repository: https://github.com/AIGNE-io/aigne-framework
|
||||
Name: @ai-sdk/deepseek
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: @langchain/core
|
||||
License: MIT
|
||||
Repository: https://github.com/langchain-ai/langchainjs
|
||||
Name: @ai-sdk/devtools
|
||||
License: MIT*
|
||||
Repository: N/A
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: @langchain/openai
|
||||
License: MIT
|
||||
Repository: https://github.com/langchain-ai/langchainjs
|
||||
Name: @ai-sdk/google
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: @ai-sdk/openai-compatible
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: @ai-sdk/openai
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: @ai-sdk/xai
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
@ -64,6 +82,12 @@ Repository: https://github.com/DefinitelyTyped/DefinitelyTyped
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: ai
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/vercel/ai
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: axios-retry
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/softonic/axios-retry
|
||||
@ -76,6 +100,12 @@ Repository: https://github.com/axios/axios
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: best-effort-json-parser
|
||||
License: BSD-2-Clause
|
||||
Repository: https://github.com/beenotung/best-effort-json-parser
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: better-sqlite3
|
||||
License: MIT
|
||||
Repository: https://github.com/WiseLibs/better-sqlite3
|
||||
@ -88,6 +118,12 @@ Repository: https://github.com/expressjs/cors
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: cross-env
|
||||
License: MIT
|
||||
Repository: https://github.com/kentcdodds/cross-env
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: dotenv
|
||||
License: BSD-2-Clause
|
||||
Repository: https://github.com/motdotla/dotenv
|
||||
@ -166,12 +202,6 @@ Repository: https://github.com/knex/knex
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: langchain
|
||||
License: MIT
|
||||
Repository: https://github.com/langchain-ai/langchainjs
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: license-checker
|
||||
License: BSD-3-Clause
|
||||
Repository: https://github.com/davglass/license-checker
|
||||
@ -190,6 +220,24 @@ Repository: https://github.com/remy/nodemon
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: qwen-ai-provider
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/Younis-Ahmed/qwen-ai-provider
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: serialize-error
|
||||
License: MIT
|
||||
Repository: https://github.com/sindresorhus/serialize-error
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: serialize-error
|
||||
License: MIT
|
||||
Repository: https://github.com/sindresorhus/serialize-error
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: sharp
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/lovell/sharp
|
||||
@ -214,9 +262,15 @@ Repository: https://github.com/microsoft/TypeScript
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: zod
|
||||
Name: uuid
|
||||
License: MIT
|
||||
Repository: https://github.com/colinhacks/zod
|
||||
Repository: https://github.com/uuidjs/uuid
|
||||
|
||||
-----------------------------
|
||||
|
||||
Name: zhipu-ai-provider
|
||||
License: Apache-2.0
|
||||
Repository: https://github.com/Xiang-CH/zhipu-ai-provider
|
||||
|
||||
-----------------------------
|
||||
|
||||
|
||||
164
README.md
164
README.md
@ -64,7 +64,7 @@ Toonflow 是一款 AI 工具，能够利用 AI 技术将小说自动转化为剧
|
||||
|
||||
- 短视频内容创作
|
||||
- 小说影视化实验
|
||||
- AI 文学 Adaptation 工具（改编工具）
|
||||
- AI 文学改编工具
|
||||
- 剧本开发与快速原型
|
||||
- 视频素材生成
|
||||
|
||||
@ -92,7 +92,7 @@ https://www.bilibili.com/video/BV1na6wB6Ea2
|
||||
|
||||
在安装和使用本软件之前，请准备以下内容：
|
||||
|
||||
- ✅ 大语言模型 AI 服务接口地址。
|
||||
- ✅ 大语言模型 AI 服务接口地址
|
||||
- ✅ Sora 或豆包视频服务接口地址
|
||||
- ✅ Nano Banana Pro 图片生成模型服务接口
|
||||
|
||||
@ -100,17 +100,17 @@ https://www.bilibili.com/video/BV1na6wB6Ea2
|
||||
|
||||
### 1. 下载与安装
|
||||
|
||||
| 操作系统 | GitHub 下载 | 123云盘下载 | 说明 |
|
||||
| :------: | :----------------------------------------------------------- | :-------------------------------------------------------- | :----------------------------------------------------------- |
|
||||
| Windows | [Release](https://github.com/HBAI-Ltd/Toonflow-app/releases) | [123云盘](https://www.123865.com/s/bkn5Vv-E67cv) | 官方发布安装包 |
|
||||
| Linux | ⚙️ 敬请期待 | ⚙️ 敬请期待 | 即将发布 |
|
||||
| macOS | ⚙️ 敬请期待 | ⚙️ 敬请期待 | 即将发布 |
|
||||
| 操作系统 | GitHub 下载 | 123 云盘下载 | 说明 |
|
||||
| :------: | :----------------------------------------------------------- | :------------------------------------------------ | :------------- |
|
||||
| Windows | [Release](https://github.com/HBAI-Ltd/Toonflow-app/releases) | [123 云盘](https://www.123865.com/s/bkn5Vv-E67cv) | 官方发布安装包 |
|
||||
| Linux | ⚙️ 敬请期待 | ⚙️ 敬请期待 | 即将发布 |
|
||||
| macOS | ⚙️ 敬请期待 | ⚙️ 敬请期待 | 即将发布 |
|
||||
|
||||
> ⚠️ 如123云盘提示需付费，仅因云盘流量受限，本软件完全开源免费。
|
||||
> ⚠️ 如 123 云盘提示需付费，仅因云盘流量受限，本软件完全开源免费。
|
||||
|
||||
> 目前仅支持 Windows 版本，其他系统将陆续开放。
|
||||
|
||||
> 因 Gitee OS 环境限制及Release文件上传大小限制，暂不提供 Gitee Release 下载地址。
|
||||
> 因 Gitee OS 环境限制及 Release 文件上传大小限制，暂不提供 Gitee Release 下载地址。
|
||||
|
||||
### 2. 启动服务
|
||||
|
||||
@ -120,13 +120,105 @@ https://www.bilibili.com/video/BV1na6wB6Ea2
|
||||
> 账号：`admin`
|
||||
> 密码：`admin123`
|
||||
|
||||
## Docker 部署
|
||||
|
||||
### 前置条件
|
||||
|
||||
- 已安装 [Docker](https://docs.docker.com/get-docker/)（版本 20.10+）
|
||||
- 已安装 [Docker Compose](https://docs.docker.com/compose/install/)（版本 2.0+）
|
||||
|
||||
### 方式一：在线部署（推荐）
|
||||
|
||||
从 GitHub / Gitee 自动拉取源码并构建镜像：
|
||||
|
||||
```shell
|
||||
docker-compose -f docker/docker-compose.yml up -d --build
|
||||
```
|
||||
|
||||
**支持的构建参数：**
|
||||
|
||||
| 参数 | 说明 | 默认值 | 示例 |
|
||||
| ---------- | ----------------------- | ---------- | --------------------------- |
|
||||
| `GIT` | 代码仓库源 | `github` | `github` / `gitee` |
|
||||
| `TAG` | 指定版本标签 | 最新 tag | `v1.0.6` |
|
||||
| `BRANCH` | 指定分支 | 默认分支 | `main` / `dev` |
|
||||
|
||||
**版本选择优先级**：指定 TAG > 指定 BRANCH > 自动获取最新 tag > 默认分支
|
||||
|
||||
**指定参数示例：**
|
||||
|
||||
```shell
|
||||
# 使用 Gitee 源（国内推荐，速度更快）
|
||||
GIT=gitee docker-compose -f docker/docker-compose.yml up -d --build
|
||||
|
||||
# 指定版本标签
|
||||
TAG=v1.0.6 docker-compose -f docker/docker-compose.yml up -d --build
|
||||
|
||||
# 指定分支 + Gitee 源
|
||||
GIT=gitee BRANCH=dev docker-compose -f docker/docker-compose.yml up -d --build
|
||||
```
|
||||
|
||||
### 方式二：本地构建
|
||||
|
||||
使用本地已有的源码直接构建，适合开发者或已克隆仓库的用户：
|
||||
|
||||
```shell
|
||||
# 先克隆项目（如已有则跳过）
|
||||
git clone https://github.com/HBAI-Ltd/Toonflow-app.git
|
||||
cd Toonflow-app
|
||||
|
||||
# 使用本地源码构建
|
||||
docker-compose -f docker/docker-compose.local.yml up -d --build
|
||||
```
|
||||
|
||||
### 服务端口说明
|
||||
|
||||
| 端口 | 用途 | 在线部署映射 | 本地构建映射 |
|
||||
| ------- | ----------------------- | --------------- | ----------------- |
|
||||
| `80` | Nginx 前端页面 | 随机端口 | `8080:80` |
|
||||
| `60000` | 后端 API 服务 | `60000:60000` | `60000:60000` |
|
||||
|
||||
### 数据持久化
|
||||
|
||||
默认日志目录会挂载到宿主机 `./logs` 目录。如需持久化上传文件或数据库，可在 `docker-compose.yml` 中添加 volumes：
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ./logs:/var/log
|
||||
- ./uploads:/app/uploads # 持久化上传文件
|
||||
- ./data:/app/data # 持久化数据库（如有）
|
||||
```
|
||||
|
||||
### 常用操作命令
|
||||
|
||||
```shell
|
||||
# 查看容器状态
|
||||
docker-compose -f docker/docker-compose.yml ps
|
||||
|
||||
# 查看实时日志
|
||||
docker-compose -f docker/docker-compose.yml logs -f
|
||||
|
||||
# 停止服务
|
||||
docker-compose -f docker/docker-compose.yml down
|
||||
|
||||
# 重新构建并启动（更新版本时使用）
|
||||
docker-compose -f docker/docker-compose.yml up -d --build
|
||||
|
||||
# 进入容器调试
|
||||
docker exec -it toonflow sh
|
||||
```
|
||||
|
||||
> ⚠️ **首次登录**
|
||||
> 账号：`admin`
|
||||
> 密码：`admin123`
|
||||
|
||||
## 云端部署
|
||||
|
||||
### 一、服务器环境要求
|
||||
|
||||
- **系统**：Ubuntu 20.04+ / CentOS 7+
|
||||
- **Node.js**：23.11.1+
|
||||
- **内存**：1GB+
|
||||
- **Node.js**：24.x（推荐，最低 23.11.1+）
|
||||
- **内存**：2GB+
|
||||
|
||||
### 二、服务器部署
|
||||
|
||||
@ -208,6 +300,10 @@ pm2 restart all # 重启服务
|
||||
pm2 monit # 监控面板
|
||||
```
|
||||
|
||||
> ⚠️ **首次登录**
|
||||
> 账号：`admin`
|
||||
> 密码：`admin123`
|
||||
|
||||
#### 6. 部署前端网站
|
||||
|
||||
如需单独部署或定制前端界面，请参考前端仓库：
|
||||
@ -254,12 +350,31 @@ pm2 monit # 监控面板
|
||||
|
||||
3. **启动开发环境**
|
||||
|
||||
- 使用 Node.js 运行开发服务：
|
||||
本项目包含 **后端 API 服务** 和 **前端页面** 两部分，请根据需要选择启动方式：
|
||||
|
||||
- **方式一：仅启动后端服务（开发调试用）**
|
||||
|
||||
```bash
|
||||
yarn dev #端口60000
|
||||
yarn dev
|
||||
```
|
||||
|
||||
> ⚠️ 此命令仅启动后端 API 服务（端口 60000），**不包含前端页面**。直接访问 `http://localhost:60000` 只能调用 API 接口，无法看到完整的网页界面。如需同时使用前端页面，请配合前端项目单独启动，或使用下方的 GUI 模式。
|
||||
|
||||
- **方式二：启动 Electron 桌面客户端（推荐完整体验）**
|
||||
|
||||
```bash
|
||||
yarn dev:gui
|
||||
```
|
||||
|
||||
> 此命令会同时启动后端服务和 Electron 桌面窗口，自带内置前端页面，开箱即用，无需额外配置。适合想要完整体验所有功能的开发者。
|
||||
|
||||
**两种模式对比：**
|
||||
|
||||
| 命令 | 启动内容 | 前端页面 | 适用场景 |
|
||||
| -------------- | ---------------------- | -------- | -------------------------------- |
|
||||
| `yarn dev` | 仅后端 API（端口 60000） | ❌ 无 | 后端开发调试、配合前端项目联调 |
|
||||
| `yarn dev:gui` | 后端 + Electron 桌面端 | ✅ 内置 | 完整功能体验、桌面客户端开发调试 |
|
||||
|
||||
4. **项目打包**
|
||||
|
||||
- 编译并生成 TypeScript 文件：
|
||||
@ -273,6 +388,18 @@ pm2 monit # 监控面板
|
||||
```bash
|
||||
yarn dist:win
|
||||
```
|
||||
|
||||
- 打包为 Mac 平台可执行程序：
|
||||
|
||||
```bash
|
||||
yarn dist:mac
|
||||
```
|
||||
|
||||
- 打包为 Linux 平台可执行程序：
|
||||
|
||||
```bash
|
||||
yarn dist:linux
|
||||
```
|
||||
|
||||
5. **代码质量检查**
|
||||
|
||||
@ -282,6 +409,14 @@ pm2 monit # 监控面板
|
||||
yarn lint
|
||||
```
|
||||
|
||||
6. **AI 调试面板（可选）**
|
||||
|
||||
启动 AI SDK 的可视化调试工具，方便调试 AI 调用：
|
||||
|
||||
```bash
|
||||
yarn debug:ai
|
||||
```
|
||||
|
||||
## 前端开发
|
||||
|
||||
如需修改前端界面，请前往前端仓库进行开发：
|
||||
@ -294,6 +429,7 @@ pm2 monit # 监控面板
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
📂 docker/ # Docker 配置文件
|
||||
📂 docs/ # 文档资源
|
||||
📂 scripts/ # 构建脚本与静态资源
|
||||
│ └─ 📂 web/ # 前端编译产物（内置）
|
||||
@ -415,7 +551,7 @@ Toonflow 基于 AGPL-3.0 协议开源发布，许可证详情：https://www.gnu.
|
||||
感谢以下开源项目为 Toonflow 提供强大支持：
|
||||
|
||||
- [Express](https://expressjs.com/) - 快速、开放、极简的 Node.js Web 框架
|
||||
- [LangChain](https://js.langchain.com/) - 构建 LLM 应用的开发框架
|
||||
- [AI](https://ai-sdk.dev/) - 面向 TypeScript 的 AI 工具包
|
||||
- [Better-SQLite3](https://github.com/WiseLibs/better-sqlite3) - 高性能 SQLite3 绑定库
|
||||
- [Sharp](https://sharp.pixelplumbing.com/) - 高性能 Node.js 图像处理库
|
||||
- [Axios](https://axios-http.com/) - 基于 Promise 的 HTTP 客户端
|
||||
|
||||
769
backup/agents/outlineScript/index.ts
Normal file
769
backup/agents/outlineScript/index.ts
Normal file
@ -0,0 +1,769 @@
|
||||
// @/agents/outlineScript.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
type AgentType = "AI1" | "AI2" | "director";
|
||||
type AssetType = "角色" | "道具" | "场景";
|
||||
type RefreshEvent = "storyline" | "outline" | "assets";
|
||||
|
||||
interface AssetItem {
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface EpisodeData {
|
||||
episodeIndex: number;
|
||||
title: string;
|
||||
chapterRange: number[];
|
||||
scenes: AssetItem[]; // 按 outline 出场顺序排列
|
||||
characters: AssetItem[]; // 按 outline 出场顺序排列
|
||||
props: AssetItem[]; // 按 outline 出场顺序排列
|
||||
coreConflict: string;
|
||||
outline: string; // 最高优先级，剧本生成的唯一权威
|
||||
openingHook: string; // outline 第一句话的视觉化，开篇第一个镜头
|
||||
keyEvents: string[]; // 4个元素：[起, 承, 转, 合]，严格按 outline 顺序
|
||||
emotionalCurve: string; // 对应 keyEvents 各阶段
|
||||
visualHighlights: string[]; // 按 outline 顺序排列的标志性镜头
|
||||
endingHook: string; // outline 之后的悬念延伸
|
||||
classicQuotes: string[];
|
||||
}
|
||||
|
||||
// ==================== Schema 定义 ====================
|
||||
|
||||
const sceneItemSchema = z.object({
|
||||
name: z.string().describe("场景名称，如'五星酒店宴会厅'、'老旧出租屋'"),
|
||||
description: z.string().describe("环境描写：空间结构、光线氛围、装饰陈设、环境细节"),
|
||||
});
|
||||
|
||||
const characterItemSchema = z.object({
|
||||
name: z.string().describe("角色姓名（必须是具体人名，禁止'众人'、'群众'等集合描述）"),
|
||||
description: z.string().describe("人设样貌：年龄体态、五官特征、发型妆容、服装配饰、气质神态"),
|
||||
});
|
||||
|
||||
const propItemSchema = z.object({
|
||||
name: z.string().describe("道具名称"),
|
||||
description: z.string().describe("样式描写：材质质感、颜色图案、形状尺寸、磨损痕迹、特殊标记"),
|
||||
});
|
||||
|
||||
const episodeSchema = z.object({
|
||||
episodeIndex: z.number().describe("集数索引，从1开始递增"),
|
||||
title: z.string().describe("8字内标题，疑问/感叹句，含情绪爆点"),
|
||||
chapterRange: z.array(z.number()).describe("关联章节号数组"),
|
||||
scenes: z.array(sceneItemSchema).describe("场景列表，按 outline 出场顺序排列"),
|
||||
characters: z.array(characterItemSchema).describe("角色列表，按 outline 出场顺序排列，必须是独立个体"),
|
||||
props: z.array(propItemSchema).describe("道具列表，按 outline 出场顺序排列，至少3个"),
|
||||
coreConflict: z.string().describe("核心矛盾：A想要X vs B阻碍X"),
|
||||
outline: z.string().describe("100-300字剧情主干，最高优先级，剧本生成的唯一权威，按时间顺序完整叙述"),
|
||||
openingHook: z.string().describe("开场镜头：outline 第一句话的视觉化，必须作为剧本第一个镜头"),
|
||||
keyEvents: z.array(z.string()).length(4).describe("4个元素的数组：[起, 承, 转, 合]，严格按 outline 顺序从中提取"),
|
||||
emotionalCurve: z.string().describe("情绪曲线，如：2(压抑)→5(反抗)→9(爆发)→3(余波)，对应 keyEvents 各阶段"),
|
||||
visualHighlights: z.array(z.string()).describe("3-5个标志性镜头，按 outline 叙事顺序排列"),
|
||||
endingHook: z.string().describe("结尾悬念：outline 之后的延伸，勾引下集"),
|
||||
classicQuotes: z.array(z.string()).describe("1-2句金句，每句≤15字，必须从原文提取"),
|
||||
});
|
||||
|
||||
// ==================== 常量配置 ====================
|
||||
|
||||
// ==================== 主类 ====================
|
||||
|
||||
export default class OutlineScript {
|
||||
private readonly projectId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number) {
|
||||
this.projectId = projectId;
|
||||
}
|
||||
|
||||
// ==================== 公共方法 ====================
|
||||
|
||||
get events() {
|
||||
return this.emitter;
|
||||
}
|
||||
|
||||
setNovel(chapters: DB["t_novel"][]) {
|
||||
this.novelChapters = chapters;
|
||||
}
|
||||
|
||||
// ==================== 私有工具方法 ====================
|
||||
|
||||
private emit(event: string, data?: any) {
|
||||
this.emitter.emit(event, data);
|
||||
}
|
||||
|
||||
private refresh(type: RefreshEvent) {
|
||||
this.emit("refresh", type);
|
||||
}
|
||||
|
||||
private log(action: string, detail?: string) {
|
||||
const msg = detail ? `${action}: ${detail}` : action;
|
||||
console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`);
|
||||
}
|
||||
|
||||
private safeParseJson<T>(str: string, fallback: T): T {
|
||||
try {
|
||||
return JSON.parse(str);
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private uniqueByName<T extends { name: string }>(items: T[]): T[] {
|
||||
return Array.from(new Map(items.map((item) => [item.name, item])).values());
|
||||
}
|
||||
|
||||
// ==================== 数据库操作 ====================
|
||||
|
||||
private async getProjectInfo(): Promise<any> {
|
||||
return u.db("t_project").where({ id: this.projectId }).first();
|
||||
}
|
||||
|
||||
private async getNovelInfo(asString = false): Promise<any> {
|
||||
const info = await this.getProjectInfo();
|
||||
if (!info) return asString ? "未查询到项目信息" : null;
|
||||
|
||||
if (asString) {
|
||||
const fields = [
|
||||
`小说名称: ${info.name}`,
|
||||
`小说简介: ${info.intro}`,
|
||||
`小说类型: ${info.type}`,
|
||||
`目标短剧类型: ${info.artStyle}`,
|
||||
`短剧画幅: ${info.videoRatio}`,
|
||||
];
|
||||
return fields.join("\n");
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
// ==================== 故事线操作 ====================
|
||||
|
||||
private async findStoryline() {
|
||||
return u.db("t_storyline").where({ projectId: this.projectId }).first();
|
||||
}
|
||||
|
||||
private async upsertStorylineContent(content: string) {
|
||||
const existing = await this.findStoryline();
|
||||
if (existing) {
|
||||
await u.db("t_storyline").where({ projectId: this.projectId }).update({ content });
|
||||
} else {
|
||||
await u.db("t_storyline").insert({ projectId: this.projectId, content });
|
||||
}
|
||||
this.refresh("storyline");
|
||||
}
|
||||
|
||||
private async deleteStorylineContent() {
|
||||
const deleted = await u.db("t_storyline").where({ projectId: this.projectId }).del();
|
||||
this.refresh("storyline");
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// ==================== 大纲操作 ====================
|
||||
|
||||
private async findOutlines() {
|
||||
return u.db("t_outline").where({ projectId: this.projectId }).orderBy("episode", "asc");
|
||||
}
|
||||
|
||||
private async findOutlineById(id: number) {
|
||||
return u.db("t_outline").where({ id, projectId: this.projectId }).first();
|
||||
}
|
||||
|
||||
private async getMaxEpisode(): Promise<number> {
|
||||
const result: any = await u.db("t_outline").where({ projectId: this.projectId }).max("episode as max").first();
|
||||
return result?.max ?? 0;
|
||||
}
|
||||
|
||||
private async clearOutlinesAndScripts() {
|
||||
const outlines = await u.db("t_outline").select("id").where({ projectId: this.projectId });
|
||||
if (outlines.length === 0) return 0;
|
||||
|
||||
const outlineIds = outlines.map((o) => o.id);
|
||||
await u.db("t_script").whereIn("outlineId", outlineIds).del();
|
||||
await u.db("t_outline").where({ projectId: this.projectId }).del();
|
||||
|
||||
return outlines.length;
|
||||
}
|
||||
|
||||
private async insertOutlines(episodes: EpisodeData[], startEpisode: number) {
|
||||
const insertList = episodes.map((ep, idx) => ({
|
||||
projectId: this.projectId,
|
||||
data: JSON.stringify({ ...ep, episodeIndex: startEpisode + idx }),
|
||||
episode: startEpisode + idx,
|
||||
}));
|
||||
|
||||
await u.db("t_outline").insert(insertList);
|
||||
return insertList.length;
|
||||
}
|
||||
|
||||
private async createEmptyScripts(outlineIds: Array<{ id: number; data: string }>) {
|
||||
const scripts = outlineIds.map((item) => {
|
||||
const data = this.safeParseJson<Partial<EpisodeData>>(item.data, {});
|
||||
return {
|
||||
name: `第${data.episodeIndex ?? ""}集`,
|
||||
content: "",
|
||||
projectId: this.projectId,
|
||||
outlineId: item.id,
|
||||
};
|
||||
});
|
||||
|
||||
if (scripts.length > 0) {
|
||||
await u.db("t_script").insert(scripts);
|
||||
}
|
||||
return scripts.length;
|
||||
}
|
||||
|
||||
private async saveOutlineData(episodes: EpisodeData[], overwrite: boolean, startEpisode?: number) {
|
||||
if (overwrite) {
|
||||
const cleared = await this.clearOutlinesAndScripts();
|
||||
if (cleared > 0) {
|
||||
this.log("清理旧数据", `删除了 ${cleared} 条大纲及关联剧本`);
|
||||
}
|
||||
}
|
||||
|
||||
const actualStart = overwrite ? 1 : startEpisode ?? (await this.getMaxEpisode()) + 1;
|
||||
const insertedCount = await this.insertOutlines(episodes, actualStart);
|
||||
|
||||
const newOutlines = await u
|
||||
.db("t_outline")
|
||||
.select("id", "data")
|
||||
.where({ projectId: this.projectId })
|
||||
.orderBy("episode", "desc")
|
||||
.limit(insertedCount);
|
||||
|
||||
const scriptCount = await this.createEmptyScripts(newOutlines as Array<{ id: number; data: string }>);
|
||||
|
||||
this.refresh("outline");
|
||||
return { insertedCount, scriptCount };
|
||||
}
|
||||
|
||||
private async updateOutlineData(id: number, data: EpisodeData) {
|
||||
const existing = await this.findOutlineById(id);
|
||||
if (!existing) return false;
|
||||
|
||||
await u
|
||||
.db("t_outline")
|
||||
.where({ id })
|
||||
.update({ data: JSON.stringify(data) });
|
||||
this.refresh("outline");
|
||||
return true;
|
||||
}
|
||||
|
||||
private async deleteOutlineData(ids: number[]) {
|
||||
const results = await Promise.allSettled(ids.map((id) => u.deleteOutline(id, this.projectId)));
|
||||
this.refresh("outline");
|
||||
return results;
|
||||
}
|
||||
|
||||
private formatOutlineDetail(ep: any): string {
|
||||
const formatList = (items: any[], formatter: (item: any) => string) =>
|
||||
items?.map((item, i) => ` ${i + 1}. ${formatter(item)}`).join("\n") || " 无";
|
||||
|
||||
// keyEvents 按顺序显示：起、承、转、合
|
||||
const keyEventsLabels = ["起", "承", "转", "合"];
|
||||
const formatKeyEvents = (events: string[]) => events?.map((e, i) => ` 【${keyEventsLabels[i] || i + 1}】${e}`).join("\n") || " 无";
|
||||
|
||||
return `
|
||||
大纲ID: ${ep.id}
|
||||
第 ${ep.episodeIndex} 集: ${ep.title || ""}
|
||||
${"=".repeat(50)}
|
||||
章节范围: ${ep.chapterRange?.join(", ") || ""}
|
||||
核心矛盾: ${ep.coreConflict || ""}
|
||||
|
||||
【剧情主干】(最高优先级，剧本生成的唯一权威):
|
||||
${ep.outline || "无"}
|
||||
|
||||
【开场镜头】(必须作为剧本第一个镜头):
|
||||
${ep.openingHook || "无"}
|
||||
|
||||
【剧情节点】(严格按顺序：起→承→转→合):
|
||||
${formatKeyEvents(ep.keyEvents)}
|
||||
|
||||
情绪曲线: ${ep.emotionalCurve || ""}
|
||||
|
||||
【视觉重点】(按剧情主干顺序排列):
|
||||
${formatList(ep.visualHighlights, (v) => v)}
|
||||
|
||||
【结尾悬念】:
|
||||
${ep.endingHook || "无"}
|
||||
|
||||
【经典台词】:
|
||||
${formatList(ep.classicQuotes, (q) => q)}
|
||||
|
||||
角色(按出场顺序): ${ep.characters?.map((c: AssetItem) => `${c.name}(${c.description})`).join("; ") || "无"}
|
||||
场景(按出场顺序): ${ep.scenes?.map((s: AssetItem) => `${s.name}(${s.description})`).join("; ") || "无"}
|
||||
道具(按出场顺序): ${ep.props?.map((p: AssetItem) => `${p.name}(${p.description})`).join("; ") || "无"}`;
|
||||
}
|
||||
|
||||
private async getOutlineText(simplified: boolean): Promise<string> {
|
||||
const records = await this.findOutlines();
|
||||
|
||||
if (!records.length) return "当前项目暂无大纲";
|
||||
|
||||
const episodes = records.map((r) => ({
|
||||
id: r.id,
|
||||
episode: r.episode,
|
||||
...this.safeParseJson<Partial<EpisodeData>>(r.data ?? "{}", {}),
|
||||
}));
|
||||
|
||||
if (simplified) {
|
||||
const list = episodes.map((ep) => `第 ${ep.episodeIndex ?? ep.episode} 集 (id=${ep.id})`).join("\n");
|
||||
return `项目大纲 (共 ${episodes.length} 集):\n${list}`;
|
||||
}
|
||||
|
||||
const details = episodes.map((ep) => this.formatOutlineDetail(ep)).join("\n");
|
||||
return `项目大纲 (共 ${episodes.length} 集)\n\n${details}`;
|
||||
}
|
||||
|
||||
// ==================== 资产操作 ====================
|
||||
|
||||
private async findAssetByTypeAndName(type: AssetType, name: string) {
|
||||
return u.db("t_assets").where({ projectId: this.projectId, type, name }).first();
|
||||
}
|
||||
|
||||
private async upsertAsset(type: AssetType, item: AssetItem): Promise<"inserted" | "updated" | "skipped"> {
|
||||
const existing = await this.findAssetByTypeAndName(type, item.name);
|
||||
|
||||
if (!existing) {
|
||||
await u.db("t_assets").insert({
|
||||
projectId: this.projectId,
|
||||
type,
|
||||
name: item.name,
|
||||
intro: item.description,
|
||||
prompt: item.description,
|
||||
});
|
||||
return "inserted";
|
||||
}
|
||||
|
||||
if (existing.intro !== item.description) {
|
||||
await u.db("t_assets").where({ id: existing.id }).update({
|
||||
intro: item.description,
|
||||
prompt: item.description,
|
||||
});
|
||||
return "updated";
|
||||
}
|
||||
|
||||
return "skipped";
|
||||
}
|
||||
|
||||
private extractAssetsFromOutlines(outlines: Array<{ data?: string | null | undefined }>): {
|
||||
characters: AssetItem[];
|
||||
props: AssetItem[];
|
||||
scenes: AssetItem[];
|
||||
} {
|
||||
const result = { characters: [] as AssetItem[], props: [] as AssetItem[], scenes: [] as AssetItem[] };
|
||||
|
||||
for (const outline of outlines) {
|
||||
const data = this.safeParseJson<Partial<EpisodeData>>(outline.data ?? "{}", {});
|
||||
if (data.characters) result.characters.push(...data.characters);
|
||||
if (data.props) result.props.push(...data.props);
|
||||
if (data.scenes) result.scenes.push(...data.scenes);
|
||||
}
|
||||
|
||||
return {
|
||||
characters: this.uniqueByName(result.characters),
|
||||
props: this.uniqueByName(result.props),
|
||||
scenes: this.uniqueByName(result.scenes),
|
||||
};
|
||||
}
|
||||
|
||||
private async generateAssetsFromOutlines() {
|
||||
const outlines = await u.db("t_outline").select("data").where({ projectId: this.projectId });
|
||||
|
||||
if (!outlines.length) return { inserted: 0, updated: 0, skipped: 0 };
|
||||
|
||||
const { characters, props, scenes } = this.extractAssetsFromOutlines(outlines);
|
||||
|
||||
// 只做新增和更新，不做删除
|
||||
const stats = { inserted: 0, updated: 0, skipped: 0 };
|
||||
|
||||
const processItems = async (items: AssetItem[], type: AssetType) => {
|
||||
for (const item of items) {
|
||||
const result = await this.upsertAsset(type, item);
|
||||
stats[result]++;
|
||||
}
|
||||
};
|
||||
|
||||
await processItems(characters, "角色");
|
||||
await processItems(props, "道具");
|
||||
await processItems(scenes, "场景");
|
||||
|
||||
this.refresh("assets");
|
||||
return { ...stats };
|
||||
}
|
||||
|
||||
// ==================== Tool 定义：故事线 ====================
|
||||
|
||||
getStoryline = tool(
|
||||
async () => {
|
||||
this.log("获取故事线");
|
||||
const storyline = await this.findStoryline();
|
||||
return storyline?.content ?? "当前项目暂无故事线";
|
||||
},
|
||||
{
|
||||
name: "getStoryline",
|
||||
description: "获取当前项目的故事线内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
saveStoryline = tool(
|
||||
async ({ content }) => {
|
||||
this.log("保存故事线");
|
||||
await this.upsertStorylineContent(content);
|
||||
return "故事线保存成功";
|
||||
},
|
||||
{
|
||||
name: "saveStoryline",
|
||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||
schema: z.object({
|
||||
content: z.string().describe("故事线完整内容"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
deleteStoryline = tool(
|
||||
async () => {
|
||||
this.log("删除故事线");
|
||||
const deleted = await this.deleteStorylineContent();
|
||||
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
||||
},
|
||||
{
|
||||
name: "deleteStoryline",
|
||||
description: "删除当前项目的故事线",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== Tool 定义：大纲 ====================
|
||||
|
||||
getOutline = tool(
|
||||
async ({ simplified = false }) => {
|
||||
this.log("获取大纲", `简化模式: ${simplified}`);
|
||||
return this.getOutlineText(simplified);
|
||||
},
|
||||
{
|
||||
name: "getOutline",
|
||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||
schema: z.object({
|
||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
saveOutline = tool(
|
||||
async ({ episodes, overwrite = true, startEpisode }) => {
|
||||
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||
},
|
||||
{
|
||||
name: "saveOutline",
|
||||
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||
schema: z.object({
|
||||
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
updateOutline = tool(
|
||||
async ({ id, data }) => {
|
||||
this.log("更新大纲", `ID: ${id}`);
|
||||
const success = await this.updateOutlineData(id, data as EpisodeData);
|
||||
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
||||
},
|
||||
{
|
||||
name: "updateOutline",
|
||||
description: "更新指定ID的单集大纲内容",
|
||||
schema: z.object({
|
||||
id: z.number().describe("大纲ID"),
|
||||
data: episodeSchema.describe("更新后的大纲数据"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
deleteOutline = tool(
|
||||
async ({ ids }) => {
|
||||
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
||||
const results = await this.deleteOutlineData(ids);
|
||||
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
||||
return `删除结果: ${summary}`;
|
||||
},
|
||||
{
|
||||
name: "deleteOutline",
|
||||
description: "根据大纲ID删除指定大纲及关联数据",
|
||||
schema: z.object({
|
||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== Tool 定义：章节 ====================
|
||||
|
||||
getChapter = tool(
|
||||
async ({ chapterNumbers }) => {
|
||||
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
||||
|
||||
const results = await Promise.all(
|
||||
chapterNumbers.map(async (num) => {
|
||||
const chapter = await u
|
||||
.db("t_novel")
|
||||
.where({ projectId: this.projectId, chapterIndex: num })
|
||||
.select("chapterData", "chapterIndex", "chapter")
|
||||
.first();
|
||||
|
||||
if (chapter) {
|
||||
return `\n【第${chapter.chapterIndex}章 ${chapter.chapter || ""}】\n${chapter.chapterData}`;
|
||||
}
|
||||
return `\n【第${num}章】未找到`;
|
||||
}),
|
||||
);
|
||||
|
||||
return results.join("\n\n---\n");
|
||||
},
|
||||
{
|
||||
name: "getChapter",
|
||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||
schema: z.object({
|
||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== Tool 定义：资产 ====================
|
||||
|
||||
generateAssets = tool(
|
||||
async () => {
|
||||
this.log("生成资产");
|
||||
const stats = await this.generateAssetsFromOutlines();
|
||||
|
||||
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
||||
return "当前项目没有大纲数据，无法生成资产";
|
||||
}
|
||||
|
||||
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
||||
},
|
||||
{
|
||||
name: "generateAssets",
|
||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== 上下文构建 ====================
|
||||
|
||||
private getChapterContext(): string {
|
||||
if (!this.novelChapters.length) return "无章节数据";
|
||||
return this.novelChapters.map((c) => `章节号:${c.chapterIndex}，分卷:${c.reel}，章节名:${c.chapter}`).join("\n");
|
||||
}
|
||||
|
||||
private async buildEnvironmentContext(): Promise<string> {
|
||||
const [novelInfo, storyline, outlineCount] = await Promise.all([
|
||||
this.getNovelInfo(true),
|
||||
this.findStoryline(),
|
||||
u.db("t_outline").where({ projectId: this.projectId }).count("id as count").first() as any,
|
||||
]);
|
||||
|
||||
return `<环境信息>
|
||||
项目ID: ${this.projectId}
|
||||
系统时间: ${new Date().toLocaleString()}
|
||||
|
||||
${novelInfo}
|
||||
|
||||
已加载章节列表:
|
||||
${this.getChapterContext()}
|
||||
|
||||
故事线状态: ${storyline ? "已生成" : "未生成"}
|
||||
大纲状态: 共 ${outlineCount?.count ?? 0} 集
|
||||
|
||||
可用工具:
|
||||
- getChapter: 获取章节原文
|
||||
- getStoryline/saveStoryline/deleteStoryline: 故事线操作
|
||||
- getOutline/saveOutline/updateOutline/deleteOutline: 大纲操作
|
||||
- generateAssets: 从大纲生成资产
|
||||
</环境信息>`;
|
||||
}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
const env = await this.buildEnvironmentContext();
|
||||
const history = this.buildConversationHistory();
|
||||
|
||||
return `${env}
|
||||
|
||||
<对话历史>
|
||||
${history}
|
||||
</对话历史>
|
||||
|
||||
<当前任务>
|
||||
${task}
|
||||
</当前任务>`;
|
||||
}
|
||||
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private getSubAgentTools() {
|
||||
return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline];
|
||||
}
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用 Sub-Agent（流式传输）
|
||||
*/
|
||||
private async invokeSubAgent(agentType: AgentType, task: string): Promise<string> {
|
||||
this.emit("transfer", { to: agentType });
|
||||
this.log(`Sub-Agent 调用`, agentType);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]);
|
||||
const a1Prompt = promptsList.find((p) => p.code === "outlineScript-a1");
|
||||
const a2Prompt = promptsList.find((p) => p.code === "outlineScript-a2");
|
||||
const directorPrompt = promptsList.find((p) => p.code === "outlineScript-director");
|
||||
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
AI1: a1Prompt?.customValue || a1Prompt?.defaultValue || errPrompts,
|
||||
AI2: a2Prompt?.customValue || a2Prompt?.defaultValue || errPrompts,
|
||||
director: directorPrompt?.customValue || directorPrompt?.defaultValue || errPrompts,
|
||||
};
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(),
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
|
||||
return fullResponse ?? `${agentType}已完成任务`;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||
this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||
this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getChapter,
|
||||
this.getStoryline,
|
||||
this.saveStoryline,
|
||||
this.deleteStoryline,
|
||||
this.getOutline,
|
||||
this.saveOutline,
|
||||
this.updateOutline,
|
||||
this.deleteOutline,
|
||||
this.generateAssets,
|
||||
];
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
this.history.push(["user", msg]);
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first();
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
}
|
||||
}
|
||||
128
backup/agents/storyboard/generateImagePromptsTool.ts
Normal file
128
backup/agents/storyboard/generateImagePromptsTool.ts
Normal file
@ -0,0 +1,128 @@
|
||||
import u from "@/utils";
|
||||
|
||||
type AspectRatio = "16:9" | "9:16" | "21:9" | "1:1" | "4:3" | "3:4" | "3:2" | "2:3";
|
||||
|
||||
interface GridLayoutResult {
|
||||
cols: number;
|
||||
rows: number;
|
||||
totalCells: number;
|
||||
placeholderCount: number;
|
||||
}
|
||||
|
||||
interface GridPromptOptions {
|
||||
prompts: string[];
|
||||
style: string;
|
||||
aspectRatio: AspectRatio;
|
||||
assetsName: { name: string; intro: string }[];
|
||||
}
|
||||
|
||||
interface GridPromptResult {
|
||||
prompt: string;
|
||||
gridLayout: GridLayoutResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据prompts数量计算宫格布局
|
||||
*/
|
||||
function calculateGridLayout(count: number): GridLayoutResult {
|
||||
let cols: number;
|
||||
let rows: number;
|
||||
if (count <= 0) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 1) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 2) {
|
||||
cols = 2;
|
||||
rows = 1;
|
||||
} else if (count === 3) {
|
||||
cols = 3;
|
||||
rows = 1;
|
||||
} else if (count === 4) {
|
||||
cols = 2;
|
||||
rows = 2;
|
||||
} else if (count <= 9) {
|
||||
cols = 3;
|
||||
rows = 3;
|
||||
} else {
|
||||
cols = 3;
|
||||
rows = Math.ceil(count / 3);
|
||||
}
|
||||
const totalCells = cols * rows;
|
||||
const placeholderCount = totalCells - count;
|
||||
return { cols, rows, totalCells, placeholderCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取宽高比描述
|
||||
*/
|
||||
function getAspectRatioDescription(aspectRatio: AspectRatio): string {
|
||||
const descriptions: Record<AspectRatio, string> = {
|
||||
"16:9": "电影宽银幕",
|
||||
"9:16": "竖屏短剧",
|
||||
"21:9": "超宽银幕史诗感",
|
||||
"1:1": "方形构图",
|
||||
"4:3": "经典银幕",
|
||||
"3:4": "竖版经典",
|
||||
"3:2": "摄影标准",
|
||||
"2:3": "竖版摄影",
|
||||
};
|
||||
return descriptions[aspectRatio] || "标准比例";
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成电影级宫格分镜提示词
|
||||
*/
|
||||
async function generateGridPrompt(options: GridPromptOptions): Promise<GridPromptResult> {
|
||||
const { prompts, style, aspectRatio, assetsName } = options;
|
||||
const layout = calculateGridLayout(prompts.length);
|
||||
const aspectRatioDesc = getAspectRatioDescription(aspectRatio);
|
||||
|
||||
// 构建宫格位置描述
|
||||
const gridPositions: string[] = [];
|
||||
for (let i = 0; i < layout.totalCells; i++) {
|
||||
const row = Math.floor(i / layout.cols) + 1;
|
||||
const col = (i % layout.cols) + 1;
|
||||
if (i < prompts.length) {
|
||||
gridPositions.push(`[第${row}行第${col}列]: ${prompts[i]}`);
|
||||
} else {
|
||||
gridPositions.push(`[第${row}行第${col}列]: 纯黑图`);
|
||||
}
|
||||
}
|
||||
|
||||
// 构建资产说明
|
||||
const assetsSection =
|
||||
assetsName.length > 0
|
||||
? `\n【可用资产】\n${assetsName.map((a) => `- ${a.name}：${a.intro}`).join("\n")}\n\n⚠️ 必须使用完整资产名称，禁止简称或代词。`
|
||||
: "";
|
||||
|
||||
const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first();
|
||||
|
||||
const mainPrompts = promptsData?.customValue || promptsData?.defaultValue;
|
||||
const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`;
|
||||
|
||||
if (!mainPrompts) return { prompt: errData, gridLayout: layout };
|
||||
|
||||
const result = await u.ai.text.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: mainPrompts,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${
|
||||
layout.totalCells
|
||||
}格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
prompt: result?.text ?? errData,
|
||||
gridLayout: layout,
|
||||
};
|
||||
}
|
||||
|
||||
export default generateGridPrompt;
|
||||
334
backup/agents/storyboard/generateImageTool.ts
Normal file
334
backup/agents/storyboard/generateImageTool.ts
Normal file
@ -0,0 +1,334 @@
|
||||
import generateImagePromptsTool from "@/agents/storyboard/generateImagePromptsTool";
|
||||
import u from "@/utils";
|
||||
import sharp from "sharp";
|
||||
import { z } from "zod";
|
||||
|
||||
interface AssetItem {
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface EpisodeData {
|
||||
episodeIndex: number;
|
||||
title: string;
|
||||
chapterRange: number[];
|
||||
scenes: AssetItem[];
|
||||
characters: AssetItem[];
|
||||
props: AssetItem[];
|
||||
coreConflict: string;
|
||||
openingHook: string;
|
||||
outline: string;
|
||||
keyEvents: string[];
|
||||
emotionalCurve: string;
|
||||
visualHighlights: string[];
|
||||
endingHook: string;
|
||||
classicQuotes: string[];
|
||||
}
|
||||
|
||||
interface ImageInfo {
|
||||
name: string;
|
||||
type: string;
|
||||
filePath: string;
|
||||
}
|
||||
|
||||
interface ResourceItem {
|
||||
name: string;
|
||||
intro: string;
|
||||
}
|
||||
|
||||
// 资产过滤响应的 schema
|
||||
const filteredAssetsSchema = z.object({
|
||||
relevantAssets: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().describe("资产名称"),
|
||||
reason: z.string().describe("选择该资产的原因"),
|
||||
}),
|
||||
)
|
||||
.describe("与分镜内容相关的资产列表"),
|
||||
});
|
||||
|
||||
// 压缩图片直到不超过指定大小
|
||||
async function compressImage(buffer: Buffer, maxSizeBytes: number = 3 * 1024 * 1024): Promise<Buffer> {
|
||||
if (buffer.length <= maxSizeBytes) {
|
||||
return buffer;
|
||||
}
|
||||
let quality = 90;
|
||||
let compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||
while (compressedBuffer.length > maxSizeBytes && quality > 10) {
|
||||
quality -= 10;
|
||||
compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||
}
|
||||
if (compressedBuffer.length > maxSizeBytes) {
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
let scale = 0.9;
|
||||
while (compressedBuffer.length > maxSizeBytes && scale > 0.1) {
|
||||
const newWidth = Math.round((metadata.width || 1000) * scale);
|
||||
const newHeight = Math.round((metadata.height || 1000) * scale);
|
||||
compressedBuffer = await sharp(buffer)
|
||||
.resize(newWidth, newHeight, { fit: "inside" })
|
||||
.jpeg({ quality: Math.max(quality, 30) })
|
||||
.toBuffer();
|
||||
scale -= 0.1;
|
||||
}
|
||||
}
|
||||
return compressedBuffer;
|
||||
}
|
||||
|
||||
// 拼接多张图片为一张
|
||||
async function mergeImages(imagePaths: string[]): Promise<Buffer> {
|
||||
const imageBuffers = await Promise.all(imagePaths.map((path) => u.oss.getFile(path)));
|
||||
const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata()));
|
||||
const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0));
|
||||
const resizedImages = await Promise.all(
|
||||
imageBuffers.map(async (buffer, index) => {
|
||||
const metadata = imageMetadatas[index];
|
||||
const aspectRatio = (metadata.width || 1) / (metadata.height || 1);
|
||||
const newWidth = Math.round(maxHeight * aspectRatio);
|
||||
return {
|
||||
buffer: await sharp(buffer).resize(newWidth, maxHeight, { fit: "cover" }).toBuffer(),
|
||||
width: newWidth,
|
||||
};
|
||||
}),
|
||||
);
|
||||
let currentX = 0;
|
||||
const compositeInputs = resizedImages.map(({ buffer, width }) => {
|
||||
const input = {
|
||||
input: buffer,
|
||||
left: currentX,
|
||||
top: 0,
|
||||
};
|
||||
currentX += width;
|
||||
return input;
|
||||
});
|
||||
const mergedImage = await sharp({
|
||||
create: {
|
||||
width: currentX,
|
||||
height: maxHeight,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.composite(compositeInputs)
|
||||
.jpeg({ quality: 90 })
|
||||
.toBuffer();
|
||||
return compressImage(mergedImage);
|
||||
}
|
||||
|
||||
// 进一步压缩单张图片到指定大小
|
||||
async function compressToSize(buffer: Buffer, targetSize: number): Promise<Buffer> {
|
||||
if (buffer.length <= targetSize) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
let quality = 80;
|
||||
let scale = 1.0;
|
||||
let compressedBuffer = buffer;
|
||||
|
||||
// 先尝试降低质量
|
||||
while (compressedBuffer.length > targetSize && quality > 10) {
|
||||
compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||
quality -= 10;
|
||||
}
|
||||
|
||||
// 如果还是太大，缩小尺寸
|
||||
while (compressedBuffer.length > targetSize && scale > 0.2) {
|
||||
scale -= 0.1;
|
||||
const newWidth = Math.round((metadata.width || 1000) * scale);
|
||||
const newHeight = Math.round((metadata.height || 1000) * scale);
|
||||
compressedBuffer = await sharp(buffer)
|
||||
.resize(newWidth, newHeight, { fit: "inside" })
|
||||
.jpeg({ quality: Math.max(quality, 20) })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
return compressedBuffer;
|
||||
}
|
||||
|
||||
// 确保图片列表总大小不超过指定限制
|
||||
async function ensureTotalSizeLimit(buffers: Buffer[], maxTotalBytes: number = 10 * 1024 * 1024): Promise<Buffer[]> {
|
||||
let totalSize = buffers.reduce((sum, buf) => sum + buf.length, 0);
|
||||
|
||||
if (totalSize <= maxTotalBytes) {
|
||||
return buffers;
|
||||
}
|
||||
|
||||
// 计算每张图片的平均目标大小
|
||||
const avgTargetSize = Math.floor(maxTotalBytes / buffers.length);
|
||||
|
||||
// 按大小降序排列，优先压缩大图片
|
||||
const indexedBuffers = buffers.map((buf, idx) => ({ buf, idx, size: buf.length }));
|
||||
indexedBuffers.sort((a, b) => b.size - a.size);
|
||||
|
||||
const result = [...buffers];
|
||||
|
||||
for (const item of indexedBuffers) {
|
||||
totalSize = result.reduce((sum, buf) => sum + buf.length, 0);
|
||||
if (totalSize <= maxTotalBytes) {
|
||||
break;
|
||||
}
|
||||
|
||||
// 计算这张图片需要压缩到的目标大小
|
||||
const excessSize = totalSize - maxTotalBytes;
|
||||
const targetSize = Math.max(item.buf.length - excessSize, avgTargetSize, 100 * 1024); // 最小100KB
|
||||
|
||||
if (item.buf.length > targetSize) {
|
||||
result[item.idx] = await compressToSize(item.buf, targetSize);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// 处理图片列表，确保不超过10张且每张不超过3MB，总大小不超过10MB
|
||||
async function processImages(images: ImageInfo[]): Promise<Buffer[]> {
|
||||
const maxImages = 10;
|
||||
let processedBuffers: Buffer[];
|
||||
|
||||
if (images.length <= maxImages) {
|
||||
const buffers = await Promise.all(images.map((img) => u.oss.getFile(img.filePath)));
|
||||
processedBuffers = await Promise.all(buffers.map((buffer) => compressImage(buffer)));
|
||||
} else {
|
||||
const mergeStartIndex = maxImages - 1;
|
||||
const firstBuffers = await Promise.all(images.slice(0, mergeStartIndex).map((img) => u.oss.getFile(img.filePath)));
|
||||
const compressedFirstImages = await Promise.all(firstBuffers.map((buffer) => compressImage(buffer)));
|
||||
const imagesToMergeList = images.slice(mergeStartIndex).map((img) => img.filePath);
|
||||
const mergedImage = await mergeImages(imagesToMergeList);
|
||||
processedBuffers = [...compressedFirstImages, mergedImage];
|
||||
}
|
||||
|
||||
// 确保总大小不超过10MB
|
||||
return ensureTotalSizeLimit(processedBuffers);
|
||||
}
|
||||
|
||||
// 使用 AI 过滤与分镜相关的资产
|
||||
async function filterRelevantAssets(prompts: string[], allResources: ResourceItem[], availableImages: ImageInfo[]): Promise<ImageInfo[]> {
|
||||
if (allResources.length === 0 || availableImages.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const availableNames = new Set(availableImages.map((img) => img.name));
|
||||
const availableResources = allResources.filter((r) => availableNames.has(r.name));
|
||||
|
||||
if (availableResources.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const result = await u.ai.text.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。
|
||||
|
||||
分镜描述：
|
||||
${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")}
|
||||
|
||||
可用资产列表：
|
||||
${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")}
|
||||
|
||||
请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`,
|
||||
},
|
||||
],
|
||||
output: {
|
||||
relevantAssets: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().describe("资产名称"),
|
||||
reason: z.string().describe("选择该资产的原因"),
|
||||
}),
|
||||
)
|
||||
.describe("与分镜内容相关的资产列表"),
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
if (!result?.relevantAssets || result.relevantAssets.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const relevantNames = new Set(result.relevantAssets.map((a) => a.name));
|
||||
const filteredImages = availableImages.filter((img) => relevantNames.has(img.name));
|
||||
|
||||
return filteredImages.length > 0 ? filteredImages : availableImages;
|
||||
}
|
||||
|
||||
// 构建资产映射提示词
|
||||
function buildResourcesMapPrompts(images: ImageInfo[]): string {
|
||||
if (images.length === 0) return "";
|
||||
|
||||
const mapping = images.map((item, index) => {
|
||||
if (index < 9) {
|
||||
return `${item.name}=图片${index + 1}`;
|
||||
} else {
|
||||
return `${item.name}=图10-${index - 8}`;
|
||||
}
|
||||
});
|
||||
|
||||
return `其中人物、场景、道具参考对照关系如下：${mapping.join(", ")}。`;
|
||||
}
|
||||
|
||||
export default async (cells: { prompt: string }[], scriptId: number, projectId: number) => {
|
||||
const scriptData = await u.db("t_script").where({ id: scriptId, projectId }).first();
|
||||
const projectInfo = await u.db("t_project").where({ id: projectId }).first();
|
||||
|
||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId }).first();
|
||||
const outline: EpisodeData | null = row?.data ? JSON.parse(row.data) : null;
|
||||
|
||||
const resources: ResourceItem[] = outline
|
||||
? (["characters", "props", "scenes"] as const).flatMap((k) => outline[k]?.map((i) => ({ name: i.name, intro: i.description })) ?? [])
|
||||
: [];
|
||||
|
||||
const resourceNames = resources.map((r) => r.name);
|
||||
const imagesRaw = await u.db("t_assets").whereIn("name", resourceNames).andWhere({ projectId }).select("name", "type", "filePath");
|
||||
|
||||
const allImages = imagesRaw
|
||||
.sort((a, b) => {
|
||||
const order = ["角色", "场景", "道具"];
|
||||
return order.indexOf(a.type!) - order.indexOf(b.type!);
|
||||
})
|
||||
.filter((img) => img.filePath) as ImageInfo[];
|
||||
|
||||
if (allImages.length === 0) {
|
||||
throw new Error("未找到可用的图片资源");
|
||||
}
|
||||
|
||||
const cellPrompts = cells.map((c) => c.prompt);
|
||||
|
||||
// 使用 AI 过滤相关资产
|
||||
const filteredImages = await filterRelevantAssets(cellPrompts, resources, allImages);
|
||||
|
||||
const resourcesMapPrompts = buildResourcesMapPrompts(filteredImages);
|
||||
console.log("====润色前：", cellPrompts);
|
||||
const promptsData = await generateImagePromptsTool({
|
||||
prompts: cellPrompts,
|
||||
style: `类型：${projectInfo?.type!}，风格：${projectInfo?.artStyle!}`,
|
||||
aspectRatio: projectInfo?.videoRatio! as any,
|
||||
assetsName: resources,
|
||||
});
|
||||
|
||||
// const prompts = `请生成${promptsData.gridLayout.totalCells}格,${promptsData.gridLayout.cols}列×${promptsData.gridLayout.rows}行宫格图。
|
||||
|
||||
// ${promptsData.prompt}
|
||||
|
||||
// 注意：请严格按照提示词内容生成图片，确保人物样貌、艺术风格、色调光影一致。
|
||||
// `;
|
||||
const prompts = promptsData.prompt;
|
||||
console.log("====润色后：", prompts);
|
||||
|
||||
const processedImages = await processImages(filteredImages);
|
||||
|
||||
const contentStr = await u.ai.image({
|
||||
systemPrompt: resourcesMapPrompts,
|
||||
prompt: prompts,
|
||||
size: "4K",
|
||||
aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9",
|
||||
imageBase64: processedImages.map((buf) => buf.toString("base64")),
|
||||
});
|
||||
|
||||
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
const base64Str = match?.[1] ?? contentStr;
|
||||
const buffer = Buffer.from(base64Str, "base64");
|
||||
|
||||
return buffer;
|
||||
};
|
||||
94
backup/agents/storyboard/imageSplitting.ts
Normal file
94
backup/agents/storyboard/imageSplitting.ts
Normal file
@ -0,0 +1,94 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
interface GridLayoutResult {
|
||||
cols: number;
|
||||
rows: number;
|
||||
totalCells: number;
|
||||
placeholderCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算宫格布局
|
||||
* 1张: 1x1
|
||||
* 2张: 2x1
|
||||
* 3张: 3x1
|
||||
* 4张: 2x2
|
||||
* 5-9张: 3x3
|
||||
* 10-12张: 3x4
|
||||
* 13-15张: 3x5
|
||||
* ...以此类推（3列，行数递增）
|
||||
*/
|
||||
function calculateGridLayout(count: number): GridLayoutResult {
|
||||
let cols: number;
|
||||
let rows: number;
|
||||
if (count <= 0) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 1) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 2) {
|
||||
cols = 2;
|
||||
rows = 1;
|
||||
} else if (count === 3) {
|
||||
cols = 3;
|
||||
rows = 1;
|
||||
} else if (count === 4) {
|
||||
cols = 2;
|
||||
rows = 2;
|
||||
} else if (count <= 9) {
|
||||
// 5-9格统一用3x3
|
||||
cols = 3;
|
||||
rows = 3;
|
||||
} else {
|
||||
cols = 3;
|
||||
rows = Math.ceil(count / 3);
|
||||
}
|
||||
const totalCells = cols * rows;
|
||||
const placeholderCount = totalCells - count;
|
||||
return { cols, rows, totalCells, placeholderCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* 分割宫格图片
|
||||
* @param image - 输入的宫格图片 Buffer
|
||||
* @param length - 实际需要的图片数量（不包含占位图）
|
||||
* @returns 分割后的单张图片 Buffer 数组
|
||||
*/
|
||||
export default async (image: Buffer, length: number): Promise<Buffer[]> => {
|
||||
const metadata = await sharp(image).metadata();
|
||||
const { width: totalWidth, height: totalHeight } = metadata;
|
||||
|
||||
if (!totalWidth || !totalHeight) {
|
||||
throw new Error("无法获取图片尺寸");
|
||||
}
|
||||
|
||||
const { cols, rows } = calculateGridLayout(length);
|
||||
|
||||
const cellWidth = Math.floor(totalWidth / cols);
|
||||
const cellHeight = Math.floor(totalHeight / rows);
|
||||
|
||||
const buffers: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
const row = Math.floor(i / cols);
|
||||
const col = i % cols;
|
||||
|
||||
const left = col * cellWidth;
|
||||
const top = row * cellHeight;
|
||||
|
||||
const cellBuffer = await sharp(image)
|
||||
.extract({
|
||||
left,
|
||||
top,
|
||||
width: cellWidth,
|
||||
height: cellHeight,
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
buffers.push(cellBuffer);
|
||||
}
|
||||
|
||||
return buffers;
|
||||
};
|
||||
737
backup/agents/storyboard/index.ts
Normal file
737
backup/agents/storyboard/index.ts
Normal file
@ -0,0 +1,737 @@
|
||||
// @/agents/Storyboard.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
import generateImageTool from "./generateImageTool";
|
||||
import imageSplitting from "./imageSplitting";
|
||||
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
type AgentType = "segmentAgent" | "shotAgent";
|
||||
type RefreshEvent = "storyline" | "outline" | "assets";
|
||||
|
||||
// ==================== 常量配置 ====================
|
||||
|
||||
// const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
// segmentAgent: segmentPrompts,
|
||||
// shotAgent: shotPrompts,
|
||||
// director: directorPrompts,
|
||||
// };
|
||||
|
||||
// ==================== 类型定义：片段和画面 ====================
|
||||
|
||||
interface Segment {
|
||||
index: number;
|
||||
description: string;
|
||||
emotion?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
interface Shot {
|
||||
id: number; // 分镜独立ID
|
||||
segmentId: number; // 所属片段ID
|
||||
title: string;
|
||||
x: number;
|
||||
y: number;
|
||||
cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头
|
||||
}
|
||||
|
||||
// ==================== 主类 ====================
|
||||
|
||||
export default class Storyboard {
|
||||
private readonly projectId: number;
|
||||
private readonly scriptId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
// 存储 segmentAgent 生成的片段结果
|
||||
private segments: Segment[] = [];
|
||||
// 存储 shotAgent 生成的分镜结果
|
||||
private shots: Shot[] = [];
|
||||
// 分镜ID计数器
|
||||
private shotIdCounter: number = 0;
|
||||
// 存储正在生成分镜图的分镜ID
|
||||
private generatingShots: Set<number> = new Set();
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number, scriptId: number) {
|
||||
this.projectId = projectId;
|
||||
this.scriptId = scriptId;
|
||||
}
|
||||
|
||||
// 更新shopts
|
||||
public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) {
|
||||
console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId);
|
||||
console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId);
|
||||
console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell);
|
||||
const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId);
|
||||
if (shotIndex === -1) {
|
||||
return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`;
|
||||
}
|
||||
const cellIndex = this.shots[shotIndex].cells.findIndex((item) => item.id === cellId.toString());
|
||||
if (cellIndex === -1) {
|
||||
return `镜头 ${cellId} 不存在，请检查镜头ID是否正确`;
|
||||
}
|
||||
this.shots[shotIndex].cells[cellIndex] = { ...this.shots[shotIndex].cells[cellIndex], ...cell };
|
||||
}
|
||||
|
||||
// ==================== 公共方法 ====================
|
||||
|
||||
get events() {
|
||||
return this.emitter;
|
||||
}
|
||||
// ==================== 私有工具方法 ====================
|
||||
|
||||
private emit(event: string, data?: any) {
|
||||
this.emitter.emit(event, data);
|
||||
}
|
||||
|
||||
private refresh(type: RefreshEvent) {
|
||||
this.emit("refresh", type);
|
||||
}
|
||||
|
||||
private log(action: string, detail?: string) {
|
||||
const msg = detail ? `${action}: ${detail}` : action;
|
||||
console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`);
|
||||
}
|
||||
|
||||
// ==================== 剧本相关操作 ====================
|
||||
|
||||
getScript = tool(
|
||||
async () => {
|
||||
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
||||
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
if (!script) throw new Error("剧本不存在");
|
||||
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
||||
},
|
||||
{
|
||||
name: "getScript",
|
||||
description: "获取剧本内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== 资产相关操作 ====================
|
||||
|
||||
/**
|
||||
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
||||
*/
|
||||
getAssets = tool(
|
||||
async () => {
|
||||
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
||||
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
||||
const outline: any | null = row?.data ? JSON.parse(row.data) : null;
|
||||
|
||||
if (!outline) {
|
||||
return "暂无资产数据";
|
||||
}
|
||||
|
||||
// 提取资源名称和描述（与generateImageTool保持一致的字段名）
|
||||
const resources = outline
|
||||
? (["characters", "props", "scenes"] as const).flatMap(
|
||||
(k) => outline[k]?.map((i: any) => ({ name: i.name, description: i.description })) ?? [],
|
||||
)
|
||||
: [];
|
||||
|
||||
if (resources.length === 0) {
|
||||
return "暂无资产数据";
|
||||
}
|
||||
|
||||
// 分类提取资源并格式化
|
||||
const characters = outline?.characters?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||
const props = outline?.props?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||
const scenes = outline?.scenes?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||
|
||||
const sections = [
|
||||
characters.length ? `【角色】\n${characters.join("\n")}` : "",
|
||||
props.length ? `【道具】\n${props.join("\n")}` : "",
|
||||
scenes.length ? `【场景】\n${scenes.join("\n")}` : "",
|
||||
].filter(Boolean);
|
||||
|
||||
if (sections.length === 0) {
|
||||
return "暂无资产数据";
|
||||
}
|
||||
|
||||
return `<资产列表>
|
||||
${sections.join("\n\n")}
|
||||
</资产列表>
|
||||
|
||||
⚠️ 重要规则：
|
||||
1. 必须原封不动地使用上述资产名称，禁止使用近义词、缩写或任何变体
|
||||
2. 禁止在资产名称前后添加修饰词
|
||||
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
||||
},
|
||||
{
|
||||
name: "getAssets",
|
||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== 片段和分镜工具 ====================
|
||||
|
||||
/**
|
||||
* 获取当前存储的片段数据（供 shotAgent 调用）
|
||||
*/
|
||||
getSegments = tool(
|
||||
async () => {
|
||||
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
||||
if (this.segments.length === 0) {
|
||||
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
||||
}
|
||||
return JSON.stringify(this.segments, null, 2);
|
||||
},
|
||||
{
|
||||
name: "getSegments",
|
||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 更新/存储片段数据（供 segmentAgent 调用）
|
||||
*/
|
||||
updateSegments = tool(
|
||||
async ({ segments }: { segments: Segment[] }) => {
|
||||
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||
this.segments = segments;
|
||||
this.emit("segmentsUpdated", this.segments);
|
||||
return `成功存储 ${segments.length} 个片段`;
|
||||
},
|
||||
{
|
||||
name: "updateSegments",
|
||||
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||
schema: z.object({
|
||||
segments: z
|
||||
.array(
|
||||
z.object({
|
||||
index: z.number().describe("片段序号"),
|
||||
description: z.string().describe("片段描述"),
|
||||
emotion: z.string().optional().describe("情绪氛围"),
|
||||
action: z.string().optional().describe("主要动作"),
|
||||
}),
|
||||
)
|
||||
.describe("片段数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 添加分镜（供 shotAgent 调用）
|
||||
*/
|
||||
addShots = tool(
|
||||
async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||
const added: { id: number; segmentIndex: number }[] = [];
|
||||
const skipped: number[] = [];
|
||||
|
||||
for (const item of shots) {
|
||||
const exists = this.shots.some((f) => f.segmentId === item.segmentIndex);
|
||||
if (exists) {
|
||||
skipped.push(item.segmentIndex);
|
||||
continue;
|
||||
}
|
||||
// 分配独立的分镜ID
|
||||
this.shotIdCounter++;
|
||||
const shotId = this.shotIdCounter;
|
||||
this.shots.push({
|
||||
id: shotId,
|
||||
segmentId: item.segmentIndex,
|
||||
title: `分镜 ${shotId}`,
|
||||
x: 0,
|
||||
y: 0,
|
||||
cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })),
|
||||
});
|
||||
added.push({ id: shotId, segmentIndex: item.segmentIndex });
|
||||
}
|
||||
|
||||
const addedInfo = added.map((a) => `分镜${a.id}(片段${a.segmentIndex})`).join(", ");
|
||||
this.log("添加分镜", `新增: [${addedInfo}], 跳过片段: [${skipped.join(", ")}]`);
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
|
||||
if (skipped.length) {
|
||||
return `已添加${addedInfo}；片段 ${skipped.join(", ")} 已存在分镜被跳过。当前共 ${this.shots.length} 个分镜`;
|
||||
}
|
||||
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "addShots",
|
||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||
schema: z.object({
|
||||
shots: z
|
||||
.array(
|
||||
z.object({
|
||||
segmentIndex: z.number().describe("对应的片段序号"),
|
||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||
}),
|
||||
)
|
||||
.describe("要添加的分镜数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 更新指定分镜（供 shotAgent 调用）
|
||||
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
||||
*/
|
||||
updateShots = tool(
|
||||
async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
||||
|
||||
if (existingIndex === -1) {
|
||||
return `分镜 ${shotId} 不存在，请检查分镜ID是否正确`;
|
||||
}
|
||||
|
||||
const existingCells = this.shots[existingIndex].cells;
|
||||
|
||||
// 更新 cells，保留原有的 id 和 src 字段
|
||||
this.shots[existingIndex].cells = prompts.map((prompt, i) => {
|
||||
const existingCell = existingCells[i];
|
||||
if (existingCell) {
|
||||
// 保留原有 cell 的 id 和 src，只更新 prompt
|
||||
return { ...existingCell, prompt };
|
||||
} else {
|
||||
// 新增的 cell
|
||||
return { id: u.uuid(), prompt };
|
||||
}
|
||||
});
|
||||
|
||||
this.log("更新分镜", `分镜 ${shotId}`);
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
|
||||
return `已更新分镜 ${shotId}`;
|
||||
},
|
||||
{
|
||||
name: "updateShots",
|
||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||
schema: z.object({
|
||||
shotId: z.number().describe("要更新的分镜ID"),
|
||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 删除指定分镜（供 shotAgent 调用）
|
||||
*/
|
||||
deleteShots = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const deleted: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
|
||||
for (const shotId of shotIds) {
|
||||
const idx = this.shots.findIndex((item) => item.id === shotId);
|
||||
if (idx === -1) {
|
||||
notFound.push(shotId);
|
||||
} else {
|
||||
this.shots.splice(idx, 1);
|
||||
deleted.push(shotId);
|
||||
}
|
||||
}
|
||||
|
||||
this.log("删除分镜", `删除: [分镜${deleted.join(", 分镜")}], 未找到: [分镜${notFound.join(", 分镜")}]`);
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
|
||||
if (notFound.length) {
|
||||
return `已删除分镜 ${deleted.join(", ")}；分镜 ${notFound.join(", ")} 不存在。当前共 ${this.shots.length} 个分镜`;
|
||||
}
|
||||
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "deleteShots",
|
||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 生成分镜图（异步执行，使用 nanoBanana）
|
||||
*/
|
||||
generateShotImage = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const toGenerate: number[] = [];
|
||||
const alreadyGenerating: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
|
||||
for (const shotId of shotIds) {
|
||||
const shot = this.shots.find((f) => f.id === shotId);
|
||||
if (!shot) {
|
||||
notFound.push(shotId);
|
||||
continue;
|
||||
}
|
||||
if (this.generatingShots.has(shotId)) {
|
||||
alreadyGenerating.push(shotId);
|
||||
continue;
|
||||
}
|
||||
toGenerate.push(shotId);
|
||||
}
|
||||
|
||||
if (toGenerate.length === 0) {
|
||||
if (notFound.length) {
|
||||
return `分镜 ${notFound.join(", ")} 不存在，请检查分镜ID是否正确`;
|
||||
}
|
||||
if (alreadyGenerating.length) {
|
||||
return `分镜 ${alreadyGenerating.join(", ")} 正在生成中，请稍候`;
|
||||
}
|
||||
return "没有需要生成的分镜";
|
||||
}
|
||||
|
||||
// 标记为正在生成
|
||||
for (const id of toGenerate) {
|
||||
this.generatingShots.add(id);
|
||||
}
|
||||
|
||||
// 通知前端开始生成
|
||||
this.emit("shotImageGenerateStart", { shotIds: toGenerate });
|
||||
this.log("开始生成分镜图", `分镜: [${toGenerate.join(", ")}]`);
|
||||
|
||||
// 异步执行图片生成（不阻塞 Agent 流程）
|
||||
this.executeShotImageGeneration(toGenerate).catch((err) => {
|
||||
this.log("分镜图生成错误", err.message);
|
||||
this.emit("shotImageGenerateError", { shotIds: toGenerate, error: err.message });
|
||||
});
|
||||
|
||||
let result = `已开始为分镜 ${toGenerate.join(", ")} 生成分镜图，生成过程在后台进行`;
|
||||
if (alreadyGenerating.length) {
|
||||
result += `；分镜 ${alreadyGenerating.join(", ")} 正在生成中`;
|
||||
}
|
||||
if (notFound.length) {
|
||||
result += `；分镜 ${notFound.join(", ")} 不存在`;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
{
|
||||
name: "generateShotImage",
|
||||
description:
|
||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 执行分镜图生成的具体逻辑（异步并发）
|
||||
* 每个分镜包含多个镜头，所有镜头的提示词合并生成一张宫格图，再分割为单张镜头图片
|
||||
*/
|
||||
async executeShotImageGeneration(shotIds: number[]): Promise<void> {
|
||||
await Promise.all(shotIds.map((shotId) => this.generateSingleShotImage(shotId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成单个分镜的图片
|
||||
*/
|
||||
private async generateSingleShotImage(shotId: number): Promise<void> {
|
||||
try {
|
||||
const shot = this.shots.find((f) => f.id === shotId);
|
||||
if (!shot) return;
|
||||
|
||||
// 提取所有镜头的有效提示词
|
||||
const prompts: string[] = shot.cells.map((c) => c.prompt).filter((p): p is string => Boolean(p));
|
||||
|
||||
if (prompts.length === 0) {
|
||||
this.log("跳过分镜图生成", `分镜 ${shotId} 没有有效的镜头提示词`);
|
||||
this.generatingShots.delete(shotId);
|
||||
return;
|
||||
}
|
||||
|
||||
// 通知前端正在生成该分镜
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "generating", message: "正在调用 AI 生成宫格图片" });
|
||||
|
||||
// 根据所有镜头提示词生成宫格图片
|
||||
const gridImage = await generateImageTool(
|
||||
prompts.map((p) => ({ prompt: p })),
|
||||
this.scriptId,
|
||||
this.projectId,
|
||||
);
|
||||
|
||||
// 通知前端正在分割图片
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "splitting", message: "正在分割宫格图片为单张镜头图" });
|
||||
|
||||
// 分割宫格图片为单张镜头图片
|
||||
const imageBuffers = await imageSplitting(gridImage, prompts.length);
|
||||
|
||||
// 通知前端正在保存图片
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "saving", message: `正在保存 ${imageBuffers.length} 张镜头图片` });
|
||||
|
||||
// 保存分割后的镜头图片到 OSS，并获取文件路径
|
||||
const timestamp = Date.now();
|
||||
const imagePaths: string[] = [];
|
||||
|
||||
for (let i = 0; i < imageBuffers.length; i++) {
|
||||
const fileName = `${this.projectId}/chat/${this.scriptId}/storyboard/shot_${shotId}_take_${i}_${timestamp}.png`;
|
||||
await u.oss.writeFile(fileName, imageBuffers[i]);
|
||||
const imageUrl = await u.oss.getFileUrl(fileName);
|
||||
imagePaths.push(imageUrl);
|
||||
|
||||
// 每保存一张镜头图片通知进度
|
||||
this.emit("shotImageGenerateProgress", {
|
||||
shotId,
|
||||
status: "saving",
|
||||
message: `已保存 ${i + 1}/${imageBuffers.length} 张镜头图片`,
|
||||
progress: Math.round(((i + 1) / imageBuffers.length) * 100),
|
||||
});
|
||||
}
|
||||
|
||||
// 更新每个镜头的 src 字段
|
||||
shot.cells = shot.cells.map((cell, i) => ({
|
||||
id: u.uuid(),
|
||||
...cell,
|
||||
src: imagePaths[i] || cell.src,
|
||||
}));
|
||||
|
||||
// 生成完成后更新状态
|
||||
this.generatingShots.delete(shotId);
|
||||
this.emit("shotImageGenerateComplete", { shotId, shot, imagePaths });
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
this.log("分镜图生成完成", `分镜 ${shotId}，共 ${imagePaths.length} 张镜头图片`);
|
||||
} catch (err: any) {
|
||||
this.generatingShots.delete(shotId);
|
||||
this.emit("shotImageGenerateError", { shotId, error: err.message });
|
||||
this.log("分镜图生成失败", `分镜 ${shotId}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 公共访问器 ====================
|
||||
|
||||
/**
|
||||
* 获取当前片段数据
|
||||
*/
|
||||
getSegmentsData(): Segment[] {
|
||||
return this.segments;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前分镜数据
|
||||
*/
|
||||
getShotsData(): Shot[] {
|
||||
return this.shots;
|
||||
}
|
||||
|
||||
// ==================== 上下文构建 ====================
|
||||
|
||||
private async buildEnvironmentContext(): Promise<string> {
|
||||
const projectInfo = await u.db("t_project").where({ id: this.projectId }).first();
|
||||
|
||||
const row = await u.db("t_outline").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
const outline: any | null = row?.data ? JSON.parse(row.data) : null;
|
||||
|
||||
// 分类提取资源名称
|
||||
const characters = outline?.characters?.map((i: any) => i.name) ?? [];
|
||||
const props = outline?.props?.map((i: any) => i.name) ?? [];
|
||||
const scenes = outline?.scenes?.map((i: any) => i.name) ?? [];
|
||||
|
||||
const assetList =
|
||||
[
|
||||
characters.length ? `【角色】${characters.join("、")}` : "",
|
||||
props.length ? `【道具】${props.join("、")}` : "",
|
||||
scenes.length ? `【场景】${scenes.join("、")}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n") || "无";
|
||||
|
||||
return `<环境信息>
|
||||
项目ID: ${this.projectId}
|
||||
系统时间: ${new Date().toLocaleString()}
|
||||
|
||||
项目名称: ${projectInfo?.name || "未知"}
|
||||
项目简介: ${projectInfo?.intro || "无"}
|
||||
类型: ${projectInfo?.type || "未知"}
|
||||
风格: ${projectInfo?.artStyle || "未知"}
|
||||
视频比例: ${projectInfo?.videoRatio || "未知"}
|
||||
|
||||
资产列表:
|
||||
${assetList}
|
||||
|
||||
</环境信息>`;
|
||||
}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
const env = await this.buildEnvironmentContext();
|
||||
const history = this.buildConversationHistory();
|
||||
|
||||
return `${env}
|
||||
|
||||
<对话历史>
|
||||
${history}
|
||||
</对话历史>
|
||||
|
||||
<当前任务>
|
||||
${task}
|
||||
</当前任务>`;
|
||||
}
|
||||
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取不同 Sub-Agent 可用的工具
|
||||
*/
|
||||
private getSubAgentTools(agentType: AgentType) {
|
||||
switch (agentType) {
|
||||
case "segmentAgent":
|
||||
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
||||
return [this.getScript, this.getAssets, this.updateSegments];
|
||||
case "shotAgent":
|
||||
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
||||
return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage];
|
||||
default:
|
||||
return [this.getScript];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用 Sub-Agent（流式传输）
|
||||
*/
|
||||
private async invokeSubAgent(agentType: AgentType, task: string): Promise<string> {
|
||||
this.emit("transfer", { to: agentType });
|
||||
this.log(`Sub-Agent 调用`, agentType);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]);
|
||||
const segmentAgent = promptsList.find((p) => p.code === "storyboard-segment");
|
||||
const shotAgent = promptsList.find((p) => p.code === "storyboard-shot");
|
||||
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
segmentAgent: segmentAgent?.customValue || segmentAgent?.defaultValue || errPrompts,
|
||||
shotAgent: shotAgent?.customValue || shotAgent?.defaultValue || errPrompts,
|
||||
};
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(agentType),
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
}
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
return fullResponse;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool(
|
||||
"segmentAgent",
|
||||
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
||||
),
|
||||
this.createSubAgentTool(
|
||||
"shotAgent",
|
||||
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
||||
),
|
||||
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getScript,
|
||||
this.getSegments,
|
||||
this.generateShotImage,
|
||||
...this.getSubAgentTools("segmentAgent"),
|
||||
...this.getSubAgentTools("shotAgent"),
|
||||
];
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
console.log("模型名称:", this.modelName);
|
||||
this.history.push(["user", msg]);
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "storyboard-main").first();
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
}
|
||||
}
|
||||
120
docker/Dockerfile
Normal file
120
docker/Dockerfile
Normal file
@ -0,0 +1,120 @@
|
||||
# 构建阶段
|
||||
FROM node:24-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 定义构建参数
|
||||
ARG GIT=github
|
||||
ARG TAG=""
|
||||
ARG BRANCH=""
|
||||
|
||||
# 安装 git
|
||||
RUN apk add --no-cache git
|
||||
|
||||
RUN npm config set registry https://registry.npmmirror.com/ && \
|
||||
yarn config set registry https://registry.npmmirror.com/
|
||||
|
||||
# 根据参数选择仓库源，支持 TAG / BRANCH 切换
|
||||
# 优先级: TAG > BRANCH > 最新 tag > 默认分支
|
||||
RUN if [ "$GIT" = "gitee" ]; then \
|
||||
REPO_URL="https://gitee.com/HBAI-Ltd/Toonflow-app.git"; \
|
||||
else \
|
||||
REPO_URL="https://github.com/HBAI-Ltd/Toonflow-app.git"; \
|
||||
fi && \
|
||||
echo "Cloning from: $REPO_URL" && \
|
||||
git clone "$REPO_URL" . && \
|
||||
if [ -n "$TAG" ]; then \
|
||||
echo "Checking out specified tag: $TAG" && \
|
||||
git checkout "$TAG"; \
|
||||
elif [ -n "$BRANCH" ]; then \
|
||||
echo "Checking out specified branch: $BRANCH" && \
|
||||
git checkout "$BRANCH"; \
|
||||
else \
|
||||
LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || git tag --sort=-v:refname | head -n 1) && \
|
||||
if [ -n "$LATEST_TAG" ]; then \
|
||||
echo "Checking out latest tag: $LATEST_TAG" && \
|
||||
git checkout "$LATEST_TAG"; \
|
||||
else \
|
||||
echo "No tags found, using default branch"; \
|
||||
fi; \
|
||||
fi && \
|
||||
echo "Current version:" && git describe --tags --always
|
||||
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
RUN yarn build
|
||||
|
||||
# 生产阶段
|
||||
FROM node:24-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装 nginx 和 supervisor
|
||||
RUN apk add --no-cache nginx supervisor && \
|
||||
mkdir -p /var/lib/nginx/logs /var/log/nginx && \
|
||||
npm config set registry https://registry.npmmirror.com/ && \
|
||||
yarn config set registry https://registry.npmmirror.com/ && \
|
||||
npm install -g pm2
|
||||
|
||||
# 复制后端文件
|
||||
COPY --from=builder /app/build ./build
|
||||
COPY --from=builder /app/package.json ./
|
||||
COPY --from=builder /app/yarn.lock ./
|
||||
|
||||
# 复制静态页面到 nginx 目录
|
||||
COPY --from=builder /app/scripts/web /usr/share/nginx/html
|
||||
|
||||
# 只安装生产依赖
|
||||
RUN yarn install --frozen-lockfile --production
|
||||
|
||||
# 配置 nginx
|
||||
RUN cat > /etc/nginx/http.d/default.conf << 'EOF'
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# 配置 nginx 主配置，日志输出到 stderr/stdout
|
||||
RUN sed -i 's|error_log /var/log/nginx/error.log warn;|error_log /dev/stderr warn;|g' /etc/nginx/nginx.conf || true && \
|
||||
sed -i 's|access_log /var/log/nginx/access.log main;|access_log /dev/stdout main;|g' /etc/nginx/nginx.conf || true
|
||||
|
||||
# 配置 supervisor
|
||||
RUN cat > /etc/supervisord.conf << 'EOF'
|
||||
[supervisord]
|
||||
nodaemon=true
|
||||
logfile=/var/log/supervisord.log
|
||||
pidfile=/var/run/supervisord.pid
|
||||
|
||||
[program:nginx]
|
||||
command=nginx -g "daemon off;"
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
|
||||
[program:app]
|
||||
command=pm2-runtime start build/app.js --name app
|
||||
directory=/app
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
environment=NODE_ENV=prod
|
||||
EOF
|
||||
|
||||
ENV NODE_ENV=prod
|
||||
|
||||
EXPOSE 80
|
||||
EXPOSE 60000
|
||||
|
||||
# 启动时创建必要目录（防止 volume 挂载覆盖）
|
||||
CMD sh -c "mkdir -p /var/log/nginx /var/lib/nginx/logs && exec supervisord -c /etc/supervisord.conf"
|
||||
94
docker/Dockerfile.local
Normal file
94
docker/Dockerfile.local
Normal file
@ -0,0 +1,94 @@
|
||||
# 本地构建阶段 - 使用本地源码，不从 git 克隆
|
||||
FROM node:24-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN npm config set registry https://registry.npmmirror.com/ && \
|
||||
yarn config set registry https://registry.npmmirror.com/
|
||||
|
||||
# 复制依赖文件
|
||||
COPY package.json yarn.lock ./
|
||||
|
||||
RUN yarn install --frozen-lockfile
|
||||
|
||||
# 复制源码
|
||||
COPY tsconfig.json ./
|
||||
COPY src/ ./src/
|
||||
COPY scripts/ ./scripts/
|
||||
|
||||
RUN yarn build
|
||||
|
||||
# 生产阶段
|
||||
FROM node:24-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 安装 nginx 和 supervisor
|
||||
RUN apk add --no-cache nginx supervisor && \
|
||||
mkdir -p /var/lib/nginx/logs /var/log/nginx && \
|
||||
npm config set registry https://registry.npmmirror.com/ && \
|
||||
yarn config set registry https://registry.npmmirror.com/ && \
|
||||
npm install -g pm2
|
||||
|
||||
# 复制后端文件
|
||||
COPY --from=builder /app/build ./build
|
||||
COPY --from=builder /app/package.json ./
|
||||
COPY --from=builder /app/yarn.lock ./
|
||||
|
||||
# 复制静态页面到 nginx 目录
|
||||
COPY --from=builder /app/scripts/web /usr/share/nginx/html
|
||||
|
||||
# 只安装生产依赖
|
||||
RUN yarn install --frozen-lockfile --production
|
||||
|
||||
# 配置 nginx
|
||||
RUN cat > /etc/nginx/http.d/default.conf << 'EOF'
|
||||
server {
|
||||
listen 80;
|
||||
location / {
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# 配置 nginx 主配置，日志输出到 stderr/stdout
|
||||
RUN sed -i 's|error_log /var/log/nginx/error.log warn;|error_log /dev/stderr warn;|g' /etc/nginx/nginx.conf || true && \
|
||||
sed -i 's|access_log /var/log/nginx/access.log main;|access_log /dev/stdout main;|g' /etc/nginx/nginx.conf || true
|
||||
|
||||
# 配置 supervisor
|
||||
RUN cat > /etc/supervisord.conf << 'EOF'
|
||||
[supervisord]
|
||||
nodaemon=true
|
||||
logfile=/var/log/supervisord.log
|
||||
pidfile=/var/run/supervisord.pid
|
||||
|
||||
[program:nginx]
|
||||
command=nginx -g "daemon off;"
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
|
||||
[program:app]
|
||||
command=pm2-runtime start build/app.js --name app
|
||||
directory=/app
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
environment=NODE_ENV=prod
|
||||
EOF
|
||||
|
||||
ENV NODE_ENV=prod
|
||||
|
||||
EXPOSE 80
|
||||
EXPOSE 60000
|
||||
|
||||
# 启动时创建必要目录（防止 volume 挂载覆盖）
|
||||
CMD sh -c "mkdir -p /var/log/nginx /var/lib/nginx/logs && exec supervisord -c /etc/supervisord.conf"
|
||||
24
docker/docker-compose.local.yml
Normal file
24
docker/docker-compose.local.yml
Normal file
@ -0,0 +1,24 @@
|
||||
# 本地打包测试用，使用本地源码构建
|
||||
# 用法: docker-compose -f docker/docker-compose.local.yml up -d --build
|
||||
|
||||
services:
|
||||
toonflow:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.local
|
||||
image: toonflow:local
|
||||
container_name: toonflow-local
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:80"
|
||||
- "60000:60000"
|
||||
environment:
|
||||
- NODE_ENV=prod
|
||||
volumes:
|
||||
- ../logs:/var/log
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:80/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
26
docker/docker-compose.yml
Normal file
26
docker/docker-compose.yml
Normal file
@ -0,0 +1,26 @@
|
||||
services:
|
||||
toonflow:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
args:
|
||||
GIT: ${GIT:-github}
|
||||
TAG: ${TAG:-}
|
||||
BRANCH: ${BRANCH:-}
|
||||
image: toonflow:${TAG:-latest}
|
||||
container_name: toonflow
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "80"
|
||||
- "60000:60000"
|
||||
environment:
|
||||
- NODE_ENV=prod
|
||||
volumes:
|
||||
# 可选: 持久化日志
|
||||
- ../logs:/var/log
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:80/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
@ -9,6 +9,7 @@ directories:
|
||||
files:
|
||||
- build/**/*
|
||||
- scripts/web/**/*
|
||||
- env/**/*
|
||||
- package.json
|
||||
- node_modules/**/*
|
||||
- "!node_modules/**/*.{md,ts,map}"
|
||||
|
||||
4
env/.env.dev
vendored
Normal file
4
env/.env.dev
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
NODE_ENV=dev
|
||||
PORT=60000
|
||||
OSSURL=http://127.0.0.1:60000/
|
||||
|
||||
4
env/.env.prod
vendored
Normal file
4
env/.env.prod
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
NODE_ENV=prod
|
||||
PORT=60000
|
||||
OSSURL=http://127.0.0.1:60000/
|
||||
|
||||
37
package.json
37
package.json
@ -1,35 +1,46 @@
|
||||
{
|
||||
"name": "toonflow-serve",
|
||||
"version": "1.0.5",
|
||||
"description": "ToonFlow Serve - Electron Application",
|
||||
"name": "toonflow-app",
|
||||
"version": "1.0.6",
|
||||
"description": "Toonflow 是一款 AI 短剧漫剧工具，能够利用 AI 技术将小说自动转化为剧本，并结合 AI 生成的图片和视频，实现高效的短剧创作。",
|
||||
"author": "HBAI-Ltd <ltlctools@outlook.com>",
|
||||
"homepage": "https://github.com/HBAI-Ltd/Toonflow-app#readme",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/HBAI-Ltd/Toonflow-app.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/HBAI-Ltd/Toonflow-app/issues",
|
||||
"email": "ltlctools@outlook.com"
|
||||
},
|
||||
"main": "build/main.js",
|
||||
"author": "ToonFlow Team",
|
||||
"packageManager": "yarn@1.0.0",
|
||||
"engines": {
|
||||
"node": ">=1.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "nodemon --inspect --exec tsx src/app.ts",
|
||||
"dev:win": "chcp 65001 && electronmon -r tsx scripts/main.ts",
|
||||
"dev:gui": "electronmon -r tsx scripts/main.ts",
|
||||
"lint": "tsc --noEmit",
|
||||
"build": "tsx scripts/build.ts",
|
||||
"build": "cross-env NODE_ENV=prod tsx scripts/build.ts",
|
||||
"pack": "electron-builder --dir",
|
||||
"dist": "yarn build && electron-builder",
|
||||
"dist:win": "yarn build && electron-builder --win",
|
||||
"dist:mac": "yarn build && electron-builder --mac",
|
||||
"dist:linux": "yarn build && electron-builder --linux",
|
||||
"test": "node build/app.js",
|
||||
"test": "cross-env NODE_ENV=prod node build/app.js",
|
||||
"docker:build": "docker-compose -f docker/docker-compose.yml up -d --build",
|
||||
"docker:local": "docker-compose -f docker/docker-compose.local.yml up -d --build",
|
||||
"debug:ai": "npx @ai-sdk/devtools",
|
||||
"license": "bun run scripts/license.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "^3.0.35",
|
||||
"@ai-sdk/deepseek": "^2.0.17",
|
||||
"@ai-sdk/devtools": "^0.0.11",
|
||||
"@ai-sdk/google": "^3.0.20",
|
||||
"@ai-sdk/openai": "^3.0.25",
|
||||
"@aigne/core": "^1.72.0",
|
||||
"@aigne/openai": "^0.16.16",
|
||||
"@langchain/core": "^1.1.15",
|
||||
"@langchain/openai": "^1.2.1",
|
||||
"@ai-sdk/openai-compatible": "^2.0.27",
|
||||
"@ai-sdk/xai": "^3.0.47",
|
||||
"@rmp135/sql-ts": "^2.2.0",
|
||||
"ai": "^6.0.67",
|
||||
"axios": "^1.13.2",
|
||||
@ -46,11 +57,12 @@
|
||||
"js-md5": "^0.8.3",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"knex": "^3.1.0",
|
||||
"langchain": "^1.2.10",
|
||||
"morgan": "^1.10.1",
|
||||
"qwen-ai-provider": "^0.1.1",
|
||||
"serialize-error": "^13.0.1",
|
||||
"sharp": "^0.34.5",
|
||||
"sqlite3": "^5.1.7",
|
||||
"uuid": "^13.0.0",
|
||||
"zhipu-ai-provider": "^0.2.2",
|
||||
"zod": "^4.3.5"
|
||||
},
|
||||
@ -61,6 +73,7 @@
|
||||
"@types/jsonwebtoken": "^9.0.10",
|
||||
"@types/license-checker": "^25.0.6",
|
||||
"@types/morgan": "^1.9.10",
|
||||
"cross-env": "^10.1.0",
|
||||
"electron": "^40.0.0",
|
||||
"electron-builder": "^26.4.0",
|
||||
"electronmon": "^2.0.4",
|
||||
|
||||
@ -1,4 +1,23 @@
|
||||
import esbuild from "esbuild";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
// 打包默认使用 prod 环境变量
|
||||
if (!process.env.NODE_ENV) {
|
||||
process.env.NODE_ENV = "prod";
|
||||
}
|
||||
|
||||
// 自动创建 env 目录和环境变量文件（.gitignore 可能忽略了这些文件）
|
||||
const envDir = path.resolve("env");
|
||||
const envFile = path.join(envDir, `.env.${process.env.NODE_ENV}`);
|
||||
if (!fs.existsSync(envDir)) {
|
||||
fs.mkdirSync(envDir, { recursive: true });
|
||||
}
|
||||
if (!fs.existsSync(envFile)) {
|
||||
const defaultEnv = `NODE_ENV=${process.env.NODE_ENV}\nPORT=60000\nOSSURL=http://127.0.0.1:60000/\n`;
|
||||
fs.writeFileSync(envFile, defaultEnv, "utf8");
|
||||
console.log(`📄 已自动创建环境变量文件: ${envFile}`);
|
||||
}
|
||||
|
||||
const external = ["electron", "sqlite3", "better-sqlite3", "mysql", "mysql2", "pg", "pg-query-stream", "oracledb", "tedious", "mssql"];
|
||||
|
||||
|
||||
@ -3,17 +3,21 @@ import path from "path";
|
||||
import startServe, { closeServe } from "src/app";
|
||||
|
||||
function createMainWindow(): void {
|
||||
const isDev = process.env.NODE_ENV === "dev" || !app.isPackaged;
|
||||
const basePath = isDev ? process.cwd() : app.getAppPath();
|
||||
|
||||
const win = new BrowserWindow({
|
||||
width: 900,
|
||||
height: 600,
|
||||
show: true,
|
||||
autoHideMenuBar: true,
|
||||
icon: path.join(
|
||||
basePath,
|
||||
"scripts",
|
||||
process.platform === "win32" ? "logo.ico" : "logo.png"
|
||||
),
|
||||
});
|
||||
// 开发环境和生产环境使用不同的路径
|
||||
const isDev = process.env.NODE_ENV === "dev" || !app.isPackaged;
|
||||
const htmlPath = isDev
|
||||
? path.join(process.cwd(), "scripts", "web", "index.html")
|
||||
: path.join(app.getAppPath(), "scripts", "web", "index.html");
|
||||
const htmlPath = path.join(basePath, "scripts", "web", "index.html");
|
||||
void win.loadFile(htmlPath);
|
||||
}
|
||||
app.whenReady().then(async () => {
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -1,10 +1,8 @@
|
||||
// @/agents/outlineScript.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { tool, ModelMessage } from "ai";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
@ -75,13 +73,9 @@ const episodeSchema = z.object({
|
||||
export default class OutlineScript {
|
||||
private readonly projectId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
history: Array<ModelMessage> = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number) {
|
||||
this.projectId = projectId;
|
||||
}
|
||||
@ -230,7 +224,7 @@ export default class OutlineScript {
|
||||
}
|
||||
}
|
||||
|
||||
const actualStart = overwrite ? 1 : startEpisode ?? (await this.getMaxEpisode()) + 1;
|
||||
const actualStart = overwrite ? 1 : (startEpisode ?? (await this.getMaxEpisode()) + 1);
|
||||
const insertedCount = await this.insertOutlines(episodes, actualStart);
|
||||
|
||||
const newOutlines = await u
|
||||
@ -403,123 +397,107 @@ ${formatList(ep.classicQuotes, (q) => q)}
|
||||
|
||||
// ==================== Tool 定义：故事线 ====================
|
||||
|
||||
getStoryline = tool(
|
||||
async () => {
|
||||
getStoryline = tool({
|
||||
title: "getStoryline",
|
||||
description: "Get the weather in a location",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取故事线");
|
||||
const storyline = await this.findStoryline();
|
||||
return storyline?.content ?? "当前项目暂无故事线";
|
||||
},
|
||||
{
|
||||
name: "getStoryline",
|
||||
description: "获取当前项目的故事线内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
saveStoryline = tool(
|
||||
async ({ content }) => {
|
||||
saveStoryline = tool({
|
||||
title: "saveStoryline",
|
||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||
inputSchema: z.object({
|
||||
content: z.string().describe("故事线完整内容"),
|
||||
}),
|
||||
execute: async ({ content }) => {
|
||||
this.log("保存故事线");
|
||||
await this.upsertStorylineContent(content);
|
||||
return "故事线保存成功";
|
||||
},
|
||||
{
|
||||
name: "saveStoryline",
|
||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||
schema: z.object({
|
||||
content: z.string().describe("故事线完整内容"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
deleteStoryline = tool(
|
||||
async () => {
|
||||
deleteStoryline = tool({
|
||||
title: "deleteStoryline",
|
||||
description: "删除当前项目的故事线",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("删除故事线");
|
||||
const deleted = await this.deleteStorylineContent();
|
||||
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
||||
},
|
||||
{
|
||||
name: "deleteStoryline",
|
||||
description: "删除当前项目的故事线",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== Tool 定义：大纲 ====================
|
||||
|
||||
getOutline = tool(
|
||||
async ({ simplified = false }) => {
|
||||
getOutline = tool({
|
||||
title: "getOutline",
|
||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||
inputSchema: z.object({
|
||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||
}),
|
||||
execute: async ({ simplified }) => {
|
||||
this.log("获取大纲", `简化模式: ${simplified}`);
|
||||
return this.getOutlineText(simplified);
|
||||
},
|
||||
{
|
||||
name: "getOutline",
|
||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||
schema: z.object({
|
||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
saveOutline = tool(
|
||||
async ({ episodes, overwrite = true, startEpisode }) => {
|
||||
saveOutline = tool({
|
||||
title: "saveOutline",
|
||||
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||
inputSchema: z.object({
|
||||
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||
}),
|
||||
execute: async ({ episodes, overwrite = true, startEpisode }) => {
|
||||
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||
},
|
||||
{
|
||||
name: "saveOutline",
|
||||
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||
schema: z.object({
|
||||
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
updateOutline = tool(
|
||||
async ({ id, data }) => {
|
||||
updateOutline = tool({
|
||||
title: "updateOutline",
|
||||
description: "更新指定ID的单集大纲内容",
|
||||
inputSchema: z.object({
|
||||
id: z.number().describe("大纲ID"),
|
||||
data: episodeSchema.describe("更新后的大纲数据"),
|
||||
}),
|
||||
execute: async ({ id, data }) => {
|
||||
this.log("更新大纲", `ID: ${id}`);
|
||||
const success = await this.updateOutlineData(id, data as EpisodeData);
|
||||
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
||||
},
|
||||
{
|
||||
name: "updateOutline",
|
||||
description: "更新指定ID的单集大纲内容",
|
||||
schema: z.object({
|
||||
id: z.number().describe("大纲ID"),
|
||||
data: episodeSchema.describe("更新后的大纲数据"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
deleteOutline = tool(
|
||||
async ({ ids }) => {
|
||||
deleteOutline = tool({
|
||||
title: "deleteOutline",
|
||||
description: "根据大纲ID删除指定大纲及关联数据",
|
||||
inputSchema: z.object({
|
||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||
}),
|
||||
execute: async ({ ids }) => {
|
||||
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
||||
const results = await this.deleteOutlineData(ids);
|
||||
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
||||
return `删除结果: ${summary}`;
|
||||
},
|
||||
{
|
||||
name: "deleteOutline",
|
||||
description: "根据大纲ID删除指定大纲及关联数据",
|
||||
schema: z.object({
|
||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== Tool 定义：章节 ====================
|
||||
|
||||
getChapter = tool(
|
||||
async ({ chapterNumbers }) => {
|
||||
getChapter = tool({
|
||||
title: "getChapter",
|
||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||
inputSchema: z.object({
|
||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||
}),
|
||||
execute: async ({ chapterNumbers }) => {
|
||||
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
||||
|
||||
const results = await Promise.all(
|
||||
@ -539,36 +517,24 @@ ${formatList(ep.classicQuotes, (q) => q)}
|
||||
|
||||
return results.join("\n\n---\n");
|
||||
},
|
||||
{
|
||||
name: "getChapter",
|
||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||
schema: z.object({
|
||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== Tool 定义：资产 ====================
|
||||
|
||||
generateAssets = tool(
|
||||
async () => {
|
||||
generateAssets = tool({
|
||||
title: "generateAssets",
|
||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("生成资产");
|
||||
const stats = await this.generateAssetsFromOutlines();
|
||||
|
||||
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
||||
return "当前项目没有大纲数据，无法生成资产";
|
||||
}
|
||||
|
||||
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
||||
},
|
||||
{
|
||||
name: "generateAssets",
|
||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== 上下文构建 ====================
|
||||
|
||||
@ -606,7 +572,7 @@ ${this.getChapterContext()}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
@ -627,14 +593,14 @@ ${task}
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private getSubAgentTools() {
|
||||
return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline];
|
||||
}
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
return {
|
||||
getChapter: this.getChapter,
|
||||
getStoryline: this.getStoryline,
|
||||
saveStoryline: this.saveStoryline,
|
||||
getOutline: this.getOutline,
|
||||
saveOutline: this.saveOutline,
|
||||
updateOutline: this.updateOutline,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@ -645,123 +611,124 @@ ${task}
|
||||
this.log(`Sub-Agent 调用`, agentType);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]);
|
||||
const a1Prompt = promptsList.find((p) => p.code === "outlineScript-a1");
|
||||
const a2Prompt = promptsList.find((p) => p.code === "outlineScript-a2");
|
||||
const directorPrompt = promptsList.find((p) => p.code === "outlineScript-director");
|
||||
const promptConfig = await u.getPromptAi("outlineScriptAgent");
|
||||
|
||||
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
AI1: a1Prompt?.customValue || a1Prompt?.defaultValue || errPrompts,
|
||||
AI2: a2Prompt?.customValue || a2Prompt?.defaultValue || errPrompts,
|
||||
director: directorPrompt?.customValue || directorPrompt?.defaultValue || errPrompts,
|
||||
|
||||
const getAiPromptConfig = (code: string) => {
|
||||
const item = promptsList.find((p) => p.code === code);
|
||||
return item?.customValue || item?.defaultValue || errPrompts;
|
||||
};
|
||||
const a1Prompt = getAiPromptConfig("outlineScript-a1");
|
||||
const a2Prompt = getAiPromptConfig("outlineScript-a2");
|
||||
const directorPrompt = getAiPromptConfig("outlineScript-director");
|
||||
const SYSTEM_PROMPTS = {
|
||||
AI1: a1Prompt,
|
||||
AI2: a2Prompt,
|
||||
director: directorPrompt,
|
||||
};
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(),
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
const { fullStream } = await u.ai.text.stream(
|
||||
{
|
||||
system: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(),
|
||||
messages: [{ role: "user", content: context }],
|
||||
maxStep: 100,
|
||||
},
|
||||
promptConfig,
|
||||
);
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: item.text });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
|
||||
return fullResponse ?? `${agentType}已完成任务`;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
return tool({
|
||||
title: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
inputSchema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||
this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||
this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getChapter,
|
||||
this.getStoryline,
|
||||
this.saveStoryline,
|
||||
this.deleteStoryline,
|
||||
this.getOutline,
|
||||
this.saveOutline,
|
||||
this.updateOutline,
|
||||
this.deleteOutline,
|
||||
this.generateAssets,
|
||||
];
|
||||
return {
|
||||
AI1: this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||
AI2: this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||
director: this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
getChapter: this.getChapter,
|
||||
getStoryline: this.getStoryline,
|
||||
saveStoryline: this.saveStoryline,
|
||||
deleteStoryline: this.deleteStoryline,
|
||||
getOutline: this.getOutline,
|
||||
saveOutline: this.saveOutline,
|
||||
updateOutline: this.updateOutline,
|
||||
deleteOutline: this.deleteOutline,
|
||||
generateAssets: this.generateAssets,
|
||||
};
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
this.history.push(["user", msg]);
|
||||
this.history.push({
|
||||
role: "user",
|
||||
content: msg,
|
||||
});
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first();
|
||||
const promptConfig = await u.getPromptAi("outlineScriptAgent");
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
const { fullStream } = await u.ai.text.stream(
|
||||
{
|
||||
system: `${envContext}\n${mainPrompts}`,
|
||||
tools: this.getAllTools(),
|
||||
messages: this.history,
|
||||
maxStep: 100,
|
||||
},
|
||||
promptConfig,
|
||||
);
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("data", item.text);
|
||||
}
|
||||
}
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
|
||||
@ -98,31 +98,47 @@ async function generateGridPrompt(options: GridPromptOptions): Promise<GridPromp
|
||||
: "";
|
||||
|
||||
const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first();
|
||||
|
||||
const promptAiConfig = await u.getPromptAi("storyboardAgent");
|
||||
const mainPrompts = promptsData?.customValue || promptsData?.defaultValue;
|
||||
const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`;
|
||||
|
||||
if (!mainPrompts) return { prompt: errData, gridLayout: layout };
|
||||
|
||||
const chatModel = await u.ai.text({});
|
||||
const result = await u.ai.text.invoke(
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: mainPrompts,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${
|
||||
layout.totalCells
|
||||
}格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
promptAiConfig,
|
||||
);
|
||||
|
||||
const result = await chatModel!.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: mainPrompts,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${
|
||||
layout.totalCells
|
||||
}格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
// const result = await chatModel!.invoke({
|
||||
// messages: [
|
||||
// {
|
||||
// role: "system",
|
||||
// content: mainPrompts,
|
||||
// },
|
||||
// {
|
||||
// role: "user",
|
||||
// content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${
|
||||
// layout.totalCells
|
||||
// }格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`,
|
||||
// },
|
||||
// ],
|
||||
// });
|
||||
|
||||
return {
|
||||
prompt: result?.text ?? errData,
|
||||
prompt: result.text ?? errData,
|
||||
gridLayout: layout,
|
||||
};
|
||||
}
|
||||
|
||||
@ -36,18 +36,6 @@ interface ResourceItem {
|
||||
intro: string;
|
||||
}
|
||||
|
||||
// 资产过滤响应的 schema
|
||||
const filteredAssetsSchema = z.object({
|
||||
relevantAssets: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().describe("资产名称"),
|
||||
reason: z.string().describe("选择该资产的原因"),
|
||||
}),
|
||||
)
|
||||
.describe("与分镜内容相关的资产列表"),
|
||||
});
|
||||
|
||||
// 压缩图片直到不超过指定大小
|
||||
async function compressImage(buffer: Buffer, maxSizeBytes: number = 3 * 1024 * 1024): Promise<Buffer> {
|
||||
if (buffer.length <= maxSizeBytes) {
|
||||
@ -215,12 +203,13 @@ async function filterRelevantAssets(prompts: string[], allResources: ResourceIte
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const chatModel = await u.ai.text({});
|
||||
const result = await chatModel!.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。
|
||||
const apiConfig = await u.getPromptAi("storyboardAgent");
|
||||
const { relevantAssets } = await u.ai.text.invoke(
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。
|
||||
|
||||
分镜描述：
|
||||
${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")}
|
||||
@ -229,25 +218,27 @@ ${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")}
|
||||
${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")}
|
||||
|
||||
请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`,
|
||||
},
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
jsonSchema: {
|
||||
name: "filteredAssets",
|
||||
strict: true,
|
||||
schema: z.toJSONSchema(filteredAssetsSchema),
|
||||
},
|
||||
],
|
||||
output: {
|
||||
relevantAssets: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().describe("资产名称"),
|
||||
reason: z.string().describe("选择该资产的原因"),
|
||||
}),
|
||||
)
|
||||
.describe("与分镜内容相关的资产列表"),
|
||||
},
|
||||
},
|
||||
});
|
||||
apiConfig,
|
||||
);
|
||||
|
||||
const data = result?.json as z.infer<typeof filteredAssetsSchema>;
|
||||
|
||||
if (!data?.relevantAssets || data.relevantAssets.length === 0) {
|
||||
if (!relevantAssets || relevantAssets.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const relevantNames = new Set(data.relevantAssets.map((a) => a.name));
|
||||
const relevantNames = new Set(relevantAssets.map((a) => a.name));
|
||||
const filteredImages = availableImages.filter((img) => relevantNames.has(img.name));
|
||||
|
||||
return filteredImages.length > 0 ? filteredImages : availableImages;
|
||||
@ -317,14 +308,18 @@ export default async (cells: { prompt: string }[], scriptId: number, projectId:
|
||||
console.log("====润色后：", prompts);
|
||||
|
||||
const processedImages = await processImages(filteredImages);
|
||||
const apiConfig = await u.getPromptAi("storyboardImage");
|
||||
|
||||
const contentStr = await u.ai.generateImage({
|
||||
systemPrompt: resourcesMapPrompts,
|
||||
prompt: prompts,
|
||||
size: "4K",
|
||||
aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9",
|
||||
imageBase64: processedImages.map((buf) => buf.toString("base64")),
|
||||
});
|
||||
const contentStr = await u.ai.image(
|
||||
{
|
||||
systemPrompt: resourcesMapPrompts,
|
||||
prompt: prompts,
|
||||
size: "4K",
|
||||
aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9",
|
||||
imageBase64: processedImages.map((buf) => buf.toString("base64")),
|
||||
},
|
||||
apiConfig,
|
||||
);
|
||||
|
||||
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
const base64Str = match?.[1] ?? contentStr;
|
||||
|
||||
@ -1,13 +1,13 @@
|
||||
// @/agents/Storyboard.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { tool, ModelMessage, Tool } from "ai";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
import generateImageTool from "./generateImageTool";
|
||||
import imageSplitting from "./imageSplitting";
|
||||
import path from "path";
|
||||
import sharp from "sharp";
|
||||
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
@ -38,15 +38,20 @@ interface Shot {
|
||||
x: number;
|
||||
y: number;
|
||||
cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头
|
||||
fragmentContent: string;
|
||||
assetsTags: AssetsType[];
|
||||
}
|
||||
interface AssetsType {
|
||||
type: "role" | "props" | "scene";
|
||||
text: string;
|
||||
}
|
||||
|
||||
// ==================== 主类 ====================
|
||||
|
||||
export default class Storyboard {
|
||||
private readonly projectId: number;
|
||||
private readonly scriptId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
history: ModelMessage[] = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
// 存储 segmentAgent 生成的片段结果
|
||||
@ -58,10 +63,6 @@ export default class Storyboard {
|
||||
// 存储正在生成分镜图的分镜ID
|
||||
private generatingShots: Set<number> = new Set();
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number, scriptId: number) {
|
||||
this.projectId = projectId;
|
||||
this.scriptId = scriptId;
|
||||
@ -69,9 +70,6 @@ export default class Storyboard {
|
||||
|
||||
// 更新shopts
|
||||
public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) {
|
||||
console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId);
|
||||
console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId);
|
||||
console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell);
|
||||
const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId);
|
||||
if (shotIndex === -1) {
|
||||
return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`;
|
||||
@ -105,28 +103,28 @@ export default class Storyboard {
|
||||
|
||||
// ==================== 剧本相关操作 ====================
|
||||
|
||||
getScript = tool(
|
||||
async () => {
|
||||
getScript = tool({
|
||||
title: "getScript",
|
||||
description: "获取剧本内容",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
||||
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
if (!script) throw new Error("剧本不存在");
|
||||
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
||||
},
|
||||
{
|
||||
name: "getScript",
|
||||
description: "获取剧本内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== 资产相关操作 ====================
|
||||
|
||||
/**
|
||||
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
||||
*/
|
||||
getAssets = tool(
|
||||
async () => {
|
||||
getAssets = tool({
|
||||
title: "getAssets",
|
||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
||||
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
||||
@ -171,76 +169,84 @@ ${sections.join("\n\n")}
|
||||
2. 禁止在资产名称前后添加修饰词
|
||||
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
||||
},
|
||||
{
|
||||
name: "getAssets",
|
||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== 片段和分镜工具 ====================
|
||||
|
||||
/**
|
||||
* 获取当前存储的片段数据（供 shotAgent 调用）
|
||||
*/
|
||||
getSegments = tool(
|
||||
async () => {
|
||||
getSegments = tool({
|
||||
title: "getSegments",
|
||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
||||
if (this.segments.length === 0) {
|
||||
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
||||
}
|
||||
return JSON.stringify(this.segments, null, 2);
|
||||
},
|
||||
{
|
||||
name: "getSegments",
|
||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 更新/存储片段数据（供 segmentAgent 调用）
|
||||
*/
|
||||
updateSegments = tool(
|
||||
async ({ segments }: { segments: Segment[] }) => {
|
||||
updateSegments = tool({
|
||||
title: "updateSegments",
|
||||
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||
inputSchema: z.object({
|
||||
segments: z
|
||||
.array(
|
||||
z.object({
|
||||
index: z.number().describe("片段序号"),
|
||||
description: z.string().describe("片段描述"),
|
||||
emotion: z.string().optional().describe("情绪氛围"),
|
||||
action: z.string().optional().describe("主要动作"),
|
||||
}),
|
||||
)
|
||||
.describe("片段数组"),
|
||||
}),
|
||||
execute: async ({ segments }: { segments: Segment[] }) => {
|
||||
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||
this.segments = segments;
|
||||
this.emit("segmentsUpdated", this.segments);
|
||||
return `成功存储 ${segments.length} 个片段`;
|
||||
},
|
||||
{
|
||||
name: "updateSegments",
|
||||
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||
schema: z.object({
|
||||
segments: z
|
||||
.array(
|
||||
z.object({
|
||||
index: z.number().describe("片段序号"),
|
||||
description: z.string().describe("片段描述"),
|
||||
emotion: z.string().optional().describe("情绪氛围"),
|
||||
action: z.string().optional().describe("主要动作"),
|
||||
}),
|
||||
)
|
||||
.describe("片段数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 添加分镜（供 shotAgent 调用）
|
||||
*/
|
||||
addShots = tool(
|
||||
async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||
addShots = tool({
|
||||
title: "addShots",
|
||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||
inputSchema: z.object({
|
||||
shots: z
|
||||
.array(
|
||||
z.object({
|
||||
segmentIndex: z.number().describe("对应的片段序号"),
|
||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||
assetsTags: z.array(
|
||||
z.object({
|
||||
type: z.enum(["role", "props", "scene"]).describe("资源类型"),
|
||||
text: z.string().describe("资源名称"),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
)
|
||||
.describe("要添加的分镜数组"),
|
||||
}),
|
||||
execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[]; assetsTags: AssetsType[] }> }) => {
|
||||
const added: { id: number; segmentIndex: number }[] = [];
|
||||
const skipped: number[] = [];
|
||||
|
||||
for (const item of shots) {
|
||||
const exists = this.shots.some((f) => f.segmentId === item.segmentIndex);
|
||||
const resultIndex = item.segmentIndex - 1;
|
||||
|
||||
const exists = this.shots.some((f) => f.segmentId === resultIndex);
|
||||
if (exists) {
|
||||
skipped.push(item.segmentIndex);
|
||||
skipped.push(resultIndex);
|
||||
continue;
|
||||
}
|
||||
// 分配独立的分镜ID
|
||||
@ -248,13 +254,15 @@ ${sections.join("\n\n")}
|
||||
const shotId = this.shotIdCounter;
|
||||
this.shots.push({
|
||||
id: shotId,
|
||||
segmentId: item.segmentIndex,
|
||||
segmentId: resultIndex,
|
||||
title: `分镜 ${shotId}`,
|
||||
x: 0,
|
||||
y: 0,
|
||||
cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })),
|
||||
fragmentContent: this.segments[resultIndex]?.description,
|
||||
assetsTags: item.assetsTags,
|
||||
});
|
||||
added.push({ id: shotId, segmentIndex: item.segmentIndex });
|
||||
added.push({ id: shotId, segmentIndex: resultIndex });
|
||||
}
|
||||
|
||||
const addedInfo = added.map((a) => `分镜${a.id}(片段${a.segmentIndex})`).join(", ");
|
||||
@ -266,29 +274,20 @@ ${sections.join("\n\n")}
|
||||
}
|
||||
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "addShots",
|
||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||
schema: z.object({
|
||||
shots: z
|
||||
.array(
|
||||
z.object({
|
||||
segmentIndex: z.number().describe("对应的片段序号"),
|
||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||
}),
|
||||
)
|
||||
.describe("要添加的分镜数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 更新指定分镜（供 shotAgent 调用）
|
||||
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
||||
*/
|
||||
updateShots = tool(
|
||||
async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||
updateShots = tool({
|
||||
title: "updateShots",
|
||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||
inputSchema: z.object({
|
||||
shotId: z.number().describe("要更新的分镜ID"),
|
||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||
}),
|
||||
execute: async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
||||
|
||||
if (existingIndex === -1) {
|
||||
@ -314,22 +313,18 @@ ${sections.join("\n\n")}
|
||||
|
||||
return `已更新分镜 ${shotId}`;
|
||||
},
|
||||
{
|
||||
name: "updateShots",
|
||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||
schema: z.object({
|
||||
shotId: z.number().describe("要更新的分镜ID"),
|
||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 删除指定分镜（供 shotAgent 调用）
|
||||
*/
|
||||
deleteShots = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
deleteShots = tool({
|
||||
title: "deleteShots",
|
||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||
inputSchema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||
}),
|
||||
execute: async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const deleted: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
|
||||
@ -351,21 +346,19 @@ ${sections.join("\n\n")}
|
||||
}
|
||||
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "deleteShots",
|
||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 生成分镜图（异步执行，使用 nanoBanana）
|
||||
*/
|
||||
generateShotImage = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
generateShotImage = tool({
|
||||
title: "generateShotImage",
|
||||
description:
|
||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||
inputSchema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||
}),
|
||||
execute: async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const toGenerate: number[] = [];
|
||||
const alreadyGenerating: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
@ -417,16 +410,7 @@ ${sections.join("\n\n")}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
{
|
||||
name: "generateShotImage",
|
||||
description:
|
||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 执行分镜图生成的具体逻辑（异步并发）
|
||||
@ -462,7 +446,6 @@ ${sections.join("\n\n")}
|
||||
this.scriptId,
|
||||
this.projectId,
|
||||
);
|
||||
|
||||
// 通知前端正在分割图片
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "splitting", message: "正在分割宫格图片为单张镜头图" });
|
||||
|
||||
@ -566,7 +549,7 @@ ${assetList}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
@ -586,26 +569,33 @@ ${task}
|
||||
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取不同 Sub-Agent 可用的工具
|
||||
*/
|
||||
private getSubAgentTools(agentType: AgentType) {
|
||||
private getSubAgentTools(agentType: AgentType): Record<string, Tool> {
|
||||
switch (agentType) {
|
||||
case "segmentAgent":
|
||||
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
||||
return [this.getScript, this.getAssets, this.updateSegments];
|
||||
return {
|
||||
getScript: this.getScript,
|
||||
getAssets: this.getAssets,
|
||||
updateSegments: this.updateSegments,
|
||||
};
|
||||
case "shotAgent":
|
||||
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
||||
return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage];
|
||||
return {
|
||||
getScript: this.getScript,
|
||||
getAssets: this.getAssets,
|
||||
getSegments: this.getSegments,
|
||||
addShots: this.addShots,
|
||||
updateShots: this.updateShots,
|
||||
deleteShots: this.deleteShots,
|
||||
generateShotImage: this.generateShotImage,
|
||||
};
|
||||
default:
|
||||
return [this.getScript];
|
||||
return {
|
||||
getScript: this.getScript,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -617,119 +607,124 @@ ${task}
|
||||
this.log(`Sub-Agent 调用`, agentType);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]);
|
||||
const segmentAgent = promptsList.find((p) => p.code === "storyboard-segment");
|
||||
const shotAgent = promptsList.find((p) => p.code === "storyboard-shot");
|
||||
const promptConfig = await u.getPromptAi("storyboardAgent");
|
||||
|
||||
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
segmentAgent: segmentAgent?.customValue || segmentAgent?.defaultValue || errPrompts,
|
||||
shotAgent: shotAgent?.customValue || shotAgent?.defaultValue || errPrompts,
|
||||
|
||||
const getAiPromptConfig = (code: string) => {
|
||||
const item = promptsList.find((p) => p.code === code);
|
||||
return item?.customValue || item?.defaultValue || errPrompts;
|
||||
};
|
||||
const segmentAgent = getAiPromptConfig("storyboard-segment");
|
||||
const shotAgent = getAiPromptConfig("storyboard-shot");
|
||||
const SYSTEM_PROMPTS = {
|
||||
segmentAgent: segmentAgent,
|
||||
shotAgent: shotAgent,
|
||||
};
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(agentType),
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
const { fullStream } = await u.ai.text.stream(
|
||||
{
|
||||
system: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(agentType),
|
||||
messages: [{ role: "user", content: context }],
|
||||
maxStep: 100,
|
||||
},
|
||||
promptConfig,
|
||||
);
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: item.text });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
return fullResponse;
|
||||
|
||||
return fullResponse ?? `${agentType}已完成任务`;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
return tool({
|
||||
title: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
inputSchema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool(
|
||||
return {
|
||||
segmentAgent: this.createSubAgentTool(
|
||||
"segmentAgent",
|
||||
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
||||
),
|
||||
this.createSubAgentTool(
|
||||
shotAgent: this.createSubAgentTool(
|
||||
"shotAgent",
|
||||
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
||||
),
|
||||
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getScript,
|
||||
this.getSegments,
|
||||
this.generateShotImage,
|
||||
getScript: this.getScript,
|
||||
getSegments: this.getSegments,
|
||||
generateShotImage: this.generateShotImage,
|
||||
...this.getSubAgentTools("segmentAgent"),
|
||||
...this.getSubAgentTools("shotAgent"),
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
console.log("模型名称:", this.modelName);
|
||||
this.history.push(["user", msg]);
|
||||
this.history.push({
|
||||
role: "user",
|
||||
content: msg,
|
||||
});
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "storyboard-main").first();
|
||||
const promptConfig = await u.getPromptAi("storyboardAgent");
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
const { fullStream } = await u.ai.text.stream(
|
||||
{
|
||||
system: `${envContext}\n${mainPrompts}`,
|
||||
tools: this.getAllTools(),
|
||||
messages: this.history,
|
||||
maxStep: 100,
|
||||
},
|
||||
promptConfig,
|
||||
);
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("data", item.text);
|
||||
}
|
||||
}
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import "./logger";
|
||||
import "./err";
|
||||
import "./env";
|
||||
import express, { Request, Response, NextFunction } from "express";
|
||||
@ -6,7 +7,6 @@ import logger from "morgan";
|
||||
import cors from "cors";
|
||||
import buildRoute from "@/core";
|
||||
import fs from "fs";
|
||||
import router from "@/router";
|
||||
import path from "path";
|
||||
import u from "@/utils";
|
||||
import jwt from "jsonwebtoken";
|
||||
@ -32,6 +32,7 @@ export default async function startServe() {
|
||||
} else {
|
||||
rootDir = path.join(process.cwd(), "uploads");
|
||||
}
|
||||
|
||||
// 确保 uploads 目录存在
|
||||
if (!fs.existsSync(rootDir)) {
|
||||
fs.mkdirSync(rootDir, { recursive: true });
|
||||
@ -60,7 +61,8 @@ export default async function startServe() {
|
||||
}
|
||||
});
|
||||
|
||||
await router(app);
|
||||
const router = await import("@/router");
|
||||
await router.default(app);
|
||||
|
||||
// 404 处理
|
||||
app.use((_, res, next: NextFunction) => {
|
||||
|
||||
48
src/env.ts
48
src/env.ts
@ -1,31 +1,35 @@
|
||||
import { readFileSync, existsSync } from "fs";
|
||||
import { readFileSync, existsSync, writeFileSync, mkdirSync } from "fs";
|
||||
import path from "path";
|
||||
|
||||
function loadDotenvESM(envPath = ".env.local") {
|
||||
// 尝试从 userData 目录读取环境变量，如果不存在则使用当前目录
|
||||
let finalPath: string;
|
||||
// 默认环境变量（当 env 文件不存在时自动创建）
|
||||
const defaultEnvValues: Record<string, string> = {
|
||||
dev: `NODE_ENV=dev\nPORT=60000\nOSSURL=http://127.0.0.1:60000/`,
|
||||
prod: `NODE_ENV=prod\nPORT=60000\nOSSURL=http://127.0.0.1:60000/`,
|
||||
};
|
||||
|
||||
if (typeof process.versions?.electron !== "undefined") {
|
||||
const { app } = require("electron");
|
||||
finalPath = app.getPath("userData");
|
||||
// 如果 userData 目录中不存在，尝试使用当前目录
|
||||
if (!existsSync(finalPath)) {
|
||||
finalPath = envPath;
|
||||
}
|
||||
} else {
|
||||
finalPath = envPath;
|
||||
//加载环境变量
|
||||
const env = process.env.NODE_ENV ?? "dev";
|
||||
if (!env) {
|
||||
console.log("[环境变量为空]");
|
||||
process.exit(1);
|
||||
} else {
|
||||
const envDir = path.resolve("env");
|
||||
const envFilePath = path.join(envDir, `.env.${env}`);
|
||||
|
||||
// 自动创建 env 目录和文件（.gitignore 可能忽略了这些文件）
|
||||
if (!existsSync(envDir)) {
|
||||
mkdirSync(envDir, { recursive: true });
|
||||
}
|
||||
if (!existsSync(envFilePath)) {
|
||||
const content = defaultEnvValues[env] ?? defaultEnvValues.prod;
|
||||
writeFileSync(envFilePath, content, "utf8");
|
||||
console.log(`[环境变量] 自动创建 ${envFilePath}`);
|
||||
}
|
||||
|
||||
if (!existsSync(finalPath)) {
|
||||
console.log(`[环境变量]: ${envPath} 文件不存在`);
|
||||
return;
|
||||
}
|
||||
|
||||
const text = readFileSync(finalPath, "utf8");
|
||||
const text = readFileSync(envFilePath, "utf8");
|
||||
for (const line of text.split("\n")) {
|
||||
const idx = line.indexOf("=");
|
||||
if (idx > 0) process.env[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
|
||||
}
|
||||
console.log(`[环境变量]: ${finalPath}`);
|
||||
console.log(`[环境变量] ${env}`);
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV == "dev") loadDotenvESM(".env.local");
|
||||
|
||||
24
src/err.ts
24
src/err.ts
@ -1,10 +1,30 @@
|
||||
import { serializeError } from "serialize-error";
|
||||
|
||||
// 处理未捕获的 Promise 拒绝
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
console.error('[未处理的 Promise 拒绝]:', reason);
|
||||
console.error('[未处理的 Promise 拒绝]');
|
||||
if (reason instanceof Error) {
|
||||
console.error('错误名称:', reason.name);
|
||||
console.error('错误消息:', reason.message);
|
||||
console.error('堆栈信息:', reason.stack);
|
||||
console.error('序列化详情:', JSON.stringify(serializeError(reason), null, 2));
|
||||
} else {
|
||||
console.error('原因:', reason);
|
||||
console.error('类型:', typeof reason);
|
||||
try {
|
||||
console.error('JSON:', JSON.stringify(reason, null, 2));
|
||||
} catch {
|
||||
console.error('(无法序列化)');
|
||||
}
|
||||
}
|
||||
console.error('Promise:', promise);
|
||||
});
|
||||
|
||||
// 处理未捕获的异常
|
||||
process.on('uncaughtException', (error) => {
|
||||
console.error('[未捕获的异常]:', error);
|
||||
console.error('[未捕获的异常]');
|
||||
console.error('错误名称:', error.name);
|
||||
console.error('错误消息:', error.message);
|
||||
console.error('堆栈信息:', error.stack);
|
||||
console.error('序列化详情:', JSON.stringify(serializeError(error), null, 2));
|
||||
});
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
156
src/logger.ts
Normal file
156
src/logger.ts
Normal file
@ -0,0 +1,156 @@
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
type LogLevel = "log" | "info" | "warn" | "error" | "debug";
|
||||
type ConsoleMethod = (...args: unknown[]) => void;
|
||||
|
||||
function getLogDir(): string {
|
||||
const isElectron = typeof process.versions?.electron !== "undefined";
|
||||
if (isElectron) {
|
||||
const { app } = require("electron");
|
||||
return path.join(app.getPath("userData"), "logs");
|
||||
}
|
||||
return path.join(process.cwd(), "logs");
|
||||
}
|
||||
|
||||
const LOG_DIR = getLogDir();
|
||||
const LOG_FILE = path.join(LOG_DIR, "app.log");
|
||||
const MAX_SIZE = 1000 * 1024 * 1024;
|
||||
const LEVELS: LogLevel[] = ["log", "info", "warn", "error", "debug"];
|
||||
|
||||
class Logger {
|
||||
private stream: fs.WriteStream | null = null;
|
||||
private originalConsole: Partial<Record<LogLevel, ConsoleMethod>> = {};
|
||||
private originalStdoutWrite: typeof process.stdout.write | null = null;
|
||||
private originalStderrWrite: typeof process.stderr.write | null = null;
|
||||
private isHijacked = false;
|
||||
|
||||
init(): this {
|
||||
if (!fs.existsSync(LOG_DIR)) fs.mkdirSync(LOG_DIR, { recursive: true });
|
||||
this.stream = fs.createWriteStream(LOG_FILE, { flags: "a" });
|
||||
this.hijack();
|
||||
return this;
|
||||
}
|
||||
|
||||
private formatTime(): string {
|
||||
const d = new Date();
|
||||
const p = (n: number, l = 2) => String(n).padStart(l, "0");
|
||||
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}.${p(
|
||||
d.getMilliseconds(),
|
||||
3,
|
||||
)}`;
|
||||
}
|
||||
|
||||
private stringify(arg: unknown): string {
|
||||
if (arg == null) return String(arg);
|
||||
if (arg instanceof Error) return `${arg.message}\n${arg.stack || ""}`;
|
||||
if (typeof arg === "object") {
|
||||
try {
|
||||
return JSON.stringify(arg);
|
||||
} catch {
|
||||
return String(arg);
|
||||
}
|
||||
}
|
||||
return String(arg);
|
||||
}
|
||||
|
||||
private writing = false;
|
||||
|
||||
private write(level: LogLevel, args: unknown[]): void {
|
||||
const line = `[${this.formatTime()}] [${level.toUpperCase()}] ${args.map((a) => this.stringify(a)).join(" ")}\n`;
|
||||
if (this.stream && !this.stream.destroyed) this.stream.write(line);
|
||||
this.checkRotate();
|
||||
}
|
||||
|
||||
private writeRaw(chunk: any): void {
|
||||
if (this.writing) return;
|
||||
this.writing = true;
|
||||
try {
|
||||
let str = typeof chunk === "string" ? chunk : chunk?.toString?.("utf-8") ?? "";
|
||||
str = str.replace(/\x1B\[\d*m/g, ""); // 去除 ANSI 颜色码
|
||||
if (str.trim() && this.stream && !this.stream.destroyed) this.stream.write(str.endsWith("\n") ? str : str + "\n");
|
||||
} finally {
|
||||
this.writing = false;
|
||||
}
|
||||
}
|
||||
|
||||
private checkRotate(): void {
|
||||
try {
|
||||
if (!fs.existsSync(LOG_FILE) || fs.statSync(LOG_FILE).size < MAX_SIZE) return;
|
||||
this.stream?.end();
|
||||
// 单文件轮转：保留后半部分日志
|
||||
const content = fs.readFileSync(LOG_FILE, "utf-8");
|
||||
const half = content.slice(content.length >>> 1);
|
||||
const firstNewline = half.indexOf("\n");
|
||||
fs.writeFileSync(LOG_FILE, firstNewline >= 0 ? half.slice(firstNewline + 1) : half);
|
||||
this.stream = fs.createWriteStream(LOG_FILE, { flags: "a" });
|
||||
} catch {}
|
||||
}
|
||||
|
||||
private hijack(): void {
|
||||
if (this.isHijacked) return;
|
||||
|
||||
// 劫持 console 方法
|
||||
for (const level of LEVELS) {
|
||||
const original = console[level];
|
||||
if (typeof original !== "function") continue;
|
||||
this.originalConsole[level] = original.bind(console);
|
||||
(console as any)[level] = (...args: unknown[]) => {
|
||||
this.writing = true;
|
||||
this.write(level, args);
|
||||
this.originalConsole[level]!(...args);
|
||||
this.writing = false;
|
||||
};
|
||||
}
|
||||
|
||||
// 劫持 stdout/stderr（捕获 morgan 等直接写 stdout 的输出）
|
||||
this.originalStdoutWrite = process.stdout.write.bind(process.stdout);
|
||||
this.originalStderrWrite = process.stderr.write.bind(process.stderr);
|
||||
|
||||
process.stdout.write = ((chunk: any, ...rest: any[]) => {
|
||||
this.writeRaw(chunk);
|
||||
return this.originalStdoutWrite!(chunk, ...rest);
|
||||
}) as typeof process.stdout.write;
|
||||
|
||||
process.stderr.write = ((chunk: any, ...rest: any[]) => {
|
||||
this.writeRaw(chunk);
|
||||
return this.originalStderrWrite!(chunk, ...rest);
|
||||
}) as typeof process.stderr.write;
|
||||
|
||||
this.isHijacked = true;
|
||||
}
|
||||
|
||||
/** 导出日志内容 */
|
||||
exportLogs(): string {
|
||||
if (!fs.existsSync(LOG_FILE)) return "";
|
||||
return fs.readFileSync(LOG_FILE, "utf-8");
|
||||
}
|
||||
|
||||
/** 清空日志 */
|
||||
clear(): void {
|
||||
this.stream?.end();
|
||||
if (fs.existsSync(LOG_FILE)) fs.unlinkSync(LOG_FILE);
|
||||
this.stream = fs.createWriteStream(LOG_FILE, { flags: "a" });
|
||||
}
|
||||
|
||||
/** 关闭日志 */
|
||||
close(): void {
|
||||
if (this.isHijacked) {
|
||||
for (const level of LEVELS) {
|
||||
const original = this.originalConsole[level];
|
||||
if (original) (console as any)[level] = original;
|
||||
}
|
||||
this.originalConsole = {};
|
||||
if (this.originalStdoutWrite) process.stdout.write = this.originalStdoutWrite;
|
||||
if (this.originalStderrWrite) process.stderr.write = this.originalStderrWrite;
|
||||
this.originalStdoutWrite = null;
|
||||
this.originalStderrWrite = null;
|
||||
this.isHijacked = false;
|
||||
}
|
||||
this.stream?.end();
|
||||
this.stream = null;
|
||||
}
|
||||
}
|
||||
|
||||
const logger = new Logger().init();
|
||||
export default logger;
|
||||
124
src/router.ts
124
src/router.ts
@ -1,4 +1,4 @@
|
||||
// @routes-hash 4c67f0d89475e1a882416d9f6ab1687d
|
||||
// @routes-hash 3cfad40b3c8658b442ab766a9323d740
|
||||
import { Express } from "express";
|
||||
|
||||
import route1 from "./routes/assets/addAssets";
|
||||
@ -44,33 +44,40 @@ import route40 from "./routes/prompt/updatePrompt";
|
||||
import route41 from "./routes/script/generateScriptApi";
|
||||
import route42 from "./routes/script/generateScriptSave";
|
||||
import route43 from "./routes/script/geScriptApi";
|
||||
import route44 from "./routes/setting/getSetting";
|
||||
import route45 from "./routes/setting/updateSetting";
|
||||
import route46 from "./routes/storyboard/batchSuperScoreImage";
|
||||
import route47 from "./routes/storyboard/chatStoryboard";
|
||||
import route48 from "./routes/storyboard/generateShotImage";
|
||||
import route49 from "./routes/storyboard/generateStoryboardApi";
|
||||
import route50 from "./routes/storyboard/generateVideoPrompt";
|
||||
import route51 from "./routes/storyboard/getStoryboard";
|
||||
import route52 from "./routes/storyboard/keepStoryboard";
|
||||
import route53 from "./routes/storyboard/saveStoryboard";
|
||||
import route54 from "./routes/storyboard/uploadImage";
|
||||
import route55 from "./routes/task/getTaskApi";
|
||||
import route56 from "./routes/task/taskDetails";
|
||||
import route57 from "./routes/user/getUser";
|
||||
import route58 from "./routes/video/addVideo";
|
||||
import route59 from "./routes/video/addVideoConfig";
|
||||
import route60 from "./routes/video/deleteVideoConfig";
|
||||
import route61 from "./routes/video/generatePrompt";
|
||||
import route62 from "./routes/video/generateVideo";
|
||||
import route63 from "./routes/video/getManufacturer";
|
||||
import route64 from "./routes/video/getVideo";
|
||||
import route65 from "./routes/video/getVideoConfigs";
|
||||
import route66 from "./routes/video/getVideoModel";
|
||||
import route67 from "./routes/video/getVideoStoryboards";
|
||||
import route68 from "./routes/video/reviseVideoStoryboards";
|
||||
import route69 from "./routes/video/saveVideo";
|
||||
import route70 from "./routes/video/upDateVideoConfig";
|
||||
import route44 from "./routes/setting/addModel";
|
||||
import route45 from "./routes/setting/configurationModel";
|
||||
import route46 from "./routes/setting/delModel";
|
||||
import route47 from "./routes/setting/getAiModelMap";
|
||||
import route48 from "./routes/setting/getLog";
|
||||
import route49 from "./routes/setting/getSetting";
|
||||
import route50 from "./routes/setting/getVideoModelList";
|
||||
import route51 from "./routes/setting/updateModel";
|
||||
import route52 from "./routes/setting/updeteModel";
|
||||
import route53 from "./routes/storyboard/batchSuperScoreImage";
|
||||
import route54 from "./routes/storyboard/chatStoryboard";
|
||||
import route55 from "./routes/storyboard/generateShotImage";
|
||||
import route56 from "./routes/storyboard/generateStoryboardApi";
|
||||
import route57 from "./routes/storyboard/generateVideoPrompt";
|
||||
import route58 from "./routes/storyboard/getStoryboard";
|
||||
import route59 from "./routes/storyboard/keepStoryboard";
|
||||
import route60 from "./routes/storyboard/saveStoryboard";
|
||||
import route61 from "./routes/storyboard/uploadImage";
|
||||
import route62 from "./routes/task/getTaskApi";
|
||||
import route63 from "./routes/task/taskDetails";
|
||||
import route64 from "./routes/user/getUser";
|
||||
import route65 from "./routes/video/addVideo";
|
||||
import route66 from "./routes/video/addVideoConfig";
|
||||
import route67 from "./routes/video/deleteVideoConfig";
|
||||
import route68 from "./routes/video/generatePrompt";
|
||||
import route69 from "./routes/video/generateVideo";
|
||||
import route70 from "./routes/video/getManufacturer";
|
||||
import route71 from "./routes/video/getVideo";
|
||||
import route72 from "./routes/video/getVideoConfigs";
|
||||
import route73 from "./routes/video/getVideoModel";
|
||||
import route74 from "./routes/video/getVideoStoryboards";
|
||||
import route75 from "./routes/video/reviseVideoStoryboards";
|
||||
import route76 from "./routes/video/saveVideo";
|
||||
import route77 from "./routes/video/upDateVideoConfig";
|
||||
|
||||
export default async (app: Express) => {
|
||||
app.use("/assets/addAssets", route1);
|
||||
@ -116,31 +123,38 @@ export default async (app: Express) => {
|
||||
app.use("/script/generateScriptApi", route41);
|
||||
app.use("/script/generateScriptSave", route42);
|
||||
app.use("/script/geScriptApi", route43);
|
||||
app.use("/setting/getSetting", route44);
|
||||
app.use("/setting/updateSetting", route45);
|
||||
app.use("/storyboard/batchSuperScoreImage", route46);
|
||||
app.use("/storyboard/chatStoryboard", route47);
|
||||
app.use("/storyboard/generateShotImage", route48);
|
||||
app.use("/storyboard/generateStoryboardApi", route49);
|
||||
app.use("/storyboard/generateVideoPrompt", route50);
|
||||
app.use("/storyboard/getStoryboard", route51);
|
||||
app.use("/storyboard/keepStoryboard", route52);
|
||||
app.use("/storyboard/saveStoryboard", route53);
|
||||
app.use("/storyboard/uploadImage", route54);
|
||||
app.use("/task/getTaskApi", route55);
|
||||
app.use("/task/taskDetails", route56);
|
||||
app.use("/user/getUser", route57);
|
||||
app.use("/video/addVideo", route58);
|
||||
app.use("/video/addVideoConfig", route59);
|
||||
app.use("/video/deleteVideoConfig", route60);
|
||||
app.use("/video/generatePrompt", route61);
|
||||
app.use("/video/generateVideo", route62);
|
||||
app.use("/video/getManufacturer", route63);
|
||||
app.use("/video/getVideo", route64);
|
||||
app.use("/video/getVideoConfigs", route65);
|
||||
app.use("/video/getVideoModel", route66);
|
||||
app.use("/video/getVideoStoryboards", route67);
|
||||
app.use("/video/reviseVideoStoryboards", route68);
|
||||
app.use("/video/saveVideo", route69);
|
||||
app.use("/video/upDateVideoConfig", route70);
|
||||
app.use("/setting/addModel", route44);
|
||||
app.use("/setting/configurationModel", route45);
|
||||
app.use("/setting/delModel", route46);
|
||||
app.use("/setting/getAiModelMap", route47);
|
||||
app.use("/setting/getLog", route48);
|
||||
app.use("/setting/getSetting", route49);
|
||||
app.use("/setting/getVideoModelList", route50);
|
||||
app.use("/setting/updateModel", route51);
|
||||
app.use("/setting/updeteModel", route52);
|
||||
app.use("/storyboard/batchSuperScoreImage", route53);
|
||||
app.use("/storyboard/chatStoryboard", route54);
|
||||
app.use("/storyboard/generateShotImage", route55);
|
||||
app.use("/storyboard/generateStoryboardApi", route56);
|
||||
app.use("/storyboard/generateVideoPrompt", route57);
|
||||
app.use("/storyboard/getStoryboard", route58);
|
||||
app.use("/storyboard/keepStoryboard", route59);
|
||||
app.use("/storyboard/saveStoryboard", route60);
|
||||
app.use("/storyboard/uploadImage", route61);
|
||||
app.use("/task/getTaskApi", route62);
|
||||
app.use("/task/taskDetails", route63);
|
||||
app.use("/user/getUser", route64);
|
||||
app.use("/video/addVideo", route65);
|
||||
app.use("/video/addVideoConfig", route66);
|
||||
app.use("/video/deleteVideoConfig", route67);
|
||||
app.use("/video/generatePrompt", route68);
|
||||
app.use("/video/generateVideo", route69);
|
||||
app.use("/video/getManufacturer", route70);
|
||||
app.use("/video/getVideo", route71);
|
||||
app.use("/video/getVideoConfigs", route72);
|
||||
app.use("/video/getVideoModel", route73);
|
||||
app.use("/video/getVideoStoryboards", route74);
|
||||
app.use("/video/reviseVideoStoryboards", route75);
|
||||
app.use("/video/saveVideo", route76);
|
||||
app.use("/video/upDateVideoConfig", route77);
|
||||
}
|
||||
|
||||
@ -123,18 +123,23 @@ export default router.post(
|
||||
state: "生成中",
|
||||
assetsId: id,
|
||||
});
|
||||
const apiConfig = await u.getPromptAi("assetsImage");
|
||||
|
||||
const contentStr = await u.ai.generateImage({
|
||||
systemPrompt,
|
||||
prompt: userPrompt,
|
||||
imageBase64: base64 ? [base64] : [],
|
||||
size: "2K",
|
||||
aspectRatio: "16:9",
|
||||
});
|
||||
const contentStr = await u.ai.image(
|
||||
{
|
||||
systemPrompt,
|
||||
prompt: userPrompt,
|
||||
imageBase64: base64 ? [base64] : [],
|
||||
size: "2K",
|
||||
aspectRatio: "16:9",
|
||||
},
|
||||
apiConfig,
|
||||
);
|
||||
|
||||
let insertType;
|
||||
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
let buffer = Buffer.from(match && match.length >= 2 ? match[1]! : contentStr!, "base64");
|
||||
|
||||
if (type != "storyboard") {
|
||||
//添加文本
|
||||
// buffer = await imageAddText(name, buffer);
|
||||
@ -152,6 +157,11 @@ export default router.post(
|
||||
insertType = "道具";
|
||||
imagePath = `/${projectId}/props/${uuidv4()}.jpg`;
|
||||
}
|
||||
if (type == "storyboard") {
|
||||
insertType = "分镜";
|
||||
imagePath = `/${projectId}/storyboard/${uuidv4()}.jpg`;
|
||||
}
|
||||
|
||||
await u.oss.writeFile(imagePath!, buffer);
|
||||
|
||||
await u.db("t_image").where("id", imageId).update({
|
||||
|
||||
@ -1,12 +1,9 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import * as zod from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { error, success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
const jsonSchema = zod.object({
|
||||
prompt: zod.string().describe("提示词"),
|
||||
});
|
||||
interface OutlineItem {
|
||||
description: string;
|
||||
name: string;
|
||||
@ -88,8 +85,9 @@ export default router.post(
|
||||
const result: ResultItem[] = Object.values(itemMap);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["role-polish", "scene-polish", "storyboard-polish", "tool-polish"]);
|
||||
const apiConfigData = await u.getPromptAi("assetsPrompt");
|
||||
const errPrompts = "不论用户说什么，请直接输出AI配置异常";
|
||||
const getPromptValue = (code: string): string => {
|
||||
const getPromptValue = (code: string) => {
|
||||
const item = promptsList.find((p) => p.code === code);
|
||||
return item?.customValue ?? item?.defaultValue ?? errPrompts;
|
||||
};
|
||||
@ -97,7 +95,6 @@ export default router.post(
|
||||
const scene = getPromptValue("scene-polish");
|
||||
const tool = getPromptValue("tool-polish");
|
||||
const storyboard = getPromptValue("storyboard-polish");
|
||||
|
||||
let systemPrompt = "";
|
||||
let userPrompt = "";
|
||||
if (type == "role") {
|
||||
@ -125,6 +122,7 @@ export default router.post(
|
||||
}
|
||||
if (type == "scene") {
|
||||
const data = findItemByName(result, name, "scenes");
|
||||
|
||||
const chapterRange = Array.isArray(data?.chapterRange) ? data.chapterRange : [data?.chapterRange];
|
||||
const novelData = (await u.db("t_novel").whereIn("chapterIndex", chapterRange).select("*")) as NovelChapter[];
|
||||
const results: string = mergeNovelText(novelData);
|
||||
@ -188,33 +186,33 @@ export default router.post(
|
||||
`;
|
||||
}
|
||||
async function generatePrompt() {
|
||||
const model = await u.ai.text();
|
||||
const result = await model.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: systemPrompt,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: userPrompt,
|
||||
},
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
jsonSchema: {
|
||||
name: "json",
|
||||
strict: true,
|
||||
schema: zod.toJSONSchema(jsonSchema),
|
||||
const result = await u.ai.text.invoke(
|
||||
{
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: systemPrompt,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: userPrompt,
|
||||
},
|
||||
],
|
||||
output: {
|
||||
prompt: zod.string().describe("提示词"),
|
||||
},
|
||||
},
|
||||
});
|
||||
return result.json;
|
||||
apiConfigData,
|
||||
);
|
||||
return result.prompt;
|
||||
}
|
||||
const data = (await generatePrompt()) as any;
|
||||
try {
|
||||
const prompt = (await generatePrompt()) as any;
|
||||
if (!prompt) return res.status(500).send("失败");
|
||||
|
||||
if (!data.prompt) return res.status(500).send("失败");
|
||||
|
||||
res.status(200).send(success({ prompt: data.prompt, assetsId }));
|
||||
res.status(200).send(success({ prompt: prompt, assetsId }));
|
||||
} catch (e: any) {
|
||||
return res.status(500).send(error(e?.data?.error?.message ?? e?.message ?? "生成失败"));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -46,10 +46,10 @@ export default router.post(
|
||||
}
|
||||
|
||||
// 检查图片表里是否有这条图片
|
||||
const selectedImage = await u.db("t_image").where("filePath", savePath).first();
|
||||
if (!selectedImage) {
|
||||
return res.status(404).send({ success: false, message: "所选图片不存在，请重新生成或选定图片" });
|
||||
}
|
||||
// const selectedImage = await u.db("t_image").where("filePath", savePath).first();
|
||||
// if (!selectedImage) {
|
||||
// return res.status(500).send({ success: false, message: "所选图片不存在，请重新生成或选定图片" });
|
||||
// }
|
||||
imageUrl = savePath;
|
||||
}
|
||||
|
||||
@ -78,7 +78,9 @@ export default router.post(
|
||||
}
|
||||
|
||||
// 更新提示信息
|
||||
await u.db("t_assets").where("id", id).update({ prompt });
|
||||
if (prompt !== undefined && prompt !== null && prompt !== "") {
|
||||
await u.db("t_assets").where("id", id).update({ prompt });
|
||||
}
|
||||
|
||||
res.status(200).send(success({ message: "保存资产图片成功" }));
|
||||
},
|
||||
|
||||
@ -3,12 +3,9 @@ import { success, error } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { generateText, Output, tool, stepCountIs } from "ai";
|
||||
import { tool } from "ai";
|
||||
const router = express.Router();
|
||||
|
||||
import { createOpenAI } from "@ai-sdk/openai";
|
||||
import { createDeepSeek } from "@ai-sdk/deepseek";
|
||||
|
||||
// 检查语言模型
|
||||
export default router.post(
|
||||
"/",
|
||||
@ -16,12 +13,12 @@ export default router.post(
|
||||
modelName: z.string(),
|
||||
apiKey: z.string(),
|
||||
baseURL: z.string().optional(),
|
||||
manufacturer: z.string(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { modelName, apiKey, baseURL } = req.body;
|
||||
const { modelName, apiKey, baseURL, manufacturer } = req.body;
|
||||
|
||||
const getWeatherTool = tool({
|
||||
// strict: true,
|
||||
description: "Get the weather in a location",
|
||||
inputSchema: z.object({
|
||||
location: z.string().describe("The location to get the weather for"),
|
||||
@ -46,15 +43,14 @@ export default router.post(
|
||||
model: modelName,
|
||||
apiKey,
|
||||
baseURL,
|
||||
manufacturer,
|
||||
},
|
||||
);
|
||||
console.log("%c Line:52 🍐 reply", "background:#ffdd4d", reply);
|
||||
res.status(200).send(success(reply));
|
||||
} catch (err) {
|
||||
console.log(err);
|
||||
if (typeof err === "string") return res.status(500).send(error(err));
|
||||
const msg = err instanceof Error ? err.message : (err as any)?.error?.message;
|
||||
return res.status(500).send(error(msg || "未知错误"));
|
||||
const msg = u.error(err).message;
|
||||
console.error(msg);
|
||||
res.status(500).send(error(msg));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -17,9 +17,10 @@ export default router.post(
|
||||
async (req, res) => {
|
||||
const { modelName, apiKey, baseURL, manufacturer } = req.body;
|
||||
try {
|
||||
const contentStr = await u.ai.generateImage(
|
||||
const image = await u.ai.image(
|
||||
{
|
||||
prompt: "2D cat",
|
||||
prompt:
|
||||
"一张16:9比例的图片，完美等分为2x2四宫格布局，各区域无缝衔接：\n左上宫格：一只可爱的猫，毛发蓬松，眼睛明亮，姿态俏皮\n右上宫格：一只友善的狗，金毛犬，表情愉悦，摇着尾巴\n左下宫格：一头健壮的牛，田园背景，目光温和，皮毛光泽\n右下宫格：一匹骏马，姿态优雅，鬃毛飘逸，肌肉健美\n风格要求：四个宫格风格统一，色彩鲜艳饱和，高清画质，细节清晰锐利，专业插画风格，线条干净，统一的左上方光源，柔和阴影，和谐配色，卡通/半写实风格，宫格间用白色或浅灰细线分隔",
|
||||
imageBase64: [],
|
||||
aspectRatio: "16:9",
|
||||
size: "1K",
|
||||
@ -31,10 +32,11 @@ export default router.post(
|
||||
manufacturer,
|
||||
},
|
||||
);
|
||||
res.status(200).send(success(contentStr));
|
||||
} catch (err: any) {
|
||||
const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败";
|
||||
res.status(500).send(error(message));
|
||||
res.status(200).send(success(image));
|
||||
} catch (err) {
|
||||
const msg = u.error(err).message;
|
||||
console.error(msg);
|
||||
res.status(500).send(error(msg));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -1,9 +1,6 @@
|
||||
import express from "express";
|
||||
import { success, error } from "@/lib/responseFormat";
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { OpenAIChatModel, type OpenAIChatModelOptions } from "@aigne/openai";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import { z } from "zod";
|
||||
const router = express.Router();
|
||||
@ -15,25 +12,35 @@ export default router.post(
|
||||
modelName: z.string().optional(),
|
||||
apiKey: z.string(),
|
||||
baseURL: z.string().optional(),
|
||||
manufacturer: z.enum(["runninghub", "volcengine", "apimart", "gemini", "openAi"]),
|
||||
manufacturer: z.string(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { modelName, apiKey, baseURL, manufacturer } = req.body;
|
||||
try {
|
||||
const videoPath = await u.ai.generateVideo(
|
||||
const duration = manufacturer == "gemini" ? 4 : 5;
|
||||
const videoPath = await u.ai.video(
|
||||
{
|
||||
imageBase64: [],
|
||||
savePath: "",
|
||||
savePath: "test.mp4",
|
||||
prompt: "stickman Dances",
|
||||
duration: 10 as any,
|
||||
aspectRatio: "16:9" as any,
|
||||
duration: duration,
|
||||
resolution: "720p",
|
||||
aspectRatio: "16:9",
|
||||
audio: false,
|
||||
},
|
||||
{
|
||||
model: modelName,
|
||||
apiKey,
|
||||
baseURL,
|
||||
manufacturer,
|
||||
},
|
||||
manufacturer,
|
||||
);
|
||||
const url = await u.oss.getFileUrl(videoPath);
|
||||
res.status(200).send(success(url));
|
||||
} catch (err: any) {
|
||||
res.status(500).send(error(err.error.message || "模型调用失败"));
|
||||
const msg = u.error(err).message;
|
||||
console.error(msg);
|
||||
res.status(500).send(error(msg));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -8,7 +8,6 @@ expressWs(router as unknown as Application);
|
||||
router.ws("/", async (ws, req) => {
|
||||
let agent: OutlineScript;
|
||||
|
||||
const config = await u.getConfig("language");
|
||||
|
||||
const projectId = req.query.projectId;
|
||||
if (!projectId || typeof projectId !== "string") {
|
||||
@ -19,10 +18,6 @@ router.ws("/", async (ws, req) => {
|
||||
|
||||
agent = new OutlineScript(Number(projectId));
|
||||
|
||||
agent.modelName = config.model ?? "";
|
||||
agent.baseURL = config.baseURL ?? "";
|
||||
agent.apiKey = config.apiKey ?? "";
|
||||
|
||||
// const existing = await u
|
||||
// .db("t_chatHistory")
|
||||
// .where({ projectId: Number(projectId) })
|
||||
|
||||
33
src/routes/setting/addModel.ts
Normal file
33
src/routes/setting/addModel.ts
Normal file
@ -0,0 +1,33 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
type: z.enum(["text", "video", "image"]),
|
||||
model: z.string(),
|
||||
baseUrl: z.string(),
|
||||
apiKey: z.string(),
|
||||
modelType: z.string(),
|
||||
manufacturer: z.string(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { type, model, baseUrl, apiKey, manufacturer, modelType } = req.body;
|
||||
|
||||
await u.db("t_config").insert({
|
||||
type,
|
||||
model,
|
||||
baseUrl,
|
||||
apiKey,
|
||||
manufacturer,
|
||||
modelType,
|
||||
createTime: Date.now(),
|
||||
userId: 1,
|
||||
});
|
||||
res.status(200).send(success("新增成功"));
|
||||
},
|
||||
);
|
||||
23
src/routes/setting/configurationModel.ts
Normal file
23
src/routes/setting/configurationModel.ts
Normal file
@ -0,0 +1,23 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
id: z.number(),
|
||||
configId: z.number(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { id, configId } = req.body;
|
||||
if (id) {
|
||||
await u.db("t_aiModelMap").where("id", id).update({
|
||||
configId,
|
||||
});
|
||||
}
|
||||
res.status(200).send(success("配置成功"));
|
||||
},
|
||||
);
|
||||
19
src/routes/setting/delModel.ts
Normal file
19
src/routes/setting/delModel.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
id: z.number(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { id } = req.body;
|
||||
await u.db("t_config").where("id", id).delete();
|
||||
await u.db("t_aiModelMap").where("configId", id).update("configId",null);
|
||||
res.status(200).send(success("删除成功"));
|
||||
},
|
||||
);
|
||||
13
src/routes/setting/getAiModelMap.ts
Normal file
13
src/routes/setting/getAiModelMap.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post("/", async (req, res) => {
|
||||
const configData = await u
|
||||
.db("t_aiModelMap")
|
||||
.leftJoin("t_config", "t_aiModelMap.configId", "t_config.id")
|
||||
.select("t_aiModelMap.name", "t_config.model", "t_aiModelMap.id", "t_aiModelMap.key");
|
||||
res.status(200).send(success(configData));
|
||||
});
|
||||
17
src/routes/setting/getLog.ts
Normal file
17
src/routes/setting/getLog.ts
Normal file
@ -0,0 +1,17 @@
|
||||
import logger from "@/logger";
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success, error } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post("/", async (req, res) => {
|
||||
const { id } = (req as any).user;
|
||||
|
||||
if (id !== 1) return res.status(400).send(error("无权限查看，仅管理员USERID=1可见"));
|
||||
|
||||
const logs = logger.exportLogs();
|
||||
|
||||
res.status(200).send(success(logs));
|
||||
});
|
||||
@ -1,41 +1,11 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
userId: z.number(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { userId } = req.body;
|
||||
export default router.post("/", async (req, res) => {
|
||||
const userId = 1;
|
||||
const configData = await u.db("t_config").where("type","<>","video").where("userId", userId).select("*");
|
||||
|
||||
const settingData = await u.db("t_setting").select("*");
|
||||
|
||||
const configData = await u.db("t_config").where("userId", userId).select("*") ;
|
||||
|
||||
const parsedData = settingData.map((item) => ({
|
||||
...item,
|
||||
imageModel: (() => {
|
||||
try {
|
||||
return JSON.parse(item.imageModel ?? "{}");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})(),
|
||||
languageModel: (() => {
|
||||
try {
|
||||
return JSON.parse(item.languageModel ?? "{}");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})(),
|
||||
videoModel: configData,
|
||||
}));
|
||||
|
||||
res.status(200).send(success(parsedData));
|
||||
}
|
||||
);
|
||||
res.status(200).send(success(configData));
|
||||
});
|
||||
|
||||
11
src/routes/setting/getVideoModelList.ts
Normal file
11
src/routes/setting/getVideoModelList.ts
Normal file
@ -0,0 +1,11 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post("/", async (req, res) => {
|
||||
const userId = 1;
|
||||
const configData = await u.db("t_config").where("type","video").where("userId", userId).select("*");
|
||||
|
||||
res.status(200).send(success(configData));
|
||||
});
|
||||
32
src/routes/setting/updateModel.ts
Normal file
32
src/routes/setting/updateModel.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
id: z.number(),
|
||||
type: z.enum(["text", "video", "image"]),
|
||||
model: z.string(),
|
||||
baseUrl: z.string(),
|
||||
modelType: z.string(),
|
||||
apiKey: z.string(),
|
||||
manufacturer: z.string(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { id, type, model, baseUrl, apiKey, manufacturer, modelType } = req.body;
|
||||
|
||||
await u.db("t_config").where("id", id).update({
|
||||
type,
|
||||
model,
|
||||
baseUrl,
|
||||
apiKey,
|
||||
manufacturer,
|
||||
modelType,
|
||||
});
|
||||
res.status(200).send(success("编辑成功"));
|
||||
},
|
||||
);
|
||||
@ -1,55 +0,0 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
// 修改全局配置
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
userId: z.number(),
|
||||
imageModel: z.object().optional(),
|
||||
videoModel: z.array(z.object()).optional(),
|
||||
languageModel: z.object().optional(),
|
||||
name: z.string().optional(),
|
||||
password: z.string().optional(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { userId, imageModel, videoModel, languageModel, name, password } = req.body;
|
||||
|
||||
await u
|
||||
.db("t_setting")
|
||||
.where("userId", userId)
|
||||
.update({
|
||||
imageModel: JSON.stringify(imageModel),
|
||||
languageModel: JSON.stringify(languageModel),
|
||||
});
|
||||
|
||||
if (videoModel) {
|
||||
await u.db("t_config").where("type", "video").delete();
|
||||
|
||||
for (const item of videoModel) {
|
||||
await u.db("t_config").insert({
|
||||
type: "video",
|
||||
name: item.name,
|
||||
model: item.model,
|
||||
apiKey: item.apiKey,
|
||||
baseUrl: item.baseUrl,
|
||||
index: item.index,
|
||||
createTime: Date.now(),
|
||||
userId,
|
||||
manufacturer: item.manufacturer,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await u.db("t_user").where("id", userId).update({
|
||||
name,
|
||||
password,
|
||||
});
|
||||
|
||||
res.status(200).send(success({ message: "修改全局配置成功" }));
|
||||
}
|
||||
);
|
||||
32
src/routes/setting/updeteModel.ts
Normal file
32
src/routes/setting/updeteModel.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { z } from "zod";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
const router = express.Router();
|
||||
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
id: z.number(),
|
||||
type: z.enum(["text", "video", "image"]),
|
||||
model: z.string(),
|
||||
baseUrl: z.string(),
|
||||
modelType: z.string(),
|
||||
apiKey: z.string(),
|
||||
manufacturer: z.string(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { id, type, model, baseUrl, apiKey, manufacturer, modelType } = req.body;
|
||||
|
||||
await u.db("t_config").where("id", id).update({
|
||||
type,
|
||||
model,
|
||||
baseUrl,
|
||||
apiKey,
|
||||
manufacturer,
|
||||
modelType,
|
||||
});
|
||||
res.status(200).send(success("编辑成功"));
|
||||
},
|
||||
);
|
||||
@ -17,19 +17,19 @@ async function urlToBase64(imageUrl: string): Promise<string> {
|
||||
}
|
||||
|
||||
// 超分并保存到 oss
|
||||
async function superResolutionAndSave(
|
||||
src: string,
|
||||
projectId: number,
|
||||
videoRatio: string,
|
||||
): Promise<{ ossPath: string; base64: string }> {
|
||||
const contentStr = await u.ai.generateImage({
|
||||
aspectRatio: videoRatio,
|
||||
size: "1K",
|
||||
resType: "b64",
|
||||
systemPrompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率",
|
||||
prompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率",
|
||||
imageBase64: [await urlToBase64(src)],
|
||||
});
|
||||
async function superResolutionAndSave(src: string, projectId: number, videoRatio: string): Promise<{ ossPath: string; base64: string }> {
|
||||
const apiConfig = await u.getPromptAi("storyboardImage");
|
||||
const contentStr = await u.ai.image(
|
||||
{
|
||||
aspectRatio: videoRatio,
|
||||
size: "1K",
|
||||
resType: "b64",
|
||||
systemPrompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率",
|
||||
prompt: "你的核心任务是将所给的图片超分到 1K ，不改变图片任何内容，仅改变分辨率",
|
||||
imageBase64: [await urlToBase64(src)],
|
||||
},
|
||||
apiConfig,
|
||||
);
|
||||
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
const base64Str = match ? match[1] : contentStr;
|
||||
const buffer = Buffer.from(base64Str, "base64");
|
||||
@ -50,9 +50,9 @@ export default router.post(
|
||||
id: z.string(),
|
||||
prompt: z.string().optional(),
|
||||
src: z.string(),
|
||||
})
|
||||
}),
|
||||
),
|
||||
})
|
||||
}),
|
||||
),
|
||||
}),
|
||||
async (req, res) => {
|
||||
@ -63,9 +63,7 @@ export default router.post(
|
||||
if (!projectData) return res.status(500).send(error("项目不存在"));
|
||||
|
||||
// 遍历处理每个分镜段
|
||||
const processSegment = async (
|
||||
segment: { cells: { id: string; src: string }[] }
|
||||
) => {
|
||||
const processSegment = async (segment: { cells: { id: string; src: string }[] }) => {
|
||||
// 超分所有 cell
|
||||
const cellsWithSuperscore = await Promise.all(
|
||||
segment.cells.map(async (cell) => {
|
||||
@ -76,9 +74,9 @@ export default router.post(
|
||||
scriptId,
|
||||
filePath: ossPath, // oss 路径（未签名）
|
||||
src: cell.src,
|
||||
type: "分镜"
|
||||
type: "分镜",
|
||||
};
|
||||
})
|
||||
}),
|
||||
);
|
||||
return cellsWithSuperscore;
|
||||
};
|
||||
@ -92,9 +90,9 @@ export default router.post(
|
||||
(item.value as any[]).map(async (cell) => ({
|
||||
...cell,
|
||||
filePath: await u.oss.getFileUrl(cell.filePath ?? ""),
|
||||
}))
|
||||
)
|
||||
})),
|
||||
),
|
||||
);
|
||||
res.status(200).send(success(flatList));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -8,7 +8,6 @@ expressWs(router as unknown as Application);
|
||||
router.ws("/", async (ws, req) => {
|
||||
let agent: Storyboard;
|
||||
|
||||
const config = await u.getConfig("language");
|
||||
|
||||
const projectId = req.query.projectId;
|
||||
const scriptId = req.query.scriptId;
|
||||
@ -20,10 +19,6 @@ router.ws("/", async (ws, req) => {
|
||||
|
||||
agent = new Storyboard(Number(projectId), Number(scriptId));
|
||||
|
||||
agent.modelName = config.model ?? "";
|
||||
agent.baseURL = config.baseURL ?? "";
|
||||
agent.apiKey = config.apiKey ?? "";
|
||||
|
||||
const existing = await u
|
||||
.db("t_chatHistory")
|
||||
.where({ projectId: Number(projectId) })
|
||||
|
||||
@ -3,16 +3,10 @@ import u from "@/utils";
|
||||
import { error, success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import { z } from "zod";
|
||||
import path from "path";
|
||||
import axios from "axios";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
const cellsResultSchema = z.object({
|
||||
time: z.number().describe("时长,镜头时长 1-15"),
|
||||
content: z.string().describe("提示词内容"),
|
||||
name: z.string().describe("分镜名称"),
|
||||
});
|
||||
|
||||
const prompt = `
|
||||
你是一名资深动画导演，擅长将静态分镜转化为简洁、专业、详尽的 Motion Prompt（视频生成动作提示）。你理解镜头语言、情绪节奏，能补充丰富但不重复静态元素，只突出变化与动态。
|
||||
|
||||
@ -103,7 +97,12 @@ const prompt = `
|
||||
现在请根据我提供的分镜内容，严格按照以上规则输出 Motion Prompt JSON 对象。
|
||||
|
||||
`;
|
||||
|
||||
async function urlToBase64(imageUrl: string): Promise<string> {
|
||||
const response = await axios.get(imageUrl, { responseType: "arraybuffer" });
|
||||
const contentType = response.headers["content-type"] || "image/png";
|
||||
const base64 = Buffer.from(response.data, "binary").toString("base64");
|
||||
return `data:${contentType};base64,${base64}`;
|
||||
}
|
||||
// 生成单个分镜提示
|
||||
async function generateSingleVideoPrompt({
|
||||
scriptText,
|
||||
@ -114,25 +113,6 @@ async function generateSingleVideoPrompt({
|
||||
storyboardPrompt: string;
|
||||
ossPath: string;
|
||||
}): Promise<{ content: string; time: number; name: string }> {
|
||||
let rootDir: string;
|
||||
if (typeof process.versions?.electron !== "undefined") {
|
||||
const { app } = require("electron");
|
||||
const userDataDir: string = app.getPath("userData");
|
||||
rootDir = path.join(userDataDir, "uploads");
|
||||
} else {
|
||||
rootDir = path.join(process.cwd(), "uploads");
|
||||
}
|
||||
|
||||
let imagePath = ossPath;
|
||||
if (ossPath.includes("http")) {
|
||||
imagePath = new URL(ossPath).pathname;
|
||||
}
|
||||
|
||||
const model = await u.ai.text({});
|
||||
if (!model) {
|
||||
throw new Error("无法获取语言模型，请检查语言模型配置");
|
||||
}
|
||||
|
||||
const messages: any[] = [
|
||||
{
|
||||
role: "system",
|
||||
@ -146,38 +126,38 @@ async function generateSingleVideoPrompt({
|
||||
text: `剧本内容:${scriptText}\n分镜提示词:${storyboardPrompt}`,
|
||||
},
|
||||
{
|
||||
type: "local",
|
||||
path: path.join(rootDir, imagePath),
|
||||
type: "image",
|
||||
image: await urlToBase64(ossPath),
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
try {
|
||||
const result = await model.invoke({
|
||||
messages,
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
jsonSchema: {
|
||||
name: "json",
|
||||
strict: true,
|
||||
schema: z.toJSONSchema(cellsResultSchema),
|
||||
const apiConfig = await u.getPromptAi("videoPrompt");
|
||||
|
||||
const result = await u.ai.text.invoke(
|
||||
{
|
||||
messages,
|
||||
output: {
|
||||
time: z.number().describe("时长,镜头时长 1-15"),
|
||||
content: z.string().describe("提示词内容"),
|
||||
name: z.string().describe("分镜名称"),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (!result || !result.json) {
|
||||
apiConfig,
|
||||
);
|
||||
if (!result) {
|
||||
console.error("AI 返回结果为空:", result);
|
||||
throw new Error("AI 返回结果为空");
|
||||
}
|
||||
|
||||
const json = result.json as { content: string; time: number; name: string };
|
||||
if (!json.content || json.time === undefined || !json.name) {
|
||||
console.error("AI 返回格式错误:", result.json);
|
||||
if (!result.content || result.time === undefined || !result.name) {
|
||||
console.error("AI 返回格式错误:", result);
|
||||
throw new Error("AI 返回格式错误");
|
||||
}
|
||||
|
||||
return json;
|
||||
return result;
|
||||
} catch (err: any) {
|
||||
console.error("generateSingleVideoPrompt 调用失败:", err?.message || err);
|
||||
throw new Error(`生成视频提示词失败: ${err?.message || "未知错误"}`);
|
||||
|
||||
@ -42,5 +42,5 @@ export default router.post(
|
||||
});
|
||||
|
||||
res.status(200).send(success({ message: "新增视频成功" }));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -1,16 +1,18 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { error, success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import { z } from "zod";
|
||||
const router = express.Router();
|
||||
|
||||
// 图片项schema
|
||||
const imageItemSchema = z.object({
|
||||
id: z.number(),
|
||||
filePath: z.string(),
|
||||
prompt: z.string().optional(),
|
||||
}).nullable();
|
||||
const imageItemSchema = z
|
||||
.object({
|
||||
id: z.number(),
|
||||
filePath: z.string(),
|
||||
prompt: z.string().optional(),
|
||||
})
|
||||
.nullable();
|
||||
|
||||
// 新增视频配置
|
||||
export default router.post(
|
||||
@ -18,44 +20,40 @@ export default router.post(
|
||||
validateFields({
|
||||
scriptId: z.number(),
|
||||
projectId: z.number(),
|
||||
manufacturer: z.string(),
|
||||
mode: z.enum(["startEnd", "multi", "single"]),
|
||||
configId: z.number(),
|
||||
mode: z.enum(["startEnd", "multi", "single", "text", ""]),
|
||||
startFrame: imageItemSchema.optional(),
|
||||
endFrame: imageItemSchema.optional(),
|
||||
images: z.array(z.object({
|
||||
id: z.number(),
|
||||
filePath: z.string(),
|
||||
prompt: z.string().optional(),
|
||||
})).optional(),
|
||||
images: z
|
||||
.array(
|
||||
z.object({
|
||||
id: z.number(),
|
||||
filePath: z.string(),
|
||||
prompt: z.string().optional(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
resolution: z.string(),
|
||||
duration: z.number(),
|
||||
prompt: z.string().optional(),
|
||||
audioEnabled: z.boolean(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const {
|
||||
scriptId,
|
||||
projectId,
|
||||
manufacturer,
|
||||
mode,
|
||||
startFrame,
|
||||
endFrame,
|
||||
images,
|
||||
resolution,
|
||||
duration,
|
||||
prompt
|
||||
} = req.body;
|
||||
const { scriptId, projectId, configId, mode, startFrame, endFrame, images, resolution, duration, prompt, audioEnabled } = req.body;
|
||||
|
||||
// 生成新ID
|
||||
const maxIdResult = await u.db("t_videoConfig").max("id as maxId").first();
|
||||
const maxIdResult: any = await u.db("t_videoConfig").max("id as maxId").first();
|
||||
const newId = (maxIdResult?.maxId || 0) + 1;
|
||||
const now = Date.now();
|
||||
|
||||
const configData = await u.db("t_config").where("id", configId).first();
|
||||
if (!configData) return res.status(500).send(error("不存在的模型"));
|
||||
// 插入数据
|
||||
await u.db("t_videoConfig").insert({
|
||||
id: newId,
|
||||
scriptId,
|
||||
projectId,
|
||||
manufacturer,
|
||||
manufacturer: configData.manufacturer,
|
||||
aiConfigId: configId,
|
||||
mode,
|
||||
startFrame: startFrame ? JSON.stringify(startFrame) : null,
|
||||
endFrame: endFrame ? JSON.stringify(endFrame) : null,
|
||||
@ -66,25 +64,31 @@ export default router.post(
|
||||
selectedResultId: null,
|
||||
createTime: now,
|
||||
updateTime: now,
|
||||
audioEnabled: audioEnabled ? 1 : 0,
|
||||
});
|
||||
|
||||
res.status(200).send(success({
|
||||
message: "新增视频配置成功",
|
||||
data: {
|
||||
id: newId,
|
||||
scriptId,
|
||||
projectId,
|
||||
manufacturer,
|
||||
mode,
|
||||
startFrame,
|
||||
endFrame,
|
||||
images: images || [],
|
||||
resolution,
|
||||
duration,
|
||||
prompt: prompt || "",
|
||||
selectedResultId: null,
|
||||
createdAt: new Date(now).toISOString(),
|
||||
}
|
||||
}));
|
||||
res.status(200).send(
|
||||
success({
|
||||
message: "新增视频配置成功",
|
||||
data: {
|
||||
id: newId,
|
||||
scriptId,
|
||||
projectId,
|
||||
manufacturer: configData.manufacturer,
|
||||
aiConfigId: configId,
|
||||
model: configData.model,
|
||||
mode,
|
||||
startFrame,
|
||||
endFrame,
|
||||
images: images || [],
|
||||
resolution,
|
||||
duration,
|
||||
prompt: prompt || "",
|
||||
selectedResultId: null,
|
||||
createdAt: new Date(now).toISOString(),
|
||||
audioEnabled: audioEnabled,
|
||||
},
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@ -1,17 +1,18 @@
|
||||
import express from "express";
|
||||
import u from "@/utils";
|
||||
import { success } from "@/lib/responseFormat";
|
||||
import { error, success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import { z } from "zod";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
type GenerateMode = "startEnd" | "multi" | "single";
|
||||
type GenerateMode = "startEnd" | "multi" | "single" | "text";
|
||||
|
||||
const getSystemPrompt = async (mode: GenerateMode) => {
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["video-startEnd", "video-multi", "video-single", "video-main", "video-text"]);
|
||||
|
||||
const getSystemPrompt = async (mode: GenerateMode): Promise<string> => {
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["video-startEnd", "video-multi", "video-single", "video-main"]);
|
||||
const errPrompts = "不论用户说什么，请直接输出AI配置异常";
|
||||
const getPromptValue = (code: string): string => {
|
||||
const getPromptValue = (code: string) => {
|
||||
const item = promptsList.find((p) => p.code === code);
|
||||
return item?.customValue ?? item?.defaultValue ?? errPrompts;
|
||||
};
|
||||
@ -19,14 +20,16 @@ const getSystemPrompt = async (mode: GenerateMode): Promise<string> => {
|
||||
const multi = getPromptValue("video-multi");
|
||||
const single = getPromptValue("video-single");
|
||||
const main = getPromptValue("video-main");
|
||||
const text = getPromptValue("video-text");
|
||||
|
||||
const modeDescriptions: Record<GenerateMode, string> = {
|
||||
const modeDescriptions = {
|
||||
startEnd: startEnd,
|
||||
multi: multi,
|
||||
single: single,
|
||||
text: text,
|
||||
};
|
||||
|
||||
return `${main}\n\n${modeDescriptions[mode]}`;
|
||||
const modeData = modeDescriptions[mode];
|
||||
return `${main}\n\n${modeData}`;
|
||||
};
|
||||
|
||||
const getModeDescription = (mode: GenerateMode): string => {
|
||||
@ -34,6 +37,7 @@ const getModeDescription = (mode: GenerateMode): string => {
|
||||
startEnd: "首尾帧模式",
|
||||
multi: "宫格模式",
|
||||
single: "单图模式",
|
||||
text: "文本模式",
|
||||
};
|
||||
return map[mode];
|
||||
};
|
||||
@ -41,42 +45,62 @@ const getModeDescription = (mode: GenerateMode): string => {
|
||||
export default router.post(
|
||||
"/",
|
||||
validateFields({
|
||||
images: z.array(
|
||||
z.object({
|
||||
filePath: z.string(),
|
||||
prompt: z.string(),
|
||||
}),
|
||||
),
|
||||
images: z
|
||||
.array(
|
||||
z.object({
|
||||
filePath: z.string(),
|
||||
prompt: z.string(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
prompt: z.string(),
|
||||
duration: z.number(),
|
||||
type: z.enum(["startEnd", "multi", "single"]).optional(),
|
||||
type: z.enum(["startEnd", "multi", "single", "text", ""]).optional(),
|
||||
videoConfigId: z.number().optional(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { prompt, images, duration, type = "single" } = req.body;
|
||||
const { prompt, images, duration, type = "single", videoConfigId } = req.body;
|
||||
const mode = type as GenerateMode;
|
||||
|
||||
const model = await u.ai.text({});
|
||||
|
||||
let videoConfigData;
|
||||
if (videoConfigId) {
|
||||
videoConfigData = await u
|
||||
.db("t_videoConfig")
|
||||
.leftJoin("t_script", "t_script.id", "t_videoConfig.scriptId")
|
||||
.where("t_videoConfig.id", videoConfigId)
|
||||
.select("t_script.content")
|
||||
.first();
|
||||
if (!videoConfigData) return res.status(500).send(error("视频配置不存在"));
|
||||
}
|
||||
const imagePrompts = images.map((i: { filePath: string; prompt: string }, index: number) => `Image ${index + 1}: ${i.prompt}`).join("\n");
|
||||
|
||||
const shotCount = images.length;
|
||||
const avgDuration = (parseFloat(duration) / shotCount).toFixed(1);
|
||||
|
||||
const result = await model!.invoke({
|
||||
messages: [
|
||||
const promptConfig = await getSystemPrompt(mode);
|
||||
const promptAiConfig = await u.getPromptAi("videoPrompt");
|
||||
try {
|
||||
const result = await u.ai.text.invoke(
|
||||
{
|
||||
role: "system",
|
||||
content: await getSystemPrompt(mode),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `Mode: ${getModeDescription(mode)}
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: promptConfig,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `Mode: ${getModeDescription(mode)}
|
||||
|
||||
Reference Images:
|
||||
${imagePrompts}
|
||||
|
||||
Script:
|
||||
${prompt}
|
||||
${
|
||||
videoConfigData
|
||||
? `script content:
|
||||
${videoConfigData.content}`
|
||||
: ""
|
||||
}
|
||||
|
||||
|
||||
Parameters:
|
||||
- Total Duration: ${duration}s
|
||||
@ -84,10 +108,15 @@ Parameters:
|
||||
- Average Duration: ${avgDuration}s per shot
|
||||
|
||||
Generate storyboard prompts:`,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
promptAiConfig,
|
||||
);
|
||||
|
||||
res.status(200).send(success(result.text));
|
||||
res.status(200).send(success(result.text));
|
||||
} catch (e) {
|
||||
return res.status(500).send(error(u.error(e).message));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -4,6 +4,10 @@ import { z } from "zod";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { error, success } from "@/lib/responseFormat";
|
||||
import { validateFields } from "@/middleware/middleware";
|
||||
import { t_config } from "@/types/database";
|
||||
import sharp from "sharp";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
@ -13,44 +17,55 @@ export default router.post(
|
||||
validateFields({
|
||||
projectId: z.number(),
|
||||
scriptId: z.number(),
|
||||
configId: z.number().optional(), // 关联的视频配置ID
|
||||
configId: z.number().optional(), // 关联的视频配 置ID
|
||||
type: z.string().optional(),
|
||||
resolution: z.string(),
|
||||
aiConfigId: z.number(),
|
||||
filePath: z.array(z.string()),
|
||||
duration: z.number(),
|
||||
prompt: z.string(),
|
||||
mode: z.enum(["startEnd", "multi", "single", "text"]),
|
||||
audioEnabled: z.boolean(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { type, scriptId, projectId, configId, resolution, filePath, duration, prompt } = req.body;
|
||||
const { type, mode, scriptId, projectId, configId, aiConfigId, resolution, filePath, duration, prompt, audioEnabled } = req.body;
|
||||
|
||||
// 参数校验
|
||||
if (type === "volcengine") {
|
||||
if (duration < 4 || duration > 12) {
|
||||
return res.status(400).send(error("视频时长需在4-12秒之间"));
|
||||
}
|
||||
if (!["480p", "720p", "1080p"].includes(resolution)) {
|
||||
return res.status(400).send(error("视频分辨率不正确"));
|
||||
if (mode == "text") filePath.length = 0;
|
||||
else if (!filePath.length) {
|
||||
return res.status(500).send(error("请先选择图片"));
|
||||
}
|
||||
const configData = await u.db("t_videoConfig").where("id", configId).first();
|
||||
|
||||
if (!configData) {
|
||||
return res.status(500).send(error("视频配置不存在"));
|
||||
}
|
||||
if (configData.manufacturer == "runninghub") {
|
||||
if (filePath.length > 1) {
|
||||
const gridUrl = await sharpProcessingImage(filePath, projectId);
|
||||
if (gridUrl) {
|
||||
filePath.length = 0;
|
||||
filePath.push(gridUrl);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (type === "runninghub") {
|
||||
if (duration !== 10 && duration !== 15) {
|
||||
return res.status(400).send(error("视频时长只能是10秒或15秒"));
|
||||
}
|
||||
if (resolution !== "9:16" && resolution !== "16:9") {
|
||||
return res.status(400).send(error("视频分辨率不正确"));
|
||||
}
|
||||
// 优先使用视频配置中的AI配置ID查询,查不到再使用传入的aiConfigId
|
||||
let aiConfigData = null;
|
||||
if (configData.aiConfigId) {
|
||||
aiConfigData = await u.db("t_config").where("id", configData.aiConfigId).first();
|
||||
}
|
||||
if (!aiConfigData) {
|
||||
aiConfigData = await u.db("t_config").where("id", aiConfigId).first();
|
||||
}
|
||||
|
||||
if (!aiConfigData) {
|
||||
return res.status(500).send(error("模型配置不存在"));
|
||||
}
|
||||
// 过滤掉空值
|
||||
let fileUrl = filePath.filter((p: string) => p && p.trim() !== "");
|
||||
|
||||
if (fileUrl.length === 0) {
|
||||
return res.status(400).send(error("请至少选择一张图片"));
|
||||
}
|
||||
|
||||
// 处理文件路径，如果是 base64 则上传到 OSS
|
||||
if (fileUrl.length === 1) {
|
||||
if (fileUrl.length) {
|
||||
const match = fileUrl[0].match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
if (match && match.length >= 2) {
|
||||
const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
|
||||
@ -69,20 +84,21 @@ export default router.post(
|
||||
// 否则认为已经是路径
|
||||
return url;
|
||||
};
|
||||
if (fileUrl.length) {
|
||||
// 校验文件是否存在
|
||||
const fileExistsResults = await Promise.all(
|
||||
fileUrl.map(async (url: string) => {
|
||||
const path = getPathname(url);
|
||||
return u.oss.fileExists(path);
|
||||
}),
|
||||
);
|
||||
|
||||
// 校验文件是否存在
|
||||
const fileExistsResults = await Promise.all(
|
||||
fileUrl.map(async (url: string) => {
|
||||
const path = getPathname(url);
|
||||
return u.oss.fileExists(path);
|
||||
}),
|
||||
);
|
||||
|
||||
if (!fileExistsResults.every(Boolean)) {
|
||||
return res.status(400).send(error("选择分镜文件不存在"));
|
||||
if (!fileExistsResults.every(Boolean)) {
|
||||
return res.status(400).send(error("选择分镜文件不存在"));
|
||||
}
|
||||
}
|
||||
|
||||
const firstFrame = getPathname(fileUrl[0]);
|
||||
const firstFrame = fileUrl.length ? getPathname(fileUrl[0]) : "";
|
||||
const storyboardImgs = fileUrl.map((path: string) => getPathname(path));
|
||||
const savePath = `/${projectId}/video/${uuidv4()}.mp4`;
|
||||
|
||||
@ -103,7 +119,7 @@ export default router.post(
|
||||
res.status(200).send(success({ id: videoId, configId: configId || null }));
|
||||
|
||||
// 异步生成视频
|
||||
generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, type);
|
||||
generateVideoAsync(videoId, projectId, fileUrl, savePath, prompt, duration, resolution, audioEnabled, aiConfigData);
|
||||
},
|
||||
);
|
||||
|
||||
@ -116,10 +132,11 @@ async function generateVideoAsync(
|
||||
prompt: string,
|
||||
duration: number,
|
||||
resolution: string,
|
||||
type?: string,
|
||||
audioEnabled: boolean,
|
||||
aiConfigData: t_config,
|
||||
) {
|
||||
try {
|
||||
const projectData = await u.db("t_project").where("id", projectId).select("artStyle").first();
|
||||
const projectData = await u.db("t_project").where("id", projectId).select("artStyle", "videoRatio").first();
|
||||
|
||||
// 提取路径名的辅助函数
|
||||
const getPathname = (url: string): string => {
|
||||
@ -149,16 +166,22 @@ ${prompt}
|
||||
3. 关键人物在画面中全部清晰显示，不得被遮挡、缺失或省略
|
||||
4. 画面真实、细致，无畸形、无模糊、无杂物、无多余人物、无文字、水印、logo
|
||||
`;
|
||||
|
||||
const videoPath = await u.ai.generateVideo(
|
||||
const videoPath = await u.ai.video(
|
||||
{
|
||||
imageBase64,
|
||||
savePath,
|
||||
prompt: inputPrompt,
|
||||
duration: duration as any,
|
||||
aspectRatio: resolution as any,
|
||||
aspectRatio: projectData?.videoRatio as any,
|
||||
resolution: resolution as any,
|
||||
audio: audioEnabled,
|
||||
},
|
||||
{
|
||||
baseURL: aiConfigData?.baseUrl!,
|
||||
model: aiConfigData?.model!,
|
||||
apiKey: aiConfigData?.apiKey!,
|
||||
manufacturer: aiConfigData?.manufacturer!,
|
||||
},
|
||||
type!,
|
||||
);
|
||||
|
||||
if (videoPath) {
|
||||
@ -173,6 +196,134 @@ ${prompt}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`视频生成失败 videoId=${videoId}:`, err);
|
||||
await u.db("t_video").where("id", videoId).update({ state: -1 });
|
||||
await u
|
||||
.db("t_video")
|
||||
.where("id", videoId)
|
||||
.update({ state: -1, errorReason: u.error(err).message });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 使用sharp把图片拼接为宫格图，最多3x3，图片数量为1-9不等
|
||||
* @param imageList - 图片路径或base64数组
|
||||
* @returns 拼接后的图片Buffer
|
||||
*/
|
||||
async function sharpProcessingImage(imageList: string[], projectId: number): Promise<string> {
|
||||
if (!imageList || imageList.length === 0) {
|
||||
throw new Error("图片列表不能为空");
|
||||
}
|
||||
|
||||
if (imageList.length > 9) {
|
||||
throw new Error("图片数量不能超过9张");
|
||||
}
|
||||
|
||||
// 计算网格布局：根据图片数量确定行列数
|
||||
const count = imageList.length;
|
||||
let cols: number, rows: number;
|
||||
|
||||
if (count === 1) {
|
||||
cols = rows = 1;
|
||||
} else if (count === 2) {
|
||||
cols = 2;
|
||||
rows = 1;
|
||||
} else if (count <= 4) {
|
||||
cols = rows = 2;
|
||||
} else if (count <= 6) {
|
||||
cols = 3;
|
||||
rows = 2;
|
||||
} else {
|
||||
cols = rows = 3;
|
||||
}
|
||||
|
||||
// 第一步：加载所有图片并获取原始尺寸
|
||||
const loadedImages = await Promise.all(
|
||||
imageList.map(async (imagePath) => {
|
||||
let imageBuffer: Buffer;
|
||||
|
||||
// 判断是base64、URL还是文件路径
|
||||
if (imagePath.startsWith("data:image") || imagePath.match(/^[A-Za-z0-9+/=]+$/)) {
|
||||
// Base64格式
|
||||
const base64Data = imagePath.replace(/^data:image\/\w+;base64,/, "");
|
||||
imageBuffer = Buffer.from(base64Data, "base64");
|
||||
} else if (imagePath.startsWith("http://") || imagePath.startsWith("https://")) {
|
||||
// URL格式，提取pathname后从OSS读取
|
||||
const pathname = new URL(imagePath).pathname;
|
||||
imageBuffer = await u.oss.getFile(pathname);
|
||||
} else {
|
||||
// 文件路径，直接从OSS读取
|
||||
imageBuffer = await u.oss.getFile(imagePath);
|
||||
}
|
||||
|
||||
const metadata = await sharp(imageBuffer).metadata();
|
||||
return {
|
||||
buffer: imageBuffer,
|
||||
width: metadata.width || 0,
|
||||
height: metadata.height || 0,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
// 第二步：找出所有图片中的最大宽度和高度
|
||||
const maxWidth = Math.max(...loadedImages.map((img) => img.width));
|
||||
const maxHeight = Math.max(...loadedImages.map((img) => img.height));
|
||||
|
||||
// 第三步：将所有图片调整为统一尺寸（使用contain模式保持比例，填充背景色）
|
||||
const imageData = await Promise.all(
|
||||
loadedImages.map(async (img) => {
|
||||
const resizedBuffer = await sharp(img.buffer)
|
||||
.resize(maxWidth, maxHeight, {
|
||||
fit: "contain",
|
||||
background: { r: 0, g: 0, b: 0, alpha: 1 }, // 黑色背景填充
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
return {
|
||||
buffer: resizedBuffer,
|
||||
width: maxWidth,
|
||||
height: maxHeight,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
// 所有图片都是相同尺寸，直接计算画布大小
|
||||
const cellWidth = maxWidth;
|
||||
const cellHeight = maxHeight;
|
||||
const canvasWidth = cols * cellWidth;
|
||||
const canvasHeight = rows * cellHeight;
|
||||
|
||||
// 创建空白画布
|
||||
const canvas = sharp({
|
||||
create: {
|
||||
width: canvasWidth,
|
||||
height: canvasHeight,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
});
|
||||
|
||||
// 准备合成操作
|
||||
const compositeOperations = imageData.map((data, index) => {
|
||||
const row = Math.floor(index / cols);
|
||||
const col = index % cols;
|
||||
|
||||
// 计算当前图片的位置（无偏移，紧密排列）
|
||||
const left = col * cellWidth;
|
||||
const top = row * cellHeight;
|
||||
|
||||
return {
|
||||
input: data.buffer,
|
||||
top: top,
|
||||
left: left,
|
||||
};
|
||||
});
|
||||
|
||||
// 合成所有图片
|
||||
const result = await canvas.composite(compositeOperations).png().toBuffer();
|
||||
|
||||
const imagePath = `/${projectId}/assets/${uuidv4()}.jpg`;
|
||||
const buffer = Buffer.from(result as any, "base64");
|
||||
await u.oss.writeFile(imagePath, buffer);
|
||||
|
||||
return await u.oss.getFileUrl(imagePath);
|
||||
}
|
||||
|
||||
@ -14,8 +14,8 @@ export default router.post(
|
||||
async (req, res) => {
|
||||
const { userId } = req.body;
|
||||
|
||||
const data = await u.db("t_config").where("userId", userId).select("manufacturer", "model");
|
||||
const data = await u.db("t_config").where("type", "video").where("userId", userId).select("manufacturer", "model", "id");
|
||||
|
||||
res.status(200).send(success(data));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
@ -28,7 +28,7 @@ export default router.post(
|
||||
qb.whereIn("id", specifyIds);
|
||||
}
|
||||
})
|
||||
.select("id", "configId", "time", "resolution", "prompt", "firstFrame", "filePath", "storyboardImgs", "model", "scriptId", "state");
|
||||
.select("id", "configId", "time", "resolution", "prompt", "firstFrame", "filePath", "storyboardImgs", "model", "scriptId", "state","errorReason");
|
||||
// const videoIds: number[] = videos.map((video: any) => (typeof video.id === "string" ? parseInt(video.id) : video.id));
|
||||
|
||||
// let tempAssets: TempAsset[] = await u
|
||||
|
||||
@ -15,16 +15,20 @@ export default router.post(
|
||||
const { scriptId } = req.body;
|
||||
|
||||
// 查询该脚本下的所有视频配置
|
||||
const configs = await u.db("t_videoConfig")
|
||||
const configs = await u
|
||||
.db("t_videoConfig")
|
||||
.leftJoin("t_config", "t_config.id", "t_videoConfig.aiConfigId")
|
||||
.where({ scriptId })
|
||||
.orderBy("createTime", "desc");
|
||||
|
||||
.orderBy("createTime", "desc")
|
||||
.select("t_videoConfig.*", "t_config.manufacturer as manufacturer", "t_config.model");
|
||||
// 解析 JSON 字段
|
||||
const result = configs.map((config: any) => ({
|
||||
id: config.id,
|
||||
scriptId: config.scriptId,
|
||||
projectId: config.projectId,
|
||||
aiConfigId: config.aiConfigId,
|
||||
manufacturer: config.manufacturer,
|
||||
model: config.model,
|
||||
mode: config.mode,
|
||||
startFrame: config.startFrame ? JSON.parse(config.startFrame) : null,
|
||||
endFrame: config.endFrame ? JSON.parse(config.endFrame) : null,
|
||||
@ -34,6 +38,7 @@ export default router.post(
|
||||
prompt: config.prompt || "",
|
||||
selectedResultId: config.selectedResultId,
|
||||
createdAt: config.createTime ? new Date(config.createTime).toISOString() : new Date().toISOString(),
|
||||
audioEnabled:!!config.audioEnabled
|
||||
}));
|
||||
|
||||
res.status(200).send(success(result));
|
||||
|
||||
@ -14,9 +14,13 @@ export default router.post(
|
||||
duration: z.number().optional(),
|
||||
prompt: z.string().optional(),
|
||||
selectedResultId: z.number().nullable().optional(),
|
||||
startFrame: z.object().nullable().optional(),
|
||||
endFrame: z.object().nullable().optional(),
|
||||
images: z.array(z.object()).optional(),
|
||||
audioEnabled: z.boolean().optional(),
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { id, resolution, duration, prompt, selectedResultId } = req.body;
|
||||
const { id, resolution, duration, prompt, selectedResultId, startFrame, endFrame, images, audioEnabled } = req.body;
|
||||
|
||||
// 检查配置是否存在
|
||||
const existingConfig = await u.db("t_videoConfig").where({ id }).first();
|
||||
@ -41,30 +45,47 @@ export default router.post(
|
||||
if (selectedResultId !== undefined) {
|
||||
updateData.selectedResultId = selectedResultId;
|
||||
}
|
||||
|
||||
if (startFrame !== undefined) {
|
||||
updateData.startFrame = startFrame ? JSON.stringify(startFrame) : null;;
|
||||
}
|
||||
if (endFrame !== undefined) {
|
||||
updateData.endFrame = endFrame ? JSON.stringify(endFrame) : null;;
|
||||
}
|
||||
if (images !== undefined) {
|
||||
updateData.images = images ? JSON.stringify(images) : null;
|
||||
}
|
||||
if (audioEnabled !== undefined) {
|
||||
updateData.audioEnabled = audioEnabled;
|
||||
}
|
||||
// 更新数据
|
||||
await u.db("t_videoConfig").where({ id }).update(updateData);
|
||||
|
||||
// 获取更新后的数据
|
||||
const updatedConfig = await u.db("t_videoConfig").where({ id }).first();
|
||||
|
||||
res.status(200).send(success({
|
||||
message: "更新视频配置成功",
|
||||
data: {
|
||||
id: updatedConfig.id,
|
||||
scriptId: updatedConfig.scriptId,
|
||||
projectId: updatedConfig.projectId,
|
||||
manufacturer: updatedConfig.manufacturer,
|
||||
mode: updatedConfig.mode,
|
||||
startFrame: updatedConfig.startFrame ? JSON.parse(updatedConfig.startFrame) : null,
|
||||
endFrame: updatedConfig.endFrame ? JSON.parse(updatedConfig.endFrame) : null,
|
||||
images: updatedConfig.images ? JSON.parse(updatedConfig.images) : [],
|
||||
resolution: updatedConfig.resolution,
|
||||
duration: updatedConfig.duration,
|
||||
prompt: updatedConfig.prompt,
|
||||
selectedResultId: updatedConfig.selectedResultId,
|
||||
createdAt: new Date(updatedConfig.createTime).toISOString(),
|
||||
}
|
||||
}));
|
||||
if (updatedConfig) {
|
||||
res.status(200).send(
|
||||
success({
|
||||
message: "更新视频配置成功",
|
||||
data: {
|
||||
id: updatedConfig.id,
|
||||
scriptId: updatedConfig.scriptId,
|
||||
projectId: updatedConfig.projectId,
|
||||
manufacturer: updatedConfig.manufacturer,
|
||||
mode: updatedConfig.mode,
|
||||
startFrame: updatedConfig.startFrame ? JSON.parse(updatedConfig.startFrame) : null,
|
||||
endFrame: updatedConfig.endFrame ? JSON.parse(updatedConfig.endFrame) : null,
|
||||
images: updatedConfig.images ? JSON.parse(updatedConfig.images) : [],
|
||||
resolution: updatedConfig.resolution,
|
||||
duration: updatedConfig.duration,
|
||||
prompt: updatedConfig.prompt,
|
||||
selectedResultId: updatedConfig.selectedResultId,
|
||||
createdAt: new Date(updatedConfig.createTime!).toISOString(),
|
||||
audioEnabled: updatedConfig.audioEnabled,
|
||||
},
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
res.status(200).send(error("更新配置失败"));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
32
src/types/database.d.ts
vendored
32
src/types/database.d.ts
vendored
@ -1,6 +1,26 @@
|
||||
// @db-hash b6b4d8cdc25a2f4d60f1c239cd7e7060
|
||||
// @db-hash 5a1cbe86324cb073c1931fc53c56725f
|
||||
//该文件由脚本自动生成，请勿手动修改
|
||||
|
||||
export interface _t_video_old_20260210 {
|
||||
'aiConfigId'?: number | null;
|
||||
'configId'?: number | null;
|
||||
'filePath'?: string | null;
|
||||
'firstFrame'?: string | null;
|
||||
'id'?: number;
|
||||
'model'?: string | null;
|
||||
'prompt'?: string | null;
|
||||
'resolution'?: string | null;
|
||||
'scriptId'?: number | null;
|
||||
'state'?: number | null;
|
||||
'storyboardImgs'?: string | null;
|
||||
'time'?: number | null;
|
||||
}
|
||||
export interface t_aiModelMap {
|
||||
'configId'?: number | null;
|
||||
'id'?: number;
|
||||
'key'?: string | null;
|
||||
'name'?: string | null;
|
||||
}
|
||||
export interface t_assets {
|
||||
'duration'?: string | null;
|
||||
'episode'?: string | null;
|
||||
@ -30,10 +50,9 @@ export interface t_config {
|
||||
'baseUrl'?: string | null;
|
||||
'createTime'?: number | null;
|
||||
'id'?: number;
|
||||
'index'?: number | null;
|
||||
'manufacturer'?: string | null;
|
||||
'model'?: string | null;
|
||||
'name'?: string | null;
|
||||
'modelType'?: string | null;
|
||||
'type'?: string | null;
|
||||
'userId'?: number | null;
|
||||
}
|
||||
@ -118,7 +137,9 @@ export interface t_user {
|
||||
'password'?: string | null;
|
||||
}
|
||||
export interface t_video {
|
||||
'aiConfigId'?: number | null;
|
||||
'configId'?: number | null;
|
||||
'errorReason'?: string | null;
|
||||
'filePath'?: string | null;
|
||||
'firstFrame'?: string | null;
|
||||
'id'?: number;
|
||||
@ -131,9 +152,12 @@ export interface t_video {
|
||||
'time'?: number | null;
|
||||
}
|
||||
export interface t_videoConfig {
|
||||
'aiConfigId'?: number | null;
|
||||
'audioEnabled'?: number | null;
|
||||
'createTime'?: number | null;
|
||||
'duration'?: number | null;
|
||||
'endFrame'?: string | null;
|
||||
'errorReason'?: string | null;
|
||||
'id'?: number;
|
||||
'images'?: string | null;
|
||||
'manufacturer'?: string | null;
|
||||
@ -148,6 +172,8 @@ export interface t_videoConfig {
|
||||
}
|
||||
|
||||
export interface DB {
|
||||
"_t_video_old_20260210": _t_video_old_20260210;
|
||||
"t_aiModelMap": t_aiModelMap;
|
||||
"t_assets": t_assets;
|
||||
"t_chatHistory": t_chatHistory;
|
||||
"t_config": t_config;
|
||||
|
||||
12
src/utils.ts
12
src/utils.ts
@ -6,18 +6,28 @@ import number2Chinese from "@/utils/number2Chinese";
|
||||
import deleteOutline from "@/utils/deleteOutline";
|
||||
import getConfig from "./utils/getConfig";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import error from "@/utils/error";
|
||||
import * as imageTools from "@/utils/imageTools";
|
||||
|
||||
import AIText from "@/utils/ai/text";
|
||||
import AIText from "@/utils/ai/text/index";
|
||||
import AIImage from "@/utils/ai/image/index";
|
||||
import AIVideo from "@/utils/ai/video/index";
|
||||
|
||||
import getPromptAi from "./utils/getPromptAi";
|
||||
export default {
|
||||
db,
|
||||
oss,
|
||||
ai: {
|
||||
text: AIText,
|
||||
image: AIImage,
|
||||
video: AIVideo,
|
||||
},
|
||||
editImage,
|
||||
number2Chinese,
|
||||
deleteOutline,
|
||||
getConfig,
|
||||
uuid,
|
||||
error,
|
||||
imageTools,
|
||||
getPromptAi,
|
||||
};
|
||||
|
||||
@ -1,31 +1,28 @@
|
||||
import axios from "axios";
|
||||
import u from "@/utils";
|
||||
import FormData from "form-data";
|
||||
import axiosRetry from "axios-retry";
|
||||
import { OpenAIChatModel, type OpenAIChatModelOptions } from "@aigne/openai";
|
||||
import sharp from "sharp";
|
||||
|
||||
axiosRetry(axios, { retries: 3, retryDelay: () => 200 });
|
||||
|
||||
export const text = async (config: OpenAIChatModelOptions = {}) => {
|
||||
const { model, apiKey, baseURL } = await u.getConfig("language");
|
||||
return new OpenAIChatModel({
|
||||
apiKey: apiKey ?? "",
|
||||
baseURL: baseURL ?? "",
|
||||
model: model ?? "gpt-4.1",
|
||||
modelOptions: { temperature: 0.7 },
|
||||
...config,
|
||||
});
|
||||
};
|
||||
|
||||
interface ImageConfig {
|
||||
systemPrompt?: string;
|
||||
type VideoAspectRatio = "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive";
|
||||
interface BaseVideoConfig {
|
||||
prompt: string;
|
||||
imageBase64: string[];
|
||||
size: "1K" | "2K" | "4K";
|
||||
aspectRatio: string;
|
||||
resType?: "url" | "b64";
|
||||
savePath: string;
|
||||
imageBase64?: string[]; // 单张参考图片 base64
|
||||
}
|
||||
interface DoubaoVideoConfig extends BaseVideoConfig {
|
||||
duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒
|
||||
aspectRatio: VideoAspectRatio;
|
||||
audio?: boolean;
|
||||
}
|
||||
interface RunninghubVideoConfig extends BaseVideoConfig {
|
||||
duration: 10 | 15; // 仅支持 10 或 15 秒
|
||||
aspectRatio: "16:9" | "9:16" | "1:1"; // 仅支持这三种比例
|
||||
}
|
||||
interface OpenAIVideoConfig extends BaseVideoConfig {
|
||||
duration: 10 | 15; // 仅支持 10 或 15 秒
|
||||
aspectRatio: Exclude<VideoAspectRatio, "adaptive">; // 不支持 adaptive
|
||||
}
|
||||
type VideoConfig = DoubaoVideoConfig | RunninghubVideoConfig | OpenAIVideoConfig;
|
||||
|
||||
const urlToBase64 = async (url: string): Promise<string> => {
|
||||
const res = await axios.get(url, { responseType: "arraybuffer" });
|
||||
@ -110,127 +107,6 @@ const uploadBase64ToRunninghub = async (base64Image: string, apiKey: string, bas
|
||||
}
|
||||
};
|
||||
|
||||
const generators = {
|
||||
volcengine: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => {
|
||||
if (config.size == "1K") config.size = "2K";
|
||||
apiKey = apiKey.replace("Bearer ", "");
|
||||
const body: Record<string, any> = {
|
||||
model,
|
||||
prompt: config.prompt,
|
||||
size: config.size,
|
||||
response_format: "url",
|
||||
sequential_image_generation: "disabled",
|
||||
stream: false,
|
||||
watermark: false,
|
||||
};
|
||||
// 图生图：存在图片时添加 image 字段
|
||||
if (config.imageBase64) {
|
||||
body.image = config.imageBase64;
|
||||
}
|
||||
const res = await axios.post(`https://ark.cn-beijing.volces.com/api/v3/images/generations`, body, {
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
});
|
||||
return res.data.data[0].url;
|
||||
},
|
||||
|
||||
gemini: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => {
|
||||
apiKey = apiKey.replace("Bearer ", "");
|
||||
const messages = [
|
||||
...(config.systemPrompt ? [{ role: "system", content: config.systemPrompt }] : []),
|
||||
{ role: "user", content: config.prompt },
|
||||
...config.imageBase64.map((img) => ({ role: "user", content: { image: img } })),
|
||||
];
|
||||
const res = await axios.post(
|
||||
`${baseURL}/chat/completions`,
|
||||
{ model, stream: false, messages, extra_body: { google: { image_config: { aspect_ratio: config.aspectRatio, image_size: config.size } } } },
|
||||
{ headers: { Authorization: "Bearer " + apiKey } },
|
||||
);
|
||||
|
||||
return res.data.choices[0].message.content;
|
||||
},
|
||||
|
||||
runninghub: async (config: ImageConfig, apiKey: string, baseURL: string) => {
|
||||
apiKey = apiKey.replace("Bearer ", "");
|
||||
const imageUrls = await Promise.all(config.imageBase64.map((base64Image) => uploadBase64ToRunninghub(base64Image, apiKey, baseURL)));
|
||||
|
||||
const endpoint = config.imageBase64.length === 0 ? "/openapi/v2/rhart-image-n-pro/text-to-image" : "/openapi/v2/rhart-image-n-pro/edit";
|
||||
const taskRes = await axios.post(
|
||||
`https://www.runninghub.cn${endpoint}`,
|
||||
{ prompt: config.prompt, resolution: config.size, aspectRatio: config.aspectRatio, ...(imageUrls.length > 0 && { imageUrls }) },
|
||||
{ headers: { Authorization: "Bearer " + apiKey } },
|
||||
);
|
||||
const taskId = taskRes.data.taskId;
|
||||
if (!taskId) throw new Error(`任务创建失败，${JSON.stringify(taskRes.data)}`);
|
||||
|
||||
return pollTask(async () => {
|
||||
const res = await axios.post(`https://www.runninghub.cn/task/openapi/outputs`, { taskId, apiKey: apiKey });
|
||||
const { code, msg, data } = res.data;
|
||||
if (code === 0 && msg === "success") return { completed: true, imageUrl: data?.[0]?.fileUrl };
|
||||
if (code === 804 || code === 813) return { completed: false };
|
||||
if (code === 805) return { completed: false, error: `任务失败: ${data?.[0]?.failedReason?.exception_message || "未知原因"}` };
|
||||
return { completed: false, error: `未知状态: code=${code}, msg=${msg}` };
|
||||
});
|
||||
},
|
||||
|
||||
apimart: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => {
|
||||
apiKey = apiKey.replace("Bearer ", "");
|
||||
const taskRes = await axios.post(
|
||||
`https://api.apimart.ai/v1/images/generations`,
|
||||
{ model: "gemini-3-pro-image-preview", prompt: config.prompt, size: config.aspectRatio, n: 1, resolution: config.size },
|
||||
{ headers: { Authorization: apiKey } },
|
||||
);
|
||||
|
||||
if (taskRes.data.code !== 200 || !taskRes.data.data?.[0]?.task_id) throw new Error("任务创建失败: " + JSON.stringify(taskRes.data));
|
||||
|
||||
const taskId = taskRes.data.data[0].task_id;
|
||||
return pollTask(async () => {
|
||||
const res = await axios.get(`https://api.apimart.ai/v1/tasks/${taskId}`, { headers: { Authorization: apiKey }, params: { language: "en" } });
|
||||
if (res.data.code !== 200) return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` };
|
||||
const { status, result } = res.data.data;
|
||||
if (status === "completed") return { completed: true, imageUrl: result?.images?.[0]?.url?.[0] };
|
||||
if (status === "failed" || status === "cancelled") return { completed: false, error: `任务${status}` };
|
||||
return { completed: false };
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export const generateImage = async (config: ImageConfig, replaceConfig?: Awaited<ReturnType<typeof u.getConfig<"image">>>): Promise<string> => {
|
||||
let { model, apiKey, baseURL, manufacturer } = await u.getConfig("image");
|
||||
if (replaceConfig) {
|
||||
model = replaceConfig.model || model;
|
||||
apiKey = replaceConfig.apiKey || apiKey;
|
||||
baseURL = replaceConfig.baseURL || baseURL;
|
||||
manufacturer = replaceConfig.manufacturer || manufacturer;
|
||||
}
|
||||
const generator = generators[manufacturer as keyof typeof generators];
|
||||
if (!generator) throw new Error(`不支持的厂商: ${manufacturer}`);
|
||||
|
||||
let imageUrl = await generator(config, apiKey ?? "", baseURL ?? "", model ?? "");
|
||||
if (!config.resType) config.resType = "b64";
|
||||
if (config.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl);
|
||||
return imageUrl;
|
||||
};
|
||||
|
||||
type VideoAspectRatio = "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive";
|
||||
interface BaseVideoConfig {
|
||||
prompt: string;
|
||||
savePath: string;
|
||||
imageBase64?: string[]; // 单张参考图片 base64
|
||||
}
|
||||
interface DoubaoVideoConfig extends BaseVideoConfig {
|
||||
duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒
|
||||
aspectRatio: VideoAspectRatio;
|
||||
audio?: boolean;
|
||||
}
|
||||
interface RunninghubVideoConfig extends BaseVideoConfig {
|
||||
duration: 10 | 15; // 仅支持 10 或 15 秒
|
||||
aspectRatio: "16:9" | "9:16" | "1:1"; // 仅支持这三种比例
|
||||
}
|
||||
interface OpenAIVideoConfig extends BaseVideoConfig {
|
||||
duration: 10 | 15; // 仅支持 10 或 15 秒
|
||||
aspectRatio: Exclude<VideoAspectRatio, "adaptive">; // 不支持 adaptive
|
||||
}
|
||||
type VideoConfig = DoubaoVideoConfig | RunninghubVideoConfig | OpenAIVideoConfig;
|
||||
const generateVideoWithConfig = async (config: VideoConfig, configItem: { model: string; apiKey: string; baseURL: string; manufacturer: string }) => {
|
||||
const { apiKey, baseURL, manufacturer, model } = configItem;
|
||||
const imageArrPath = [];
|
||||
@ -527,34 +403,35 @@ const generateVideoWithConfig = async (config: VideoConfig, configItem: { model:
|
||||
}
|
||||
return videoUrl;
|
||||
};
|
||||
export const generateVideo = async (config: VideoConfig, manufacturer: string) => {
|
||||
|
||||
export default async (config: VideoConfig, manufacturer: string) => {
|
||||
if (!config.imageBase64 || config.imageBase64.length <= 0) throw new Error("未传图片");
|
||||
const configList = await u.getConfig("video", manufacturer);
|
||||
if (!configList || configList.length === 0) {
|
||||
const configItem = await u.getConfig("video", manufacturer);
|
||||
if (!configItem) {
|
||||
throw new Error("未找到任何视频配置");
|
||||
}
|
||||
let lastError: Error | null = null;
|
||||
for (const configItem of configList) {
|
||||
// 每个配置项重试1次，共2次尝试
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const videoUrl = await generateVideoWithConfig(config, configItem);
|
||||
if (videoUrl) {
|
||||
const response = await axios.get(videoUrl, { responseType: "stream" });
|
||||
await u.oss.writeFile(config.savePath, response.data);
|
||||
return config.savePath;
|
||||
}
|
||||
return videoUrl;
|
||||
} catch (error: any) {
|
||||
lastError = error as Error;
|
||||
console.warn(`配置 ${configItem.model} 第 ${attempt + 1} 次尝试失败:`, error?.response?.data || error.message);
|
||||
// 如果是第一次尝试失败，继续重试
|
||||
if (attempt === 0) continue;
|
||||
// 第二次也失败了,跳到下一个配置项
|
||||
break;
|
||||
// for (const configItem of configList) {
|
||||
// 每个配置项重试1次，共2次尝试
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const videoUrl = await generateVideoWithConfig(config, configItem);
|
||||
if (videoUrl) {
|
||||
const response = await axios.get(videoUrl, { responseType: "stream" });
|
||||
await u.oss.writeFile(config.savePath, response.data);
|
||||
return config.savePath;
|
||||
}
|
||||
return videoUrl;
|
||||
} catch (error: any) {
|
||||
lastError = error as Error;
|
||||
console.warn(`配置 ${configItem.model} 第 ${attempt + 1} 次尝试失败:`, error?.response?.data || error.message);
|
||||
// 如果是第一次尝试失败，继续重试
|
||||
if (attempt === 0) continue;
|
||||
// 第二次也失败了,跳到下一个配置项
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
// 所有配置都失败了
|
||||
throw new Error(`所有视频配置都失败了。最后一次错误: ${lastError?.message || "未知错误"}`);
|
||||
};
|
||||
72
src/utils/ai/image/index.ts
Normal file
72
src/utils/ai/image/index.ts
Normal file
@ -0,0 +1,72 @@
|
||||
import "./type";
|
||||
import u from "@/utils";
|
||||
import modelList from "./modelList";
|
||||
import axios from "axios";
|
||||
|
||||
import volcengine from "./owned/volcengine";
|
||||
import kling from "./owned/kling";
|
||||
import vidu from "./owned/vidu";
|
||||
import runninghub from "./owned/runninghub";
|
||||
import apimart from "./owned/apimart";
|
||||
import other from "./owned/other";
|
||||
import gemini from "./owned/gemini";
|
||||
|
||||
const urlToBase64 = async (url: string): Promise<string> => {
|
||||
const res = await axios.get(url, { responseType: "arraybuffer" });
|
||||
const base64 = Buffer.from(res.data).toString("base64");
|
||||
const mimeType = res.headers["content-type"] || "image/png";
|
||||
return `data:${mimeType};base64,${base64}`;
|
||||
};
|
||||
|
||||
const modelInstance = {
|
||||
gemini: gemini,
|
||||
volcengine: volcengine,
|
||||
kling: kling,
|
||||
vidu: vidu,
|
||||
runninghub: runninghub,
|
||||
// apimart: apimart,
|
||||
other,
|
||||
} as const;
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig) => {
|
||||
console.log("%c Line:32 🥪 config", "background:#33a5ff", config);
|
||||
const { model, apiKey, baseURL, manufacturer } = { ...config };
|
||||
if (!config || !config?.model || !config?.apiKey || !config?.manufacturer) throw new Error("请检查模型配置是否正确");
|
||||
|
||||
const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance];
|
||||
if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商");
|
||||
if (manufacturer !== "other") {
|
||||
const owned = modelList.find((m) => m.model === model);
|
||||
if (!owned) throw new Error("不支持的模型");
|
||||
}
|
||||
|
||||
// 补充图片的 base64 内容类型字符串
|
||||
if (input.imageBase64 && input.imageBase64.length > 0) {
|
||||
input.imageBase64 = input.imageBase64.map((img) => {
|
||||
if (img.startsWith("data:image/")) {
|
||||
return img;
|
||||
}
|
||||
// 根据 base64 头部判断图片类型
|
||||
if (img.startsWith("/9j/")) {
|
||||
return `data:image/jpeg;base64,${img}`;
|
||||
}
|
||||
if (img.startsWith("iVBORw")) {
|
||||
return `data:image/png;base64,${img}`;
|
||||
}
|
||||
if (img.startsWith("R0lGOD")) {
|
||||
return `data:image/gif;base64,${img}`;
|
||||
}
|
||||
if (img.startsWith("UklGR")) {
|
||||
return `data:image/webp;base64,${img}`;
|
||||
}
|
||||
// 默认使用 png
|
||||
return `data:image/png;base64,${img}`;
|
||||
});
|
||||
}
|
||||
|
||||
let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL });
|
||||
console.log("%c Line:68 🍷 imageUrl", "background:#4fff4B", imageUrl);
|
||||
if (!input.resType) input.resType = "b64";
|
||||
if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl);
|
||||
return imageUrl;
|
||||
};
|
||||
71
src/utils/ai/image/modelList.ts
Normal file
71
src/utils/ai/image/modelList.ts
Normal file
@ -0,0 +1,71 @@
|
||||
interface Owned {
|
||||
manufacturer: string;
|
||||
model: string;
|
||||
grid: boolean;
|
||||
type: "t2i" | "ti2i" | "i2i";
|
||||
}
|
||||
|
||||
const modelList: Owned[] = [
|
||||
// 火山引擎
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedream-4-5-251128",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedream-4-0-250828",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
//可灵
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-image-o1",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
//gemini
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.5-flash-image",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-3-pro-image-preview",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
//Vidu
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq1",
|
||||
grid: false,
|
||||
type: "i2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
//RunningHub
|
||||
{
|
||||
manufacturer: "runninghub",
|
||||
model: "nanobanana",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
//ApiMart
|
||||
{
|
||||
manufacturer: "apimart",
|
||||
model: "nanobanana",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
];
|
||||
|
||||
export default modelList;
|
||||
25
src/utils/ai/image/owned/apimart.ts
Normal file
25
src/utils/ai/image/owned/apimart.ts
Normal file
@ -0,0 +1,25 @@
|
||||
import axios from "axios";
|
||||
import { pollTask } from "@/utils/ai/utils";
|
||||
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
const apiKey = config.apiKey.replace("Bearer ", "");
|
||||
const taskRes = await axios.post(
|
||||
`https://api.apimart.ai/v1/images/generations`,
|
||||
{ model: "gemini-3-pro-image-preview", prompt: input.prompt, size: input.aspectRatio, n: 1, resolution: input.size },
|
||||
{ headers: { Authorization: apiKey } },
|
||||
);
|
||||
|
||||
if (taskRes.data.code !== 200 || !taskRes.data.data?.[0]?.task_id) throw new Error("任务创建失败: " + JSON.stringify(taskRes.data));
|
||||
|
||||
const taskId = taskRes.data.data[0].task_id;
|
||||
return pollTask(async () => {
|
||||
const res = await axios.get(`https://api.apimart.ai/v1/tasks/${taskId}`, { headers: { Authorization: apiKey }, params: { language: "en" } });
|
||||
if (res.data.code !== 200) return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` };
|
||||
const { status, result } = res.data.data;
|
||||
if (status === "completed") return { completed: true, url: result?.images?.[0]?.url?.[0] };
|
||||
if (status === "failed" || status === "cancelled") return { completed: false, error: `任务${status}` };
|
||||
return { completed: false };
|
||||
});
|
||||
};
|
||||
58
src/utils/ai/image/owned/gemini.ts
Normal file
58
src/utils/ai/image/owned/gemini.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import "../type";
|
||||
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
||||
import { generateText, ModelMessage } from "ai";
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!input.prompt) throw new Error("缺少提示词");
|
||||
|
||||
const options: any = {};
|
||||
if (config.apiKey) options.apiKey = config.apiKey;
|
||||
if (config?.baseURL) options.baseURL = config.baseURL;
|
||||
const google = createGoogleGenerativeAI({
|
||||
...options,
|
||||
});
|
||||
|
||||
// 构建完整的提示词
|
||||
const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt;
|
||||
let promptData: ModelMessage[] | string = [];
|
||||
if (input.imageBase64 && input.imageBase64.length) {
|
||||
promptData = [{ role: "system", content: fullPrompt + `请直接输出图片` }];
|
||||
promptData.push({
|
||||
role: "user",
|
||||
content: input.imageBase64.map((i) => ({
|
||||
type: "image",
|
||||
image: i,
|
||||
})),
|
||||
});
|
||||
} else {
|
||||
promptData = fullPrompt + `\n请直接输出图片`;
|
||||
}
|
||||
|
||||
const result = await generateText({
|
||||
model: google.languageModel(config.model),
|
||||
prompt: promptData,
|
||||
providerOptions: {
|
||||
google: {
|
||||
imageConfig: {
|
||||
...(config.model == "gemini-2.5-flash-image"
|
||||
? { aspectRatio: input.aspectRatio }
|
||||
: { aspectRatio: input.aspectRatio, imageSize: input.size }),
|
||||
},
|
||||
},
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
|
||||
if (!result.files.length) {
|
||||
console.error(JSON.stringify(result.response, null, 2));
|
||||
throw new Error("图片生成失败");
|
||||
}
|
||||
let imageBase64;
|
||||
for (const item of result.files) {
|
||||
imageBase64 = `data:${item.mediaType};base64,${item.base64}`;
|
||||
}
|
||||
// 返回生成的图片 base64
|
||||
return imageBase64!;
|
||||
};
|
||||
107
src/utils/ai/image/owned/kling.ts
Normal file
107
src/utils/ai/image/owned/kling.ts
Normal file
@ -0,0 +1,107 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import jwt from "jsonwebtoken";
|
||||
import u from "@/utils";
|
||||
import { pollTask } from "@/utils/ai/utils";
|
||||
|
||||
function generateJwtToken(ak: string, sk: string): string {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const payload = {
|
||||
iss: ak,
|
||||
exp: now + 1800,
|
||||
nbf: now - 5,
|
||||
};
|
||||
return jwt.sign(payload, sk, {
|
||||
algorithm: "HS256",
|
||||
header: { alg: "HS256", typ: "JWT" },
|
||||
});
|
||||
}
|
||||
|
||||
function getApiToken(apiKey: string): string {
|
||||
const trimmedKey = apiKey.replace(/^Bearer\s+/i, "").trim();
|
||||
|
||||
if (trimmedKey.includes("|")) {
|
||||
const parts = trimmedKey.split("|");
|
||||
if (parts.length !== 2 || !parts[0].trim() || !parts[1].trim()) {
|
||||
throw new Error("API Key格式错误，请使用 ak|sk 格式");
|
||||
}
|
||||
return generateJwtToken(parts[0].trim(), parts[1].trim());
|
||||
}
|
||||
|
||||
return trimmedKey;
|
||||
}
|
||||
|
||||
async function processImages(imageBase64: string[]): Promise<Array<{ image: string }>> {
|
||||
let images = imageBase64.filter((img) => img?.trim());
|
||||
if (images.length === 0) return [];
|
||||
|
||||
// 压缩所有图片到10MB以内
|
||||
images = await Promise.all(images.map((img) => u.imageTools.compressImage(img, "10mb")));
|
||||
|
||||
// 参考主体数量和参考图片数量之和不得超过10
|
||||
if (images.length > 10) {
|
||||
const mergeImageList = images.splice(9);
|
||||
const mergedImage = await u.imageTools.mergeImages(mergeImageList, "10mb");
|
||||
images.push(mergedImage);
|
||||
}
|
||||
|
||||
return images.map((img) => ({
|
||||
image: img.replace(/^data:image\/[a-z]+;base64,/i, ""),
|
||||
}));
|
||||
}
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!input.prompt) throw new Error("缺少提示词，prompt为必填项");
|
||||
|
||||
const authorization = `Bearer ${getApiToken(config.apiKey)}`;
|
||||
const baseURL = (config.baseURL ?? "https://api-beijing.klingai.com/v1/images/omni-image").replace(/\/+$/, "");
|
||||
const imageList = await processImages(input.imageBase64);
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model_name: config.model || "kling-image-o1",
|
||||
prompt: input.prompt,
|
||||
n: 1,
|
||||
...(input.size !== "4K" && { resolution: input.size.toLowerCase() }),
|
||||
...(imageList.length > 0 && { image_list: imageList }),
|
||||
};
|
||||
|
||||
const headers = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
};
|
||||
|
||||
try {
|
||||
const { data: createData } = await axios.post(baseURL, body, { headers });
|
||||
|
||||
if (createData.code !== 0) {
|
||||
throw new Error(createData.message || "创建任务失败");
|
||||
}
|
||||
|
||||
const taskId = createData.data?.task_id;
|
||||
if (!taskId) throw new Error("未获取到任务ID");
|
||||
|
||||
const queryUrl = `${baseURL}/${taskId}`;
|
||||
return await pollTask(async () => {
|
||||
const { data: queryData } = await axios.get(queryUrl, { headers });
|
||||
|
||||
if (queryData.code !== 0) {
|
||||
return { completed: false, error: queryData.message || "查询任务失败" };
|
||||
}
|
||||
|
||||
const { task_status, task_status_msg, task_result } = queryData.data || {};
|
||||
|
||||
if (task_status === "failed") {
|
||||
return { completed: false, error: task_status_msg || "图片生成失败" };
|
||||
}
|
||||
|
||||
if (task_status === "succeed") {
|
||||
return { completed: true, url: task_result?.images?.[0]?.url };
|
||||
}
|
||||
|
||||
return { completed: false };
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(u.error(error).message || "可灵图片生成失败");
|
||||
}
|
||||
}
|
||||
115
src/utils/ai/image/owned/other.ts
Normal file
115
src/utils/ai/image/owned/other.ts
Normal file
@ -0,0 +1,115 @@
|
||||
import "../type";
|
||||
import { generateImage, generateText, ModelMessage } from "ai";
|
||||
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
||||
import axios from "axios";
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!config.baseURL) throw new Error("缺少baseUrl");
|
||||
|
||||
const apiKey = config.apiKey.replace("Bearer ", "");
|
||||
|
||||
const otherProvider = createOpenAICompatible({
|
||||
name: "xixixi",
|
||||
baseURL: config.baseURL,
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
});
|
||||
|
||||
// 根据 size 配置映射到具体尺寸
|
||||
const sizeMap: Record<string, `${number}x${number}`> = {
|
||||
"1K": "1024x1024",
|
||||
"2K": "2048x2048",
|
||||
"4K": "4096x4096",
|
||||
};
|
||||
// 构建完整的提示词
|
||||
const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt;
|
||||
const model = config.model;
|
||||
if (model.includes("gemini") || model.includes("nano")) {
|
||||
let promptData;
|
||||
if (input.imageBase64 && input.imageBase64.length) {
|
||||
promptData = [{ role: "system", content: fullPrompt + `请直接输出图片` }];
|
||||
(promptData as ModelMessage[]).push({
|
||||
role: "user",
|
||||
content: input.imageBase64.map((i) => ({
|
||||
type: "image",
|
||||
image: i,
|
||||
})),
|
||||
});
|
||||
} else {
|
||||
promptData = fullPrompt + `请直接输出图片`;
|
||||
}
|
||||
|
||||
const result = await generateText({
|
||||
model: otherProvider.languageModel(model),
|
||||
prompt: promptData as string | ModelMessage[],
|
||||
providerOptions: {
|
||||
google: {
|
||||
imageConfig: {
|
||||
...(config.model == "gemini-2.5-flash-image"
|
||||
? { aspectRatio: input.aspectRatio }
|
||||
: { aspectRatio: input.aspectRatio, imageSize: input.size }),
|
||||
},
|
||||
responseModalities: ["IMAGE"],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
if (result.files && result.files.length) {
|
||||
let imageBase64;
|
||||
for (const item of result.files) {
|
||||
imageBase64 = `data:${item.mediaType};base64,${item.base64}`;
|
||||
}
|
||||
// 返回生成的图片 base64
|
||||
return imageBase64!;
|
||||
} else {
|
||||
if (!result.text) {
|
||||
console.error(JSON.stringify(result.response, null, 2));
|
||||
throw new Error("图片生成失败");
|
||||
}
|
||||
const mdMatch = result.text.match(/^!\[.*?\]\((.+?)\)$/);
|
||||
if (mdMatch) {
|
||||
const imgInfo = mdMatch[1];
|
||||
const base64InMd = imgInfo.match(/data:image\/[a-z]+;base64,(.+)/);
|
||||
if (base64InMd) {
|
||||
return imgInfo;
|
||||
} else {
|
||||
return await urlToBase64(imgInfo);
|
||||
}
|
||||
}
|
||||
const base64Match = result.text.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
|
||||
if (base64Match) {
|
||||
return "data:image/jpeg;base64," + base64Match[1];
|
||||
}
|
||||
// 检查是否为图片直链 url
|
||||
if (/^https?:\/\/.*\.(png|jpg|jpeg|gif|webp|bmp)$/i.test(result.text)) {
|
||||
return await urlToBase64(result.text);
|
||||
}
|
||||
|
||||
// 默认情况
|
||||
return result.text;
|
||||
}
|
||||
} else {
|
||||
const { image } = await generateImage({
|
||||
model: otherProvider.imageModel(model),
|
||||
prompt:
|
||||
input.imageBase64 && input.imageBase64.length
|
||||
? { text: fullPrompt + `请直接输出图片`, images: input.imageBase64 }
|
||||
: fullPrompt + `请直接输出图片`,
|
||||
aspectRatio: input.aspectRatio as "1:1" | "3:4" | "4:3" | "9:16" | "16:9",
|
||||
size: sizeMap[input.size] ?? "1024x1024",
|
||||
});
|
||||
|
||||
return image.base64;
|
||||
}
|
||||
};
|
||||
|
||||
async function urlToBase64(url: string): Promise<string> {
|
||||
const res = await axios.get(url, { responseType: "arraybuffer" });
|
||||
const base64 = Buffer.from(res.data).toString("base64");
|
||||
const mimeType = res.headers["content-type"] || "image/png";
|
||||
return `data:${mimeType};base64,${base64}`;
|
||||
}
|
||||
89
src/utils/ai/image/owned/runninghub.ts
Normal file
89
src/utils/ai/image/owned/runninghub.ts
Normal file
@ -0,0 +1,89 @@
|
||||
import axios from "axios";
|
||||
import FormData from "form-data";
|
||||
import sharp from "sharp";
|
||||
import { pollTask } from "@/utils/ai/utils";
|
||||
|
||||
// 上传 base64 图片到 runninghub
|
||||
const uploadBase64ToRunninghub = async (base64Image: string, apiKey: string, baseURL: string): Promise<string> => {
|
||||
try {
|
||||
apiKey = apiKey.replace("Bearer ", "");
|
||||
// 移除 base64 前缀
|
||||
const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, "");
|
||||
let buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
// 压缩图片到 7MB 以下
|
||||
const MAX_SIZE = 7 * 1024 * 1024; // 7MB
|
||||
if (buffer.length > MAX_SIZE) {
|
||||
let quality = 90;
|
||||
|
||||
while (buffer.length > MAX_SIZE && quality > 10) {
|
||||
const compressed = await sharp(buffer).jpeg({ quality, mozjpeg: true }).toBuffer();
|
||||
buffer = Buffer.from(compressed);
|
||||
quality -= 10;
|
||||
}
|
||||
|
||||
// 如果仍然超过限制，进一步调整尺寸
|
||||
if (buffer.length > MAX_SIZE) {
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
const scale = Math.sqrt(MAX_SIZE / buffer.length);
|
||||
|
||||
const resized = await sharp(buffer)
|
||||
.resize({
|
||||
width: Math.floor((metadata.width || 1920) * scale),
|
||||
height: Math.floor((metadata.height || 1080) * scale),
|
||||
fit: "inside",
|
||||
})
|
||||
.jpeg({ quality: 80, mozjpeg: true })
|
||||
.toBuffer();
|
||||
|
||||
buffer = Buffer.from(resized);
|
||||
}
|
||||
}
|
||||
|
||||
// 创建 FormData
|
||||
const formData = new FormData();
|
||||
formData.append("file", buffer, {
|
||||
filename: "image.jpg",
|
||||
contentType: "image/jpeg",
|
||||
});
|
||||
|
||||
// 上传图片
|
||||
const uploadRes = await axios.post(`https://www.runninghub.cn/openapi/v2/media/upload/binary`, formData, {
|
||||
headers: { Authorization: `Bearer ${apiKey}` },
|
||||
});
|
||||
|
||||
if (uploadRes.data.code !== 0 || !uploadRes.data.data?.download_url) {
|
||||
throw new Error(`图片上传失败: ${JSON.stringify(uploadRes.data)}`);
|
||||
}
|
||||
|
||||
return uploadRes.data.data.download_url;
|
||||
} catch (error) {
|
||||
console.error("上传图片时发生错误:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
const apiKey = config.apiKey.replace("Bearer ", "");
|
||||
const baseURL = "https://www.runninghub.cn";
|
||||
const imageUrls = await Promise.all(input.imageBase64.map((base64Image) => uploadBase64ToRunninghub(base64Image, apiKey, baseURL)));
|
||||
|
||||
const endpoint = input.imageBase64.length === 0 ? "/openapi/v2/rhart-image-n-pro/text-to-image" : "/openapi/v2/rhart-image-n-pro/edit";
|
||||
const taskRes = await axios.post(
|
||||
`https://www.runninghub.cn${endpoint}`,
|
||||
{ prompt: input.prompt, resolution: input.size, aspectRatio: input.aspectRatio, ...(imageUrls.length > 0 && { imageUrls }) },
|
||||
{ headers: { Authorization: "Bearer " + apiKey } },
|
||||
);
|
||||
const taskId = taskRes.data.taskId;
|
||||
if (!taskId) throw new Error(`任务创建失败，${JSON.stringify(taskRes.data)}`);
|
||||
|
||||
return pollTask(async () => {
|
||||
const res = await axios.post(`https://www.runninghub.cn/task/openapi/outputs`, { taskId, apiKey: apiKey });
|
||||
const { code, msg, data } = res.data;
|
||||
if (code === 0 && msg === "success") return { completed: true, url: data?.[0]?.fileUrl };
|
||||
if (code === 804 || code === 813) return { completed: false };
|
||||
if (code === 805) return { completed: false, error: `任务失败: ${data?.[0]?.failedReason?.exception_message || "未知原因"}` };
|
||||
return { completed: false, error: `未知状态: code=${code}, msg=${msg}` };
|
||||
});
|
||||
};
|
||||
88
src/utils/ai/image/owned/vidu.ts
Normal file
88
src/utils/ai/image/owned/vidu.ts
Normal file
@ -0,0 +1,88 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import u from "@/utils";
|
||||
import { pollTask } from "@/utils/ai/utils";
|
||||
function getApiUrl(apiUrl: string) {
|
||||
if (apiUrl.includes("|")) {
|
||||
const parts = apiUrl.split("|");
|
||||
if (parts.length !== 2 || !parts[0].trim() || !parts[1].trim()) {
|
||||
throw new Error("url 格式错误，请使用 url1|url2 格式");
|
||||
}
|
||||
return { requestUrl: parts[0].trim(), queryUrl: parts[1].trim() };
|
||||
}
|
||||
throw new Error("请填写正确的url");
|
||||
}
|
||||
function template(replaceObj: Record<string, any>, url: string) {
|
||||
return url.replace(/\{(\w+)\}/g, (match, varName) => {
|
||||
return replaceObj.hasOwnProperty(varName) ? replaceObj[varName] : match;
|
||||
});
|
||||
}
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
const apiKey = "Token " + config.apiKey.replace(/Token\s+/g, "").trim();
|
||||
const viduq2Ratio = ["16:9", "9:16", "1:1", "3:4", "4:3", "21:9", "2:3", "3:2"];
|
||||
const viduq1Ratio = ["16:9", "9:16", "1:1", "3:4", "4:3"];
|
||||
let images: string[] = [];
|
||||
const baseImages = input.imageBase64;
|
||||
// 如果图片总数大于7，合并第7张及以后的图片
|
||||
if (baseImages) {
|
||||
if (baseImages.length > 7) {
|
||||
// 前6张原图
|
||||
images = baseImages.slice(0, 6);
|
||||
// 第7张及以后的图片进行合并
|
||||
const mergeImageList = baseImages.slice(6); // 注意此处使用slice，不会改变原数组
|
||||
const mergedImage = await u.imageTools.mergeImages(mergeImageList, "10mb");
|
||||
images.push(mergedImage);
|
||||
} else {
|
||||
// 不足7张，直接全部加入
|
||||
images = baseImages;
|
||||
}
|
||||
}
|
||||
|
||||
let size = "1080p";
|
||||
if (config.model == "viduq1") {
|
||||
if (!images.length) throw new Error(`viduq1 进行图片生成必须传入一张图片`);
|
||||
if (!viduq1Ratio.includes(input.aspectRatio)) throw new Error("不支持的图片比例:" + input.aspectRatio);
|
||||
size = "1080p";
|
||||
} else {
|
||||
if (input.size == "1K") size = "1080p";
|
||||
else size = input.size;
|
||||
if (!viduq2Ratio.includes(input.aspectRatio)) throw new Error("不支持的图片比例:" + input.aspectRatio);
|
||||
}
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model: config.model,
|
||||
prompt: input.prompt,
|
||||
aspect_ratio: input.aspectRatio,
|
||||
resolution: size,
|
||||
...(images.length && { images: images }),
|
||||
};
|
||||
|
||||
const urlObj = getApiUrl(config.baseURL! ?? "https://api.vidu.cn/ent/v2/reference2image|https://api.vidu.cn/ent/v2/tasks/{id}/creations");
|
||||
|
||||
try {
|
||||
const { data } = await axios.post(urlObj.requestUrl, body, { headers: { Authorization: apiKey } });
|
||||
|
||||
const queryUrl = template({ id: data.task_id }, urlObj.queryUrl);
|
||||
|
||||
return await pollTask(async () => {
|
||||
const { data: queryData } = await axios.get(queryUrl, { headers: { Authorization: apiKey } });
|
||||
|
||||
const { state, err_code, creations } = queryData || {};
|
||||
|
||||
if (state === "failed") {
|
||||
return { completed: false, error: err_code || "图片生成失败" };
|
||||
}
|
||||
|
||||
if (state === "success") {
|
||||
return { completed: true, url: creations?.[0]?.url };
|
||||
}
|
||||
|
||||
return { completed: false };
|
||||
});
|
||||
} catch (error: any) {
|
||||
const msg = u.error(error).message || "vidu 图片生成失败";
|
||||
throw new Error(msg);
|
||||
}
|
||||
};
|
||||
31
src/utils/ai/image/owned/volcengine.ts
Normal file
31
src/utils/ai/image/owned/volcengine.ts
Normal file
@ -0,0 +1,31 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import u from "@/utils";
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const apiKey = "Bearer " + config.apiKey.replace(/Bearer\s+/g, "").trim();
|
||||
const size = input.size === "1K" ? "2K" : input.size;
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model: config.model,
|
||||
prompt: input.prompt,
|
||||
size,
|
||||
response_format: "url",
|
||||
sequential_image_generation: "disabled",
|
||||
stream: false,
|
||||
watermark: false,
|
||||
...(input.imageBase64 && { image: input.imageBase64 }),
|
||||
};
|
||||
|
||||
const url = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/images/generations";
|
||||
try {
|
||||
const { data } = await axios.post(url, body, { headers: { Authorization: apiKey } });
|
||||
return data.data[0]?.url;
|
||||
} catch (error) {
|
||||
const msg = u.error(error).message || "Volcengine 图片生成失败";
|
||||
throw new Error(msg);
|
||||
}
|
||||
}
|
||||
14
src/utils/ai/image/type.ts
Normal file
14
src/utils/ai/image/type.ts
Normal file
@ -0,0 +1,14 @@
|
||||
interface ImageConfig {
|
||||
systemPrompt?: string;
|
||||
prompt: string;
|
||||
imageBase64: string[];
|
||||
size: "1K" | "2K" | "4K";
|
||||
aspectRatio: string;
|
||||
resType?: "url" | "b64";
|
||||
}
|
||||
|
||||
interface AIConfig {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseURL?: string;
|
||||
}
|
||||
@ -1,9 +1,11 @@
|
||||
import u from "@/utils";
|
||||
import { generateText, streamText, Output, stepCountIs, ModelMessage, LanguageModel, Tool, GenerateTextResult } from "ai";
|
||||
import { wrapLanguageModel } from "ai";
|
||||
import { devToolsMiddleware } from "@ai-sdk/devtools";
|
||||
import { parse } from "best-effort-json-parser";
|
||||
import modelList from "./modelList";
|
||||
import { z } from "zod";
|
||||
|
||||
import { OpenAIProvider } from "@ai-sdk/openai";
|
||||
interface AIInput<T extends Record<string, z.ZodTypeAny> | undefined = undefined> {
|
||||
system?: string;
|
||||
tools?: Record<string, Tool>;
|
||||
@ -17,20 +19,27 @@ interface AIConfig {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseURL?: string;
|
||||
manufacturer?: string;
|
||||
}
|
||||
|
||||
const buildOptions = async (input: AIInput<any>, config: AIConfig) => {
|
||||
const sqlTextModelConfig = await u.getConfig("text");
|
||||
const { model, apiKey, baseURL } = { ...sqlTextModelConfig, ...config };
|
||||
|
||||
const owned = modelList.find((m) => m.model === model);
|
||||
const buildOptions = async (input: AIInput<any>, config: AIConfig = {}) => {
|
||||
if (!config || !config?.model || !config?.apiKey || !config?.manufacturer) throw new Error("请检查模型配置是否正确");
|
||||
const { model, apiKey, baseURL, manufacturer } = { ...config };
|
||||
let owned;
|
||||
if (manufacturer == "other") {
|
||||
owned = modelList.find((m) => m.manufacturer === manufacturer);
|
||||
} else {
|
||||
owned = modelList.find((m) => m.model === model);
|
||||
}
|
||||
if (!owned) throw new Error("不支持的模型或厂商");
|
||||
|
||||
const modelInstance = owned.instance({ apiKey, baseURL });
|
||||
const modelInstance = owned.instance({ apiKey, baseURL: baseURL!, name: "xixixi" });
|
||||
|
||||
const maxStep = input.maxStep ?? (input.tools ? Object.keys(input.tools).length * 5 : undefined);
|
||||
const outputBuilders: Record<string, (schema: any) => any> = {
|
||||
schema: (s) => Output.object({ schema: z.object(s) }),
|
||||
schema: (s) => {
|
||||
return Output.object({ schema: z.object(s) });
|
||||
},
|
||||
object: () => {
|
||||
const jsonSchemaPrompt = `\n请按照以下 JSON Schema 格式返回结果:\n${JSON.stringify(
|
||||
z.toJSONSchema(z.object(input.output)),
|
||||
@ -42,11 +51,12 @@ const buildOptions = async (input: AIInput<any>, config: AIConfig) => {
|
||||
},
|
||||
};
|
||||
|
||||
const output = input.output ? outputBuilders[owned.responseFormat]?.(input.output) ?? null : null;
|
||||
|
||||
const output = input.output ? (outputBuilders[owned.responseFormat]?.(input.output) ?? null) : null;
|
||||
const chatModelManufacturer = ["doubao", "other", "openai"];
|
||||
const modelFn = chatModelManufacturer.includes(owned.manufacturer) ? (modelInstance as OpenAIProvider).chat(model!) : modelInstance(model!);
|
||||
return {
|
||||
config: {
|
||||
model: modelInstance(model) as LanguageModel,
|
||||
model: modelFn as LanguageModel,
|
||||
...(input.system && { system: input.system }),
|
||||
...(input.prompt ? { prompt: input.prompt } : { messages: input.messages! }),
|
||||
...(input.tools && owned.tool && { tools: input.tools }),
|
||||
@ -64,7 +74,7 @@ const ai = Object.create({}) as {
|
||||
stream(input: AIInput, config?: AIConfig): Promise<ReturnType<typeof streamText>>;
|
||||
};
|
||||
|
||||
ai.invoke = async (input: AIInput<any>, config: AIConfig = {}) => {
|
||||
ai.invoke = async (input: AIInput<any>, config: AIConfig) => {
|
||||
const options = await buildOptions(input, config);
|
||||
const result = await generateText(options.config);
|
||||
if (options.responseFormat === "object" && input.output) {
|
||||
@ -80,7 +90,7 @@ ai.invoke = async (input: AIInput<any>, config: AIConfig = {}) => {
|
||||
return result;
|
||||
};
|
||||
|
||||
ai.stream = async (input: AIInput, config: AIConfig = {}) => {
|
||||
ai.stream = async (input: AIInput, config: AIConfig) => {
|
||||
const options = await buildOptions(input, config);
|
||||
return streamText(options.config);
|
||||
};
|
||||
@ -4,6 +4,8 @@ import { createZhipu } from "zhipu-ai-provider";
|
||||
import { createQwen } from "qwen-ai-provider";
|
||||
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
||||
import { createAnthropic } from "@ai-sdk/anthropic";
|
||||
import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
||||
import { createXai } from '@ai-sdk/xai';
|
||||
|
||||
interface Owned {
|
||||
manufacturer: string;
|
||||
@ -18,7 +20,8 @@ interface Owned {
|
||||
| typeof createZhipu
|
||||
| typeof createQwen
|
||||
| typeof createGoogleGenerativeAI
|
||||
| typeof createAnthropic;
|
||||
| typeof createAnthropic
|
||||
| typeof createOpenAICompatible;
|
||||
}
|
||||
|
||||
const modelList: Owned[] = [
|
||||
@ -45,7 +48,7 @@ const modelList: Owned[] = [
|
||||
// 豆包
|
||||
{
|
||||
manufacturer: "doubao",
|
||||
model: "doubao-seed-1-8",
|
||||
model: "doubao-seed-1-8-251228",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
think: true,
|
||||
@ -54,7 +57,7 @@ const modelList: Owned[] = [
|
||||
},
|
||||
{
|
||||
manufacturer: "doubao",
|
||||
model: "doubao-seed-1-6",
|
||||
model: "doubao-seed-1-6-251015",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
think: true,
|
||||
@ -63,7 +66,7 @@ const modelList: Owned[] = [
|
||||
},
|
||||
{
|
||||
manufacturer: "doubao",
|
||||
model: "doubao-seed-1-6-lite",
|
||||
model: "doubao-seed-1-6-lite-251015",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
think: true,
|
||||
@ -72,7 +75,7 @@ const modelList: Owned[] = [
|
||||
},
|
||||
{
|
||||
manufacturer: "doubao",
|
||||
model: "doubao-seed-1-6-flash",
|
||||
model: "doubao-seed-1-6-flash-250828",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
think: true,
|
||||
@ -283,7 +286,7 @@ const modelList: Owned[] = [
|
||||
|
||||
// Gemini
|
||||
{
|
||||
manufacturer: "google",
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.5-pro",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
@ -292,7 +295,7 @@ const modelList: Owned[] = [
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "google",
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.5-flash",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
@ -301,7 +304,7 @@ const modelList: Owned[] = [
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "google",
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.0-flash",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
@ -310,7 +313,7 @@ const modelList: Owned[] = [
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "google",
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.0-flash-lite",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
@ -319,7 +322,7 @@ const modelList: Owned[] = [
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "google",
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-1.5-pro",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
@ -328,7 +331,7 @@ const modelList: Owned[] = [
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "google",
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-1.5-flash",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
@ -409,6 +412,44 @@ const modelList: Owned[] = [
|
||||
instance: createAnthropic,
|
||||
tool: true,
|
||||
},
|
||||
//xai
|
||||
{
|
||||
manufacturer: "xai",
|
||||
model: "grok-3",
|
||||
responseFormat: "schema",
|
||||
image: false,
|
||||
think: false,
|
||||
instance: createXai,
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "xai",
|
||||
model: "grok-4",
|
||||
responseFormat: "schema",
|
||||
image: false,
|
||||
think: false,
|
||||
instance: createXai,
|
||||
tool: true,
|
||||
},
|
||||
{
|
||||
manufacturer: "xai",
|
||||
model: "grok-4.1",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
think: false,
|
||||
instance: createXai,
|
||||
tool: true,
|
||||
},
|
||||
//其他
|
||||
{
|
||||
manufacturer: "other",
|
||||
model: "gpt-4.1",
|
||||
responseFormat: "schema",
|
||||
image: true,
|
||||
think: false,
|
||||
instance: createOpenAI,
|
||||
tool: true,
|
||||
},
|
||||
];
|
||||
|
||||
export default modelList;
|
||||
82
src/utils/ai/utils.ts
Normal file
82
src/utils/ai/utils.ts
Normal file
@ -0,0 +1,82 @@
|
||||
import modelList from "./video/modelList";
|
||||
|
||||
interface ValidateResult {
|
||||
owned: (typeof modelList)[number];
|
||||
images: string[];
|
||||
hasStartEndType: boolean;
|
||||
hasTextType: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验视频生成配置与模型是否匹配
|
||||
* @param input 视频配置
|
||||
* @param config AI配置
|
||||
* @param customOwned 自定义模型配置（如果传入则跳过模型查找）
|
||||
*/
|
||||
export const validateVideoConfig = (input: VideoConfig, config: AIConfig, customOwned?: (typeof modelList)[number]): ValidateResult => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
const owned = customOwned ?? modelList.find((m) => m.model === config.model);
|
||||
if (!owned) throw new Error(`不支持的模型: ${config.model}`);
|
||||
const images = input.imageBase64 ?? [];
|
||||
// 校验图片数量与模型类型是否匹配
|
||||
const hasTextType = owned.type.includes("text");
|
||||
const hasSingleImageType = owned.type.includes("singleImage");
|
||||
const hasStartEndType = owned.type.some((t) => ["startEndRequired", "endFrameOptional", "startFrameOptional"].includes(t));
|
||||
const hasMultiImageType = owned.type.includes("multiImage");
|
||||
const hasReferenceType = owned.type.includes("reference");
|
||||
if (images.length === 0 && !hasTextType) {
|
||||
throw new Error(`模型 ${config.model} 不支持纯文本生成，需要提供图片`);
|
||||
}
|
||||
if (images.length === 1 && !hasSingleImageType && !hasStartEndType && !hasReferenceType) {
|
||||
throw new Error(`模型 ${config.model} 不支持单图模式`);
|
||||
}
|
||||
if (images.length === 2 && !hasStartEndType) {
|
||||
throw new Error(`模型 ${config.model} 不支持首尾帧模式`);
|
||||
}
|
||||
if (images.length > 2 && !hasMultiImageType) {
|
||||
throw new Error(`模型 ${config.model} 不支持多图模式`);
|
||||
}
|
||||
// 校验duration和resolution是否在支持范围内
|
||||
const validDurationResolution = owned.durationResolutionMap.some((map) => {
|
||||
const durationMatch = map.duration.includes(input.duration);
|
||||
const resolutionMatch =
|
||||
// 若 map.resolution 和 input.resolution 均为空，视为匹配
|
||||
(!input.resolution && map.resolution.length === 0) ||
|
||||
// 否则匹配 includes
|
||||
map.resolution.includes(input.resolution as (typeof map.resolution)[number]);
|
||||
return durationMatch && resolutionMatch;
|
||||
});
|
||||
if (!validDurationResolution) {
|
||||
const supportedDurations = [...new Set(owned.durationResolutionMap.flatMap((m) => m.duration))].sort((a, b) => a - b);
|
||||
const supportedResolutions = [...new Set(owned.durationResolutionMap.flatMap((m) => m.resolution))];
|
||||
throw new Error(
|
||||
`不支持的duration(${input.duration})或resolution(${input.resolution})组合。` +
|
||||
`支持的duration: ${supportedDurations.join(", ")}，支持的resolution: ${supportedResolutions.join(", ")}`,
|
||||
);
|
||||
}
|
||||
// 校验音频设置
|
||||
if (input.audio && !owned.audio) {
|
||||
throw new Error(`模型 ${config.model} 不支持生成音频`);
|
||||
}
|
||||
// 校验宽高比（仅文本生视频需要）
|
||||
if (hasTextType && images.length === 0 && owned.aspectRatio.length > 0) {
|
||||
if (!owned.aspectRatio.includes(input.aspectRatio as `${number}:${number}`)) {
|
||||
throw new Error(`模型 ${config.model} 不支持宽高比 ${input.aspectRatio}，支持的宽高比: ${owned.aspectRatio.join(", ")}`);
|
||||
}
|
||||
}
|
||||
return { owned, images, hasStartEndType, hasTextType };
|
||||
};
|
||||
|
||||
export const pollTask = async (
|
||||
queryFn: () => Promise<{ completed: boolean; url?: string; error?: string }>,
|
||||
maxAttempts = 500,
|
||||
interval = 2000,
|
||||
): Promise<string> => {
|
||||
for (let i = 0; i < maxAttempts; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, interval));
|
||||
const { completed, url, error } = await queryFn();
|
||||
if (error) throw new Error(error);
|
||||
if (completed && url) return url;
|
||||
}
|
||||
throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`);
|
||||
};
|
||||
66
src/utils/ai/video/index.ts
Normal file
66
src/utils/ai/video/index.ts
Normal file
@ -0,0 +1,66 @@
|
||||
import "./type";
|
||||
import u from "@/utils";
|
||||
import modelList from "./modelList";
|
||||
import axios from "axios";
|
||||
|
||||
import volcengine from "./owned/volcengine";
|
||||
import kling from "./owned/kling";
|
||||
import vidu from "./owned/vidu";
|
||||
import wan from "./owned/wan";
|
||||
import runninghub from "./owned/runninghub";
|
||||
import gemini from "./owned/gemini";
|
||||
import apimart from "./owned/apimart";
|
||||
import other from "./owned/other";
|
||||
|
||||
const modelInstance = {
|
||||
volcengine: volcengine,
|
||||
kling: kling,
|
||||
vidu: vidu,
|
||||
wan: wan,
|
||||
gemini: gemini,
|
||||
runninghub: runninghub,
|
||||
apimart: apimart,
|
||||
// other: other,
|
||||
} as const;
|
||||
|
||||
export default async (input: VideoConfig, config?: AIConfig) => {
|
||||
const { model, apiKey, baseURL, manufacturer } = { ...config };
|
||||
if (!config || !config?.model || !config?.apiKey) throw new Error("请检查模型配置是否正确");
|
||||
|
||||
const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance];
|
||||
if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的视频厂商");
|
||||
const owned = modelList.find((m) => m.model === model);
|
||||
if (!owned) throw new Error("不支持的模型");
|
||||
|
||||
// 补充图片的 base64 内容类型字符串
|
||||
if (input.imageBase64 && input.imageBase64.length > 0) {
|
||||
input.imageBase64 = input.imageBase64.map((img) => {
|
||||
if (img.startsWith("data:image/")) {
|
||||
return img;
|
||||
}
|
||||
// 根据 base64 头部判断图片类型
|
||||
if (img.startsWith("/9j/")) {
|
||||
return `data:image/jpeg;base64,${img}`;
|
||||
}
|
||||
if (img.startsWith("iVBORw")) {
|
||||
return `data:image/png;base64,${img}`;
|
||||
}
|
||||
if (img.startsWith("R0lGOD")) {
|
||||
return `data:image/gif;base64,${img}`;
|
||||
}
|
||||
if (img.startsWith("UklGR")) {
|
||||
return `data:image/webp;base64,${img}`;
|
||||
}
|
||||
// 默认使用 png
|
||||
return `data:image/png;base64,${img}`;
|
||||
});
|
||||
}
|
||||
|
||||
let videoUrl = await manufacturerFn(input, { model, apiKey, baseURL });
|
||||
if (videoUrl) {
|
||||
const response = await axios.get(videoUrl, { responseType: "stream" });
|
||||
await u.oss.writeFile(input.savePath, response.data);
|
||||
return input.savePath;
|
||||
}
|
||||
return videoUrl;
|
||||
};
|
||||
489
src/utils/ai/video/modelList.ts
Normal file
489
src/utils/ai/video/modelList.ts
Normal file
@ -0,0 +1,489 @@
|
||||
type VideoGenerationType =
|
||||
| "singleImage" // 单图
|
||||
| "startEndRequired" // 首尾帧（两张都得有）
|
||||
| "endFrameOptional" // 首尾帧（尾帧可选）
|
||||
| "startFrameOptional" // 首尾帧（首帧可选）
|
||||
| "multiImage" // 多图模式
|
||||
| "reference" // 参考图模式
|
||||
| "text"; // 文本生视频
|
||||
|
||||
interface DurationResolutionMap {
|
||||
duration: number[];
|
||||
resolution: (`${number}p` | `${number}k`)[];
|
||||
}
|
||||
interface Owned {
|
||||
manufacturer: string;
|
||||
model: string;
|
||||
durationResolutionMap: DurationResolutionMap[];
|
||||
aspectRatio: `${number}:${number}`[];
|
||||
type: VideoGenerationType[];
|
||||
audio: boolean;
|
||||
}
|
||||
|
||||
const modelList: Owned[] = [
|
||||
// ================== 火山引擎/豆包系列 ==================
|
||||
// doubao-seedance-1-5-pro 文生视频/图生视频
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedance-1-5-pro-251215",
|
||||
durationResolutionMap: [{ duration: [4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||
type: ["text", "endFrameOptional"],
|
||||
audio: true,
|
||||
},
|
||||
// doubao-seedance-1-0-pro 文生视频/图生视频
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedance-1-0-pro-250528",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||
type: ["text", "endFrameOptional"],
|
||||
audio: false,
|
||||
},
|
||||
// doubao-seedance-1-0-pro-fast 文生视频/图生视频
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedance-1-0-pro-fast-251015",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||
type: ["text", "singleImage"],
|
||||
audio: false,
|
||||
},
|
||||
// doubao-seedance-1-0-lite-i2v 图生视频（仅支持图片模式）
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedance-1-0-lite-i2v-250428",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["endFrameOptional", "reference"],
|
||||
audio: false,
|
||||
},
|
||||
// doubao-seedance-1-0-lite-t2v 文生视频（仅支持文本模式）
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedance-1-0-lite-t2v-250428",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// ================== 可灵系列 ==================
|
||||
// kling-v1(STD) 文生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v1(STD)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["720p"] }],
|
||||
aspectRatio: ["16:9", "1:1", "9:16"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v1(STD) 图生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v1(STD)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["720p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v1(PRO) 文生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v1(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: ["16:9", "1:1", "9:16"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v1(PRO) 图生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v1(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v1-6(PRO) 文生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v1-6(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: ["16:9", "1:1", "9:16"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v1-6(PRO) 图生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v1-6(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v2-5-turbo(PRO) 文生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v2-5-turbo(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: ["16:9", "1:1", "9:16"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v2-5-turbo(PRO) 图生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v2-5-turbo(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v2-6(PRO) 文生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v2-6(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: ["16:9", "1:1", "9:16"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// kling-v2-6(PRO) 图生视频
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-v2-6(PRO)",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// ================== ViduQ3系列 ==================
|
||||
// viduq3-pro 文生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq3-pro",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"],
|
||||
type: ["text"],
|
||||
audio: true,
|
||||
},
|
||||
// viduq3-pro 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq3-pro",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: true,
|
||||
},
|
||||
// viduq2-pro-fast 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2-pro-fast",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage", "startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq2-pro 文生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2-pro",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq2-pro 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2-pro",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage", "reference", "startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq2-turbo 文生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2-turbo",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq2-turbo 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2-turbo",
|
||||
durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage", "reference", "startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq1 文生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq1",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "1:1"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq1 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq1",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage", "reference", "startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// viduq1-classic 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq1-classic",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage", "startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// vidu2.0 图生视频
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "vidu2.0",
|
||||
durationResolutionMap: [
|
||||
{ duration: [4], resolution: ["360p", "720p", "1080p"] },
|
||||
{ duration: [8], resolution: ["720p"] },
|
||||
],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage", "reference", "startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// ================== 万象系列 ==================
|
||||
// wan2.6-t2v 文生视频（有声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.6-t2v",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], resolution: ["720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"],
|
||||
type: ["text"],
|
||||
audio: true,
|
||||
},
|
||||
// wan2.5-t2v-preview 文生视频（有声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.5-t2v-preview",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"],
|
||||
type: ["text"],
|
||||
audio: true,
|
||||
},
|
||||
// wan2.2-t2v-plus 文生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.2-t2v-plus",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["480p", "1080p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// wanx2.1-t2v-turbo 文生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wanx2.1-t2v-turbo",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["480p", "720p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// wanx2.1-t2v-plus 文生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wanx2.1-t2v-plus",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["720p"] }],
|
||||
aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"],
|
||||
type: ["text"],
|
||||
audio: false,
|
||||
},
|
||||
// wan2.6-i2v-flash 图生视频（有声视频&无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.6-i2v-flash",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], resolution: ["720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: true,
|
||||
},
|
||||
// wan2.6-i2v 图生视频（有声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.6-i2v",
|
||||
durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], resolution: ["720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: true,
|
||||
},
|
||||
// wan2.5-i2v-preview 图生视频（有声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.5-i2v-preview",
|
||||
durationResolutionMap: [{ duration: [5, 10], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: true,
|
||||
},
|
||||
// wan2.2-i2v-flash 图生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.2-i2v-flash",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: false,
|
||||
},
|
||||
// wan2.2-i2v-plus 图生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.2-i2v-plus",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["480p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: false,
|
||||
},
|
||||
// wanx2.1-i2v-plus 图生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wanx2.1-i2v-plus",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["720p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: false,
|
||||
},
|
||||
// wanx2.1-i2v-turbo 图生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wanx2.1-i2v-turbo",
|
||||
durationResolutionMap: [{ duration: [3, 4, 5], resolution: ["480p", "720p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["singleImage"],
|
||||
audio: false,
|
||||
},
|
||||
// wan2.2-kf2v-flash 首尾帧生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wan2.2-kf2v-flash",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["480p", "720p", "1080p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// wanx2.1-kf2v-plus 首尾帧生视频（无声视频）
|
||||
{
|
||||
manufacturer: "wan",
|
||||
model: "wanx2.1-kf2v-plus",
|
||||
durationResolutionMap: [{ duration: [5], resolution: ["720p"] }],
|
||||
aspectRatio: [],
|
||||
type: ["startEndRequired"],
|
||||
audio: false,
|
||||
},
|
||||
// ================== Gemini Veo 系列 ==================
|
||||
// Veo 3.1 预览版（支持音频）
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "veo-3.1-generate-preview",
|
||||
durationResolutionMap: [
|
||||
{ duration: [4, 6], resolution: ["720p"] },
|
||||
{ duration: [8], resolution: ["720p", "1080p"] },
|
||||
],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["text", "singleImage", "startEndRequired", "endFrameOptional", "reference"],
|
||||
audio: true,
|
||||
},
|
||||
// Veo 3.1 Fast 预览版（支持音频）
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "veo-3.1-fast-generate-preview",
|
||||
durationResolutionMap: [
|
||||
{ duration: [4, 6], resolution: ["720p"] },
|
||||
{ duration: [8], resolution: ["720p", "1080p"] },
|
||||
],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["text", "singleImage", "startEndRequired", "endFrameOptional", "reference"],
|
||||
audio: true,
|
||||
},
|
||||
// Veo 3 稳定版（支持音频）
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "veo-3.0-generate-preview",
|
||||
durationResolutionMap: [
|
||||
{ duration: [4, 6], resolution: ["720p"] },
|
||||
{ duration: [8], resolution: ["720p", "1080p"] },
|
||||
],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["text", "singleImage"],
|
||||
audio: true,
|
||||
},
|
||||
// Veo 3 Fast 稳定版（支持音频）
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "veo-3.0-fast-generate-preview",
|
||||
durationResolutionMap: [
|
||||
{ duration: [4, 6], resolution: ["720p"] },
|
||||
{ duration: [8], resolution: ["720p", "1080p"] },
|
||||
],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["text", "singleImage"],
|
||||
audio: true,
|
||||
},
|
||||
// Veo 2 稳定版（无音频）
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "veo-2.0-generate-001",
|
||||
durationResolutionMap: [{ duration: [5, 6, 7, 8], resolution: ["720p"] }],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["text", "singleImage"],
|
||||
audio: false,
|
||||
},
|
||||
// ================== RunningHub 系列 ==================
|
||||
// sora
|
||||
{
|
||||
manufacturer: "runninghub",
|
||||
model: "sora-2",
|
||||
durationResolutionMap: [{ duration: [10, 15], resolution: [] }],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["singleImage", "text"],
|
||||
audio: false,
|
||||
},
|
||||
// sora 2
|
||||
{
|
||||
manufacturer: "runninghub",
|
||||
model: "sora-2-pro",
|
||||
durationResolutionMap: [{ duration: [15, 25], resolution: [] }],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["singleImage", "text"],
|
||||
audio: false,
|
||||
},
|
||||
// ================== Apimart 系列 ==================
|
||||
// sora
|
||||
{
|
||||
manufacturer: "apimart",
|
||||
model: "sora-2",
|
||||
durationResolutionMap: [{ duration: [10, 15], resolution: [] }],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["singleImage", "text"],
|
||||
audio: false,
|
||||
},
|
||||
// sora 2
|
||||
{
|
||||
manufacturer: "apimart",
|
||||
model: "sora-2-pro",
|
||||
durationResolutionMap: [{ duration: [15, 25], resolution: [] }],
|
||||
aspectRatio: ["16:9", "9:16"],
|
||||
type: ["singleImage", "text"],
|
||||
audio: false,
|
||||
},
|
||||
];
|
||||
|
||||
export default modelList;
|
||||
115
src/utils/ai/video/owned/apimart.ts
Normal file
115
src/utils/ai/video/owned/apimart.ts
Normal file
@ -0,0 +1,115 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import { pollTask } from "@/utils/ai/utils";
|
||||
import modelList from "../modelList";
|
||||
|
||||
// 上传图片到 apimart 图床
|
||||
async function uploadImageToApimart(base64Image: string): Promise<string> {
|
||||
if (base64Image.startsWith("http")) {
|
||||
return base64Image;
|
||||
}
|
||||
|
||||
const presignRes = await axios.post(
|
||||
"https://apimart.ai/api/upload/presign",
|
||||
{ contentType: "image/jpeg", fileExtension: "jpeg", permanent: false },
|
||||
{ headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
|
||||
if (!presignRes.data.success || !presignRes.data.presignedUrl || !presignRes.data.cdnUrl) {
|
||||
throw new Error(`获取预签名 URL 失败: ${JSON.stringify(presignRes.data)}`);
|
||||
}
|
||||
|
||||
const { presignedUrl, cdnUrl } = presignRes.data;
|
||||
|
||||
const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, "");
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
|
||||
await axios.put(presignedUrl, buffer, {
|
||||
headers: { "Content-Type": "image/jpeg" },
|
||||
});
|
||||
|
||||
return cdnUrl;
|
||||
}
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.model) throw new Error("缺少 Model 名称");
|
||||
if (!config.apiKey) throw new Error("缺少 API Key");
|
||||
|
||||
const owned = modelList.find((m) => m.model === config.model);
|
||||
if (!owned) throw new Error(`未找到模型: ${config.model}`);
|
||||
|
||||
// 默认 baseURL 配置
|
||||
const defaultBaseUrl = "https://api.apimart.ai/v1/videos/generations|https://api.apimart.ai/v1/tasks/{taskId}";
|
||||
const [generateUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|");
|
||||
|
||||
const authorization = `Bearer ${config.apiKey}`;
|
||||
|
||||
// 上传图片到图床
|
||||
let imageUrls: string[] = [];
|
||||
if (input.imageBase64 && input.imageBase64.length > 0) {
|
||||
for (const base64Image of input.imageBase64) {
|
||||
const imageUrl = await uploadImageToApimart(base64Image);
|
||||
imageUrls.push(imageUrl);
|
||||
}
|
||||
}
|
||||
|
||||
// 构建请求体
|
||||
const requestBody: Record<string, unknown> = {
|
||||
model: config.model,
|
||||
prompt: input.prompt,
|
||||
duration: input.duration,
|
||||
aspect_ratio: input.aspectRatio,
|
||||
};
|
||||
|
||||
if (imageUrls.length > 0) {
|
||||
requestBody.image_urls = imageUrls;
|
||||
}
|
||||
|
||||
// 创建任务
|
||||
const createRes = await axios.post(generateUrl, requestBody, {
|
||||
headers: {
|
||||
Authorization: authorization,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (createRes.data.code !== 200 || !createRes.data.data?.[0]?.task_id) {
|
||||
throw new Error(`创建任务失败: ${JSON.stringify(createRes.data)}`);
|
||||
}
|
||||
|
||||
const taskId = createRes.data.data[0].task_id;
|
||||
const actualQueryUrl = queryUrl.replace("{taskId}", taskId);
|
||||
|
||||
// 轮询任务状态
|
||||
return await pollTask(async () => {
|
||||
const queryRes = await axios.get(actualQueryUrl, {
|
||||
headers: { Authorization: authorization },
|
||||
});
|
||||
|
||||
const { code, data } = queryRes.data;
|
||||
|
||||
if (code !== 200 || !data) {
|
||||
return { completed: false, error: `查询失败: ${JSON.stringify(queryRes.data)}` };
|
||||
}
|
||||
|
||||
const { status, result, error } = data;
|
||||
|
||||
switch (status) {
|
||||
case "completed":
|
||||
const videoUrl = result?.videos?.[0]?.url?.[0];
|
||||
if (!videoUrl) {
|
||||
return { completed: false, error: "未获取到视频 URL" };
|
||||
}
|
||||
return { completed: true, url: videoUrl };
|
||||
case "failed":
|
||||
return { completed: false, error: error?.message || "任务失败" };
|
||||
case "cancelled":
|
||||
return { completed: false, error: "任务已取消" };
|
||||
case "pending":
|
||||
case "processing":
|
||||
return { completed: false };
|
||||
default:
|
||||
return { completed: false, error: `未知状态: ${status}` };
|
||||
}
|
||||
});
|
||||
};
|
||||
73
src/utils/ai/video/owned/gemini.ts
Normal file
73
src/utils/ai/video/owned/gemini.ts
Normal file
@ -0,0 +1,73 @@
|
||||
import "../type";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import axios from "axios";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
|
||||
const buildInlineImage = (data: string) => ({ inlineData: { mimeType: "image/png", data } });
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const { owned, images, hasStartEndType } = validateVideoConfig(input, config);
|
||||
|
||||
const defaultBaseUrl = [
|
||||
"https://generativelanguage.googleapis.com/v1beta/models/{model}:predictLongRunning",
|
||||
"https://generativelanguage.googleapis.com/v1beta/{name}",
|
||||
].join("|");
|
||||
|
||||
const [submitUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|");
|
||||
|
||||
|
||||
const headers = { "x-goog-api-key": config.apiKey };
|
||||
|
||||
const instance: Record<string, any> = { prompt: input.prompt };
|
||||
const parameters: Record<string, any> = {
|
||||
aspectRatio: input.aspectRatio,
|
||||
durationSeconds: +input.duration,
|
||||
...(input.resolution !== "720p" && { resolution: input.resolution }),
|
||||
};
|
||||
|
||||
// 根据图片数量和模型能力决定图片用法
|
||||
const len = images.length;
|
||||
const hasRef = owned.type.includes("reference");
|
||||
const hasSingle = owned.type.includes("singleImage");
|
||||
|
||||
if (len === 2 && hasStartEndType) {
|
||||
instance.image = buildInlineImage(images[0]);
|
||||
parameters.lastFrame = buildInlineImage(images[1]);
|
||||
} else if (len === 1 && (hasSingle || hasStartEndType)) {
|
||||
instance.image = buildInlineImage(images[0]);
|
||||
} else if (len >= 1 && len <= 3 && hasRef) {
|
||||
parameters.referenceImages = images.map((img) => ({ image: buildInlineImage(img), referenceType: "asset" }));
|
||||
}
|
||||
|
||||
const { data } = await axios.post(
|
||||
submitUrl.replace("{model}", config.model),
|
||||
{ instances: [instance], parameters },
|
||||
{ headers: { ...headers, "Content-Type": "application/json" } },
|
||||
);
|
||||
|
||||
if (!data.name) throw new Error("未获取到操作名称");
|
||||
|
||||
return pollTask(async () => {
|
||||
const { data: status } = await axios.get(queryUrl.replace("{name}", data.name), { headers });
|
||||
|
||||
const { done, response, error } = status;
|
||||
|
||||
|
||||
if (!done) return { completed: false };
|
||||
if (error) return { completed: false, error: `任务失败: ${error.message || JSON.stringify(error)}` };
|
||||
|
||||
const videoUri = response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri;
|
||||
|
||||
if (!videoUri) return { completed: false, error: "未获取到视频下载地址" };
|
||||
|
||||
const videoRes = await axios.get(videoUri, { headers, responseType: "arraybuffer", maxRedirects: 5 });
|
||||
const savePath = input.savePath.endsWith(".mp4") ? input.savePath : path.join(input.savePath, `gemini_${Date.now()}.mp4`);
|
||||
fs.writeFileSync(savePath, Buffer.from(videoRes.data));
|
||||
|
||||
return { completed: true, url: savePath };
|
||||
});
|
||||
};
|
||||
90
src/utils/ai/video/owned/kling.ts
Normal file
90
src/utils/ai/video/owned/kling.ts
Normal file
@ -0,0 +1,90 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!config.baseURL) throw new Error("缺少baseURL配置");
|
||||
|
||||
const { images } = validateVideoConfig(input, config);
|
||||
|
||||
// 解析URL配置：图生视频|文生视频|查询地址
|
||||
const defaultBaseUrl =
|
||||
"https://api-beijing.klingai.com/v1/videos/image2video|https://api-beijing.klingai.com/v1/videos/text2video|https://api-beijing.klingai.com/v1/videos/text2video/{taskId}";
|
||||
const [image2videoUrl, text2videoUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|");
|
||||
|
||||
const headers = {
|
||||
Authorization: `Bearer ${config.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
|
||||
// 解析模型名称和模式，例如 "kling-v2-6(PRO)" => modelName: "kling-v2-6", mode: "pro"
|
||||
const modelMatch = config.model!.match(/^(.+)\((STD|PRO)\)$/i);
|
||||
const modelName = modelMatch ? modelMatch[1] : config.model;
|
||||
const mode = modelMatch ? (modelMatch[2].toLowerCase() as "std" | "pro") : "std";
|
||||
|
||||
// 判断是图生视频还是文生视频
|
||||
const hasImage = images.length > 0;
|
||||
const createUrl = hasImage ? image2videoUrl : text2videoUrl;
|
||||
|
||||
// 去除图片的内容类型前缀（kling要求纯base64）
|
||||
const stripDataUrl = (str: string) => str.replace(/^data:image\/[^;]+;base64,/, "");
|
||||
|
||||
// 构建请求体
|
||||
const body: Record<string, unknown> = {
|
||||
model_name: modelName,
|
||||
mode,
|
||||
duration: String(input.duration),
|
||||
prompt: input.prompt,
|
||||
aspect_ratio: input.aspectRatio,
|
||||
};
|
||||
|
||||
if (hasImage) {
|
||||
// 图生视频：首帧和尾帧
|
||||
body.image = stripDataUrl(images[0]);
|
||||
if (images.length > 1) {
|
||||
body.image_tail = stripDataUrl(images[1]);
|
||||
}
|
||||
}
|
||||
|
||||
// 创建任务
|
||||
const createResponse = await axios.post(createUrl, body, { headers });
|
||||
const createData = createResponse.data;
|
||||
if (createData.code !== 0) {
|
||||
throw new Error(`创建任务失败: ${createData.message || "未知错误"}`);
|
||||
}
|
||||
|
||||
const taskId = createData.data?.task_id;
|
||||
if (!taskId) {
|
||||
throw new Error("创建任务失败: 未返回任务ID");
|
||||
}
|
||||
|
||||
// 轮询任务状态
|
||||
return await pollTask(async () => {
|
||||
const queryResponse = await axios.get(`${queryUrl.replace("{taskId}", taskId)}`, { headers });
|
||||
const queryData = queryResponse.data;
|
||||
if (queryData.code !== 0) {
|
||||
return { completed: false, error: `查询失败: ${queryData.message || "未知错误"}` };
|
||||
}
|
||||
|
||||
const task = queryData.data;
|
||||
const taskStatus = task?.task_status;
|
||||
|
||||
switch (taskStatus) {
|
||||
case "succeed": {
|
||||
const videoUrl = task?.task_result?.videos?.[0]?.url;
|
||||
if (!videoUrl) {
|
||||
return { completed: false, error: "任务成功但未返回视频URL" };
|
||||
}
|
||||
return { completed: true, url: videoUrl };
|
||||
}
|
||||
case "failed":
|
||||
return { completed: false, error: `任务失败: ${task?.task_status_msg || "未知原因"}` };
|
||||
case "submitted":
|
||||
case "processing":
|
||||
return { completed: false };
|
||||
default:
|
||||
return { completed: false, error: `未知状态: ${taskStatus}` };
|
||||
}
|
||||
});
|
||||
};
|
||||
50
src/utils/ai/video/owned/other.ts
Normal file
50
src/utils/ai/video/owned/other.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import sharp from "sharp";
|
||||
import FormData from "form-data";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
import { createOpenAI } from "@ai-sdk/openai";
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!config.baseURL) throw new Error("缺少baseURL");
|
||||
// const { owned, images, hasTextType } = validateVideoConfig(input, config);
|
||||
|
||||
const [requestUrl, queryUrl] = config.baseURL.split("|");
|
||||
|
||||
const authorization = `Bearer ${config.apiKey}`;
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("model", config.model);
|
||||
formData.append("prompt", input.prompt);
|
||||
formData.append("seconds", String(input.duration));
|
||||
|
||||
// 根据 aspectRatio 设置 size
|
||||
const sizeMap: Record<string, string> = {
|
||||
"16:9": "1920x1080",
|
||||
"9:16": "1080x1920",
|
||||
};
|
||||
formData.append("size", sizeMap[input.aspectRatio] || "1920x1080");
|
||||
if (input.imageBase64 && input.imageBase64.length) {
|
||||
const base64Data = input.imageBase64[0]!.replace(/^data:image\/\w+;base64,/, "");
|
||||
const buffer = Buffer.from(base64Data, "base64");
|
||||
formData.append("input_reference", buffer, { filename: "image.jpg", contentType: "image/jpeg" });
|
||||
}
|
||||
const { data } = await axios.post(requestUrl, formData, {
|
||||
headers: { "Content-Type": "application/json", Authorization: authorization, ...formData.getHeaders() },
|
||||
});
|
||||
if (data.status === "FAILED") throw new Error(`任务提交失败: ${data.errorMessage || "未知错误"}`);
|
||||
const taskId = data.id;
|
||||
return await pollTask(async () => {
|
||||
const { data } = await axios.get(`${queryUrl.replace("{id}", taskId)}`, {
|
||||
headers: { Authorization: authorization },
|
||||
});
|
||||
|
||||
if (data.status === "SUCCESS") {
|
||||
return data.results?.length ? { completed: true, url: data.results[0].url } : { completed: false, error: "任务成功但未返回视频链接" };
|
||||
}
|
||||
if (data.status === "FAILED") return { completed: false, error: `任务失败: ${data.errorMessage || "未知错误"}` };
|
||||
if (data.status === "QUEUED" || data.status === "RUNNING") return { completed: false };
|
||||
return { completed: false, error: `未知状态: ${data.status}` };
|
||||
});
|
||||
};
|
||||
99
src/utils/ai/video/owned/runninghub.ts
Normal file
99
src/utils/ai/video/owned/runninghub.ts
Normal file
@ -0,0 +1,99 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import sharp from "sharp";
|
||||
import FormData from "form-data";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const { owned, images, hasTextType } = validateVideoConfig(input, config);
|
||||
|
||||
const defaultBaseUrl = [
|
||||
"https://www.runninghub.cn/openapi/v2/rhart-video-s/image-to-video",
|
||||
"https://www.runninghub.cn/openapi/v2/rhart-video-s/image-to-video-pro",
|
||||
"https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video",
|
||||
"https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video-pro",
|
||||
"https://www.runninghub.cn/openapi/v2/query",
|
||||
"https://www.runninghub.cn/openapi/v2/media/upload/binary",
|
||||
].join("|");
|
||||
|
||||
const [image2videoUrl, image2videoProUrl, text2videoUrl, text2videoProUrl, queryUrl, uploadUrl] = (config.baseURL || defaultBaseUrl).split("|");
|
||||
|
||||
const isPro = owned.model === "sora-2-pro";
|
||||
const authorization = `Bearer ${config.apiKey}`;
|
||||
|
||||
// 上传 base64 图片
|
||||
const uploadImage = async (base64Image: string): Promise<string> => {
|
||||
const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, "");
|
||||
let buffer: Buffer = Buffer.from(base64Data, "base64");
|
||||
const MAX_SIZE = 5 * 1024 * 1024;
|
||||
|
||||
if (buffer.length > MAX_SIZE) {
|
||||
for (let quality = 90; buffer.length > MAX_SIZE && quality > 10; quality -= 10) {
|
||||
buffer = await sharp(buffer).jpeg({ quality, mozjpeg: true }).toBuffer();
|
||||
}
|
||||
if (buffer.length > MAX_SIZE) {
|
||||
const { width = 1920, height = 1080 } = await sharp(buffer).metadata();
|
||||
const scale = Math.sqrt(MAX_SIZE / buffer.length);
|
||||
buffer = await sharp(buffer)
|
||||
.resize({ width: Math.floor(width * scale), height: Math.floor(height * scale), fit: "inside" })
|
||||
.jpeg({ quality: 80, mozjpeg: true })
|
||||
.toBuffer();
|
||||
}
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("file", buffer, { filename: "image.jpg", contentType: "image/jpeg" });
|
||||
|
||||
const { data } = await axios.post(uploadUrl, formData, {
|
||||
headers: { Authorization: authorization },
|
||||
});
|
||||
|
||||
if (data.code !== 0 || !data.data?.download_url) {
|
||||
throw new Error(`图片上传失败: ${JSON.stringify(data)}`);
|
||||
}
|
||||
return data.data.download_url;
|
||||
};
|
||||
|
||||
// 提交任务
|
||||
const submitTask = async (url: string, body: Record<string, unknown>) => {
|
||||
const { data } = await axios.post(url, body, {
|
||||
headers: { "Content-Type": "application/json", Authorization: authorization },
|
||||
});
|
||||
if (data.status === "FAILED") throw new Error(`任务提交失败: ${data.errorMessage || "未知错误"}`);
|
||||
return { taskId: data.taskId, status: data.status, url: data.results?.[0]?.url };
|
||||
};
|
||||
|
||||
const isTextToVideo = images.length === 0 && hasTextType;
|
||||
const submitUrl = isTextToVideo ? (isPro ? text2videoProUrl : text2videoUrl) : isPro ? image2videoProUrl : image2videoUrl;
|
||||
|
||||
const requestBody: Record<string, unknown> = {
|
||||
prompt: input.prompt,
|
||||
duration: String(input.duration),
|
||||
aspectRatio: input.aspectRatio,
|
||||
...(isTextToVideo ? {} : { imageUrl: await uploadImage(images[0]) }),
|
||||
};
|
||||
|
||||
const { taskId } = await submitTask(submitUrl, requestBody);
|
||||
|
||||
return await pollTask(async () => {
|
||||
|
||||
const { data } = await axios.post(
|
||||
queryUrl,
|
||||
{
|
||||
taskId,
|
||||
},
|
||||
{
|
||||
headers: { Authorization: authorization },
|
||||
},
|
||||
);
|
||||
|
||||
if (data.status === "SUCCESS") {
|
||||
return data.results?.length ? { completed: true, url: data.results[0].url } : { completed: false, error: "任务成功但未返回视频链接" };
|
||||
}
|
||||
if (data.status === "FAILED") return { completed: false, error: `任务失败: ${data.errorMessage || "未知错误"}` };
|
||||
if (data.status === "QUEUED" || data.status === "RUNNING") return { completed: false };
|
||||
return { completed: false, error: `未知状态: ${data.status}` };
|
||||
});
|
||||
};
|
||||
132
src/utils/ai/video/owned/vidu.ts
Normal file
132
src/utils/ai/video/owned/vidu.ts
Normal file
@ -0,0 +1,132 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
import modelList from "../modelList";
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!input.prompt && (!input.imageBase64 || input.imageBase64.length === 0)) {
|
||||
throw new Error("至少需要提供prompt或图片");
|
||||
}
|
||||
|
||||
const defaultBaseUrl = ["https://api.vidu.cn/ent/v2/text2video", "https://api.vidu.cn/ent/v2/img2video", "https://api.vidu.cn/ent/v2/tasks"].join(
|
||||
"|",
|
||||
);
|
||||
|
||||
const [text2videoUrl, image2videoUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|");
|
||||
|
||||
const authorization = `Token ${config.apiKey}`;
|
||||
const hasImages = input.imageBase64 && input.imageBase64.length > 0;
|
||||
|
||||
// 根据是否有图片，查找匹配的模型配置
|
||||
const customOwned = modelList.find((m) => {
|
||||
if (m.manufacturer !== "vidu") return false;
|
||||
if (m.model !== config.model) return false;
|
||||
if (hasImages) {
|
||||
return m.type.some((t) => t !== "text");
|
||||
} else {
|
||||
return m.type.includes("text");
|
||||
}
|
||||
});
|
||||
|
||||
if (!customOwned) {
|
||||
throw new Error(`未找到匹配的模型配置: ${config.model}`);
|
||||
}
|
||||
|
||||
// 使用统一校验函数
|
||||
const { owned, images } = validateVideoConfig(input, config, customOwned);
|
||||
|
||||
// 判断生成类型
|
||||
const genType: "text" | "image" = images.length === 0 ? "text" : "image";
|
||||
|
||||
// 校验宽高比（仅文生视频需要）
|
||||
if (genType === "text" && owned.aspectRatio.length > 0 && !owned.aspectRatio.includes(input.aspectRatio as `${number}:${number}`)) {
|
||||
throw new Error(`模型 ${owned.model} 不支持宽高比 ${input.aspectRatio}，支持的宽高比：${owned.aspectRatio.join("、")}`);
|
||||
}
|
||||
|
||||
// 创建任务
|
||||
let taskId: string;
|
||||
|
||||
if (genType === "text") {
|
||||
// 文生视频
|
||||
const requestBody: Record<string, unknown> = {
|
||||
model: owned.model,
|
||||
prompt: input.prompt,
|
||||
duration: input.duration,
|
||||
resolution: input.resolution,
|
||||
aspect_ratio: input.aspectRatio,
|
||||
};
|
||||
if (owned.audio && input.audio !== undefined) {
|
||||
requestBody.audio = input.audio;
|
||||
}
|
||||
|
||||
const response = await axios.post(text2videoUrl, requestBody, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
},
|
||||
});
|
||||
taskId = response.data.task_id;
|
||||
} else {
|
||||
// 图生视频
|
||||
const requestBody: Record<string, unknown> = {
|
||||
model: owned.model,
|
||||
images: images,
|
||||
duration: input.duration,
|
||||
resolution: input.resolution,
|
||||
};
|
||||
if (input.prompt) {
|
||||
requestBody.prompt = input.prompt;
|
||||
}
|
||||
if (owned.audio && input.audio !== undefined) {
|
||||
requestBody.audio = input.audio;
|
||||
}
|
||||
|
||||
const response = await axios.post(image2videoUrl, requestBody, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
},
|
||||
});
|
||||
taskId = response.data.task_id;
|
||||
}
|
||||
|
||||
// 轮询任务状态
|
||||
return await pollTask(async () => {
|
||||
const response = await axios.get(queryUrl, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
},
|
||||
params: {
|
||||
task_ids: [taskId],
|
||||
},
|
||||
});
|
||||
|
||||
const tasks = response.data.tasks;
|
||||
if (!tasks || tasks.length === 0) {
|
||||
return { completed: false, error: "任务不存在" };
|
||||
}
|
||||
|
||||
const task = tasks[0];
|
||||
|
||||
switch (task.state) {
|
||||
case "success": {
|
||||
const creation = task.creations?.[0];
|
||||
return {
|
||||
completed: true,
|
||||
url: creation?.url,
|
||||
};
|
||||
}
|
||||
case "failed":
|
||||
return { completed: false, error: "任务生成失败" };
|
||||
case "created":
|
||||
case "queueing":
|
||||
case "processing":
|
||||
return { completed: false };
|
||||
default:
|
||||
return { completed: false, error: `未知状态: ${task.state}` };
|
||||
}
|
||||
});
|
||||
};
|
||||
75
src/utils/ai/video/owned/volcengine.ts
Normal file
75
src/utils/ai/video/owned/volcengine.ts
Normal file
@ -0,0 +1,75 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const { owned, images, hasStartEndType } = validateVideoConfig(input, config);
|
||||
|
||||
const authorization = "Bearer " + config.apiKey.replace(/^Bearer\s*/i, "").trim();
|
||||
const baseUrl = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks";
|
||||
|
||||
// 判断是否为首尾帧模式（需要两张图且类型支持首尾帧）
|
||||
const isStartEndMode = images.length === 2 && hasStartEndType;
|
||||
|
||||
// 构建图片内容
|
||||
const imageContent = images.map((base64, index) => {
|
||||
const item: Record<string, any> = {
|
||||
type: "image_url",
|
||||
image_url: { url: base64 },
|
||||
};
|
||||
if (isStartEndMode) {
|
||||
item.role = index === 0 ? "first_frame" : "last_frame";
|
||||
}
|
||||
return item;
|
||||
});
|
||||
|
||||
// 构建请求体
|
||||
const requestBody: Record<string, any> = {
|
||||
model: config.model,
|
||||
content: [{ type: "text", text: input.prompt }, ...imageContent],
|
||||
duration: input.duration,
|
||||
resolution: input.resolution,
|
||||
watermark: false,
|
||||
};
|
||||
|
||||
// 仅当模型支持音频时才添加 generate_audio 字段
|
||||
if (owned.audio) {
|
||||
requestBody.generate_audio = input.audio ?? false;
|
||||
}
|
||||
// 创建视频生成任务
|
||||
const createResponse = await axios.post(baseUrl, requestBody, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
},
|
||||
});
|
||||
|
||||
const taskId = createResponse.data.id;
|
||||
|
||||
if (!taskId) throw new Error("视频任务创建失败");
|
||||
|
||||
// 轮询任务状态
|
||||
return await pollTask(async () => {
|
||||
const { status, content } = (
|
||||
await axios.get(`${baseUrl}/${taskId}`, {
|
||||
headers: { Authorization: authorization },
|
||||
})
|
||||
).data;
|
||||
|
||||
switch (status) {
|
||||
case "succeeded":
|
||||
return { completed: true, url: content?.video_url };
|
||||
case "failed":
|
||||
case "cancelled":
|
||||
case "expired":
|
||||
return { completed: false, error: `任务${status}` };
|
||||
case "queued":
|
||||
case "running":
|
||||
return { completed: false };
|
||||
default:
|
||||
return { completed: false, error: `未知状态: ${status}` };
|
||||
}
|
||||
});
|
||||
};
|
||||
168
src/utils/ai/video/owned/wan.ts
Normal file
168
src/utils/ai/video/owned/wan.ts
Normal file
@ -0,0 +1,168 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import { pollTask, validateVideoConfig } from "@/utils/ai/utils";
|
||||
|
||||
// 根据分辨率档位和宽高比计算具体尺寸
|
||||
const getSizeFromConfig = (resolution: string, aspectRatio: string): string => {
|
||||
const sizeMap: Record<string, Record<string, string>> = {
|
||||
"480p": {
|
||||
"16:9": "832*480",
|
||||
"9:16": "480*832",
|
||||
"1:1": "624*624",
|
||||
},
|
||||
"720p": {
|
||||
"16:9": "1280*720",
|
||||
"9:16": "720*1280",
|
||||
"1:1": "960*960",
|
||||
"4:3": "1088*832",
|
||||
"3:4": "832*1088",
|
||||
},
|
||||
"1080p": {
|
||||
"16:9": "1920*1080",
|
||||
"9:16": "1080*1920",
|
||||
"1:1": "1440*1440",
|
||||
"4:3": "1632*1248",
|
||||
"3:4": "1248*1632",
|
||||
},
|
||||
};
|
||||
|
||||
const resolutionKey = resolution.toLowerCase();
|
||||
const size = sizeMap[resolutionKey]?.[aspectRatio];
|
||||
|
||||
if (!size) {
|
||||
throw new Error(`不支持的分辨率(${resolution})和宽高比(${aspectRatio})组合`);
|
||||
}
|
||||
|
||||
return size;
|
||||
};
|
||||
|
||||
export default async (input: VideoConfig, config: AIConfig) => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const { owned, images, hasStartEndType, hasTextType } = validateVideoConfig(input, config);
|
||||
|
||||
const defaultBaseUrl = [
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis",
|
||||
"https://dashscope.aliyuncs.com/api/v1/services/aigc/image2video/video-synthesis",
|
||||
"https://dashscope.aliyuncs.com/api/v1/tasks/{taskId}",
|
||||
].join("|");
|
||||
|
||||
const [i2vUrl, kf2vUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|");
|
||||
|
||||
const types = owned.type;
|
||||
const authorization = `Bearer ${config.apiKey}`;
|
||||
|
||||
// 确定端点和构建请求体
|
||||
let submitUrl: string;
|
||||
let body: Record<string, any>;
|
||||
|
||||
if (hasTextType && images.length === 0) {
|
||||
// 文本生视频
|
||||
submitUrl = i2vUrl;
|
||||
body = {
|
||||
model: config.model,
|
||||
input: {
|
||||
prompt: input.prompt,
|
||||
},
|
||||
parameters: {
|
||||
size: getSizeFromConfig(input.resolution, input.aspectRatio),
|
||||
duration: input.duration,
|
||||
},
|
||||
};
|
||||
} else if (types.includes("singleImage")) {
|
||||
// 图生视频
|
||||
submitUrl = i2vUrl;
|
||||
body = {
|
||||
model: config.model,
|
||||
input: {
|
||||
prompt: input.prompt,
|
||||
img_url: images[0],
|
||||
},
|
||||
parameters: {
|
||||
resolution: input.resolution.toUpperCase(),
|
||||
duration: input.duration,
|
||||
},
|
||||
};
|
||||
// audio参数仅部分模型支持
|
||||
if (owned.audio && input.audio !== undefined) {
|
||||
body.parameters.audio = input.audio;
|
||||
}
|
||||
} else if (hasStartEndType) {
|
||||
// 首尾帧
|
||||
submitUrl = kf2vUrl;
|
||||
const inputObj: Record<string, any> = {
|
||||
prompt: input.prompt,
|
||||
first_frame_url: images[0],
|
||||
};
|
||||
// 尾帧处理
|
||||
if (types.includes("startEndRequired")) {
|
||||
inputObj.last_frame_url = images[1];
|
||||
} else if ((types.includes("endFrameOptional") || types.includes("startFrameOptional")) && images.length >= 2) {
|
||||
inputObj.last_frame_url = images[1];
|
||||
}
|
||||
body = {
|
||||
model: config.model,
|
||||
input: inputObj,
|
||||
parameters: {
|
||||
resolution: input.resolution.toUpperCase(),
|
||||
duration: input.duration,
|
||||
},
|
||||
};
|
||||
} else {
|
||||
throw new Error(`不支持的视频生成类型: ${types.join(", ")}`);
|
||||
}
|
||||
|
||||
// 提交任务
|
||||
const submitResponse = await axios.post(submitUrl, body, {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
"X-DashScope-Async": "enable",
|
||||
},
|
||||
});
|
||||
|
||||
const submitData = submitResponse.data;
|
||||
if (submitData.code) {
|
||||
throw new Error(`任务提交失败: [${submitData.code}] ${submitData.message}`);
|
||||
}
|
||||
|
||||
const taskId = submitData.output?.task_id;
|
||||
if (!taskId) {
|
||||
throw new Error("任务提交失败: 未返回task_id");
|
||||
}
|
||||
|
||||
// 轮询任务状态
|
||||
return await pollTask(async () => {
|
||||
const response = await axios.get(queryUrl.replace("{taskId}", taskId), {
|
||||
headers: { Authorization: authorization },
|
||||
});
|
||||
|
||||
const data = response.data;
|
||||
|
||||
// 顶层错误
|
||||
if (data.code) {
|
||||
return { completed: false, error: `[${data.code}] ${data.message}` };
|
||||
}
|
||||
|
||||
const taskStatus = data.output?.task_status;
|
||||
|
||||
switch (taskStatus) {
|
||||
case "SUCCEEDED":
|
||||
return { completed: true, url: data.output?.video_url };
|
||||
case "FAILED":
|
||||
return {
|
||||
completed: false,
|
||||
error: `任务失败: [${data.output?.code || "UNKNOWN"}] ${data.output?.message || "未知错误"}`,
|
||||
};
|
||||
case "CANCELED":
|
||||
return { completed: false, error: "任务已取消" };
|
||||
case "UNKNOWN":
|
||||
return { completed: false, error: "任务不存在或状态未知" };
|
||||
case "PENDING":
|
||||
case "RUNNING":
|
||||
return { completed: false };
|
||||
default:
|
||||
return { completed: false, error: `未知状态: ${taskStatus}` };
|
||||
}
|
||||
});
|
||||
};
|
||||
16
src/utils/ai/video/type.ts
Normal file
16
src/utils/ai/video/type.ts
Normal file
@ -0,0 +1,16 @@
|
||||
interface VideoConfig {
|
||||
duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12;
|
||||
resolution: "480p" | "720p" | "1080p" | "2K" | "4K";
|
||||
aspectRatio: "16:9" | "9:16";
|
||||
prompt: string;
|
||||
savePath: string;
|
||||
imageBase64?: string[];
|
||||
audio?: boolean;
|
||||
}
|
||||
|
||||
interface AIConfig {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseURL?: string;
|
||||
manufacturer?: string;
|
||||
}
|
||||
@ -40,10 +40,11 @@ const db = knex({
|
||||
useNullAsDefault: true,
|
||||
});
|
||||
|
||||
initDB(db);
|
||||
fixDB(db);
|
||||
|
||||
if (process.env.NODE_ENV == "dev") initKnexType(db);
|
||||
(async () => {
|
||||
await initDB(db);
|
||||
await fixDB(db);
|
||||
if (process.env.NODE_ENV == "dev") initKnexType(db);
|
||||
})();
|
||||
|
||||
const dbClient = Object.assign(<TName extends TableName>(table: TName) => db<RowType<TName>, RowType<TName>[]>(table), db);
|
||||
dbClient.schema = db.schema;
|
||||
|
||||
@ -79,13 +79,18 @@ async function convertDirectiveAndImages(images: Record<string, string>, directi
|
||||
*/
|
||||
export default async (images: Record<string, string>, directive: string, projectId: number) => {
|
||||
const { prompt, images: base64Images } = await convertDirectiveAndImages(images, directive);
|
||||
const contentStr = await u.ai.generateImage({
|
||||
systemPrompt: "根据用户提供的具体修改指令，对上传的图片进行智能编辑。",
|
||||
prompt: prompt,
|
||||
imageBase64: base64Images,
|
||||
aspectRatio: "16:9",
|
||||
size: "1K",
|
||||
});
|
||||
const apiConfig = await u.getPromptAi("editImage");
|
||||
|
||||
const contentStr = await u.ai.image(
|
||||
{
|
||||
systemPrompt: "根据用户提供的具体修改指令，对上传的图片进行智能编辑。",
|
||||
prompt: prompt,
|
||||
imageBase64: base64Images,
|
||||
aspectRatio: "16:9",
|
||||
size: "1K",
|
||||
},
|
||||
apiConfig,
|
||||
);
|
||||
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
const buffer = Buffer.from(match && match.length >= 1 ? match[1]! : contentStr, "base64");
|
||||
const filePath = `/${projectId}/storyboard/${uuid()}.jpg`;
|
||||
|
||||
68
src/utils/error.ts
Normal file
68
src/utils/error.ts
Normal file
@ -0,0 +1,68 @@
|
||||
// utils/error.ts
|
||||
import { serializeError } from "serialize-error";
|
||||
import { isAxiosError } from "axios";
|
||||
|
||||
export interface NormalizedError {
|
||||
name: string;
|
||||
message: string;
|
||||
code?: string;
|
||||
status?: number;
|
||||
stack?: string;
|
||||
cause?: NormalizedError;
|
||||
responseData?: unknown;
|
||||
meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function normalizeError(error: unknown): NormalizedError {
|
||||
// Axios 特殊处理
|
||||
if (isAxiosError(error)) {
|
||||
return {
|
||||
name: "AxiosError",
|
||||
message: error.response?.data?.error?.message || error.response?.data?.message || error.message,
|
||||
code: error.code,
|
||||
status: error.response?.status,
|
||||
stack: error.stack,
|
||||
responseData: error.response?.data,
|
||||
meta: {
|
||||
url: error.config?.url,
|
||||
method: error.config?.method,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// 普通 Error，用 serialize-error 处理
|
||||
if (error instanceof Error) {
|
||||
const serialized = serializeError(error);
|
||||
return {
|
||||
name: serialized.name || "Error",
|
||||
message: serialized.message || "未知错误",
|
||||
code: (serialized as any).code,
|
||||
stack: serialized.stack,
|
||||
cause: error.cause ? normalizeError(error.cause) : undefined,
|
||||
meta: extractMeta(serialized),
|
||||
};
|
||||
}
|
||||
|
||||
// 非 Error
|
||||
return {
|
||||
name: "UnknownError",
|
||||
message: String(error),
|
||||
meta: { raw: serializeError(error) },
|
||||
};
|
||||
}
|
||||
|
||||
// 提取自定义属性
|
||||
function extractMeta(obj: Record<string, unknown>): Record<string, unknown> | undefined {
|
||||
const standardKeys = ["name", "message", "stack", "cause"];
|
||||
const meta: Record<string, unknown> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
if (!standardKeys.includes(key) && value !== undefined) {
|
||||
meta[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(meta).length > 0 ? meta : undefined;
|
||||
}
|
||||
|
||||
export default normalizeError;
|
||||
@ -127,16 +127,18 @@ ${episodePrompt}
|
||||
${novelData}`;
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "script").first();
|
||||
|
||||
const promptConfig = await u.getPromptAi("generateScript");
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出AI配置异常";
|
||||
|
||||
const model = await u.ai.text();
|
||||
const result = await model.invoke({
|
||||
messages: [
|
||||
{ role: "system", content: mainPrompts },
|
||||
{ role: "user", content: userPrompt },
|
||||
],
|
||||
});
|
||||
const result = await u.ai.text.invoke(
|
||||
{
|
||||
messages: [
|
||||
{ role: "system", content: mainPrompts },
|
||||
{ role: "user", content: userPrompt },
|
||||
],
|
||||
},
|
||||
promptConfig,
|
||||
);
|
||||
|
||||
return result.text ?? "";
|
||||
}
|
||||
|
||||
@ -10,11 +10,12 @@ interface BaseConfig {
|
||||
|
||||
interface TextResData extends BaseConfig {
|
||||
baseURL: string;
|
||||
manufacturer: "deepseek" | "openAi" | "doubao";
|
||||
manufacturer: "deepseek" | "openAi" | "doubao" | "other";
|
||||
}
|
||||
|
||||
// 图像模型配置接口
|
||||
interface ImageResData extends BaseConfig {
|
||||
manufacturer: "openAi" | "gemini" | "volcengine" | "runninghub" | "apimart";
|
||||
manufacturer: "gemini" | "volcengine" | "kling" | "vidu" | "runninghub" | "apimart" | "other";
|
||||
}
|
||||
|
||||
interface VideoResData extends BaseConfig {
|
||||
@ -34,10 +35,18 @@ const errorMessages: Record<AIType, string> = {
|
||||
video: "视频模型配置不存在",
|
||||
};
|
||||
|
||||
const needBaseURL: AIType[] = ["text", "video"];
|
||||
const needBaseURL: AIType[] = ["text", "video", "image"];
|
||||
|
||||
export default async function getConfig<T extends AIType>(aiType: T): Promise<ResDataMap[T]> {
|
||||
const config = await u.db("t_config").where("type", aiType).first();
|
||||
export default async function getConfig<T extends AIType>(aiType: T, manufacturer?: string): Promise<ResDataMap[T]> {
|
||||
const config = await u
|
||||
.db("t_config")
|
||||
.where("type", aiType)
|
||||
.modify((qb) => {
|
||||
if (manufacturer) {
|
||||
qb.where("manufacturer", manufacturer);
|
||||
}
|
||||
})
|
||||
.first();
|
||||
|
||||
if (!config) throw new Error(errorMessages[aiType]);
|
||||
|
||||
|
||||
19
src/utils/getPromptAi.ts
Normal file
19
src/utils/getPromptAi.ts
Normal file
@ -0,0 +1,19 @@
|
||||
import { db } from "./db";
|
||||
interface AiConfig {
|
||||
model?: string;
|
||||
apiKey: string;
|
||||
baseURL?: string;
|
||||
manufacturer: string;
|
||||
}
|
||||
|
||||
export default async function getPromptAi(key: string): Promise<AiConfig | {}> {
|
||||
const aiConfigData = await db("t_aiModelMap")
|
||||
.leftJoin("t_config", "t_config.id", "t_aiModelMap.configId")
|
||||
.where("t_aiModelMap.key", key)
|
||||
.select("t_config.model", "t_config.apiKey", "t_config.baseUrl as baseURL", "t_config.manufacturer")
|
||||
.first();
|
||||
|
||||
if (aiConfigData) {
|
||||
return aiConfigData as AiConfig;
|
||||
} else return {};
|
||||
}
|
||||
122
src/utils/imageTools.ts
Normal file
122
src/utils/imageTools.ts
Normal file
@ -0,0 +1,122 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
/**
|
||||
* 解析大小字符串为字节数
|
||||
*/
|
||||
function parseSize(size: string): number {
|
||||
const match = size.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*(kb|mb|gb|b)?$/);
|
||||
if (!match) {
|
||||
throw new Error(`无效的大小格式: ${size}`);
|
||||
}
|
||||
const value = parseFloat(match[1]);
|
||||
const unit = match[2] || "b";
|
||||
const multipliers: Record<string, number> = {
|
||||
b: 1,
|
||||
kb: 1024,
|
||||
mb: 1024 * 1024,
|
||||
gb: 1024 * 1024 * 1024,
|
||||
};
|
||||
return Math.floor(value * multipliers[unit]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将base64字符串转换为Buffer
|
||||
*/
|
||||
function base64ToBuffer(base64: string): Buffer {
|
||||
const base64Data = base64.replace(/^data:image\/\w+;base64,/, "");
|
||||
return Buffer.from(base64Data, "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩Buffer到指定大小以内
|
||||
*/
|
||||
async function compressToSize(imageBuffer: Buffer, maxBytes: number, originalWidth: number, originalHeight: number): Promise<Buffer> {
|
||||
let quality = 90;
|
||||
let scale = 1;
|
||||
|
||||
while (true) {
|
||||
const targetWidth = Math.round(originalWidth * scale);
|
||||
const targetHeight = Math.round(originalHeight * scale);
|
||||
|
||||
const resultBuffer = await sharp(imageBuffer).resize(targetWidth, targetHeight, { fit: "fill" }).jpeg({ quality }).toBuffer();
|
||||
|
||||
if (resultBuffer.length <= maxBytes) {
|
||||
return resultBuffer;
|
||||
}
|
||||
|
||||
if (quality > 10) {
|
||||
quality -= 10;
|
||||
} else {
|
||||
quality = 90;
|
||||
scale *= 0.8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩单张图片到指定大小以内
|
||||
* @param imageBase64 - base64编码的图片
|
||||
* @param maxSize - 最大输出大小，支持格式如 "10mb", "5MB", "1024kb" 等
|
||||
* @returns 压缩后的图片base64字符串
|
||||
*/
|
||||
export async function compressImage(imageBase64: string, maxSize = "10mb"): Promise<string> {
|
||||
const maxBytes = parseSize(maxSize);
|
||||
const imageBuffer = base64ToBuffer(imageBase64);
|
||||
const metadata = await sharp(imageBuffer).metadata();
|
||||
const resultBuffer = await compressToSize(imageBuffer, maxBytes, metadata.width || 1, metadata.height || 1);
|
||||
return resultBuffer.toString("base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* 将多张图片横向拼接为一张，并确保输出大小不超过指定限制
|
||||
* @param imageBase64List - base64编码的图片数组
|
||||
* @param maxSize - 最大输出大小，支持格式如 "10mb", "5MB", "1024kb" 等
|
||||
* @returns 拼接后的图片base64字符串
|
||||
*/
|
||||
export async function mergeImages(imageBase64List: string[], maxSize = "10mb"): Promise<string> {
|
||||
if (imageBase64List.length === 0) {
|
||||
throw new Error("图片列表不能为空");
|
||||
}
|
||||
|
||||
const maxBytes = parseSize(maxSize);
|
||||
const imageBuffers = imageBase64List.map(base64ToBuffer);
|
||||
const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata()));
|
||||
const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0));
|
||||
|
||||
// 计算各图片调整后的宽度
|
||||
const imageWidths = imageMetadatas.map((metadata) => {
|
||||
const aspectRatio = (metadata.width || 1) / (metadata.height || 1);
|
||||
return Math.round(maxHeight * aspectRatio);
|
||||
});
|
||||
const totalWidth = imageWidths.reduce((sum, w) => sum + w, 0);
|
||||
|
||||
// 拼接图片
|
||||
const resizedImages = await Promise.all(
|
||||
imageBuffers.map(async (buffer, index) => {
|
||||
return sharp(buffer).resize(imageWidths[index], maxHeight, { fit: "cover" }).toBuffer();
|
||||
}),
|
||||
);
|
||||
|
||||
let currentX = 0;
|
||||
const compositeInputs = resizedImages.map((buffer, index) => {
|
||||
const input = { input: buffer, left: currentX, top: 0 };
|
||||
currentX += imageWidths[index];
|
||||
return input;
|
||||
});
|
||||
|
||||
const mergedBuffer = await sharp({
|
||||
create: {
|
||||
width: totalWidth,
|
||||
height: maxHeight,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.composite(compositeInputs)
|
||||
.jpeg({ quality: 90 })
|
||||
.toBuffer();
|
||||
|
||||
// 复用压缩逻辑
|
||||
const resultBuffer = await compressToSize(mergedBuffer, maxBytes, totalWidth, maxHeight);
|
||||
return resultBuffer.toString("base64");
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user