//如需遥测AI请使用在toonflow安装目录运行npx @ai-sdk/devtools (要求在其他设置中打开遥测功能,且toonflow有权限在安装目录创建.devtools文件夹) // ==================== 类型定义 ==================== // 文本模型 interface TextModel { name: string; // 显示名称 modelName: string; type: "text"; think: boolean; // 前端显示用 } // 图像模型 interface ImageModel { name: string; // 显示名称 modelName: string; type: "image"; mode: ("text" | "singleImage" | "multiReference")[]; associationSkills?: string; // 关联技能,多个技能用逗号分隔 } // 视频模型 interface VideoModel { name: string; // 显示名称 modelName: string; //全局唯一 type: "video"; mode: ( | "singleImage" // 单图 | "startEndRequired" // 首尾帧(两张都得有) | "endFrameOptional" // 首尾帧(尾帧可选) | "startFrameOptional" // 首尾帧(首帧可选) | "text" // 文本生视频 | ("videoReference" | "imageReference" | "audioReference" | "textReference")[] // 混合参考 )[]; associationSkills?: string; // 关联技能,多个技能用逗号分隔 audio: "optional" | false | true; // 音频配置 durationResolutionMap: { duration: number[]; resolution: string[] }[]; } interface TTSModel { name: string; // 显示名称 modelName: string; type: "tts"; voices: { title: string; //显示名称 voice: string; //说话人 }[]; } // 供应商配置 interface VendorConfig { id: string; //供应商唯一标识,必须全局唯一 author: string; description?: string; //md5格式 name: string; icon?: string; //仅支持base64格式 inputs: { key: string; label: string; type: "text" | "password" | "url"; required: boolean; placeholder?: string; }[]; inputValues: Record; models: (TextModel | ImageModel | VideoModel)[]; } // ==================== 全局工具函数 ==================== //Axios实例 //压缩图片大小(1MB = 1 * 1024 * 1024) declare const zipImage: (completeBase64: string, size: number) => Promise; //压缩图片分辨率 declare const zipImageResolution: (completeBase64: string, width: number, height: number) => Promise; //多图拼接乘单图 maxSize 最大输出大小,默认为 10mb declare const mergeImages: (completeBase64: string[], maxSize?: string) => Promise; //Url转Base64 declare const urlToBase64: (url: string) => Promise; //轮询函数 declare const pollTask: ( fn: () => Promise<{ completed: boolean; data?: string; error?: string }>, interval?: number, timeout?: number, ) => Promise<{ completed: boolean; data?: string; error?: string }>; declare const axios: any; declare const createOpenAI: any; declare const createDeepSeek: any; declare const createZhipu: any; declare const createQwen: any; declare const createAnthropic: any; declare const createOpenAICompatible: any; declare const createXai: any; declare const createMinimax: any; declare const createGoogleGenerativeAI: any; declare const logger: (logstring: string) => void; declare const jsonwebtoken: any; // ==================== 供应商数据 ==================== const vendor: VendorConfig = { id: "vidu", author: "搬砖的Coder", description: "Vidu 官方视频生成平台。 [前往平台](https://platform.vidu.cn/login/)", name: "Vidu 开放平台", inputs: [ { key: "apiKey", label: "API密钥", type: "password", required: true, placeholder: "请到Vidu官方申请" }, { key: "baseUrl", label: "接口路径", type: "url", required: true, placeholder: "https://api.vidu.cn/ent/v2" }, ], inputValues: { apiKey: "", baseUrl: "https://api.vidu.cn/ent/v2", }, models: [ { name: "ViduQ3 turbo", type: "video", modelName: "ViduQ3-turbo", durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }], mode: ["singleImage", "startEndRequired", "text"], audio: true, }, { name: "ViduQ3 pro", type: "video", modelName: "ViduQ3-pro", durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }], mode: ["singleImage", "startEndRequired", "text"], audio: true, }, { name: "ViduQ2 pro fast", type: "video", modelName: "ViduQ2-pro-fast", durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["720p", "1080p"] }], mode: ["singleImage", "startEndRequired"], audio: true, }, { name: "viduQ2 turbo", type: "video", modelName: "ViduQ2-turbo", durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], mode: ["singleImage", "startEndRequired"], audio: true, }, { name: "ViduQ2 pro", type: "video", modelName: "ViduQ2-pro", durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], mode: ["singleImage", "startEndRequired"], //参考生视频无有效设置值 audio: true, }, { name: "ViduQ2", type: "video", modelName: "ViduQ2", durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], mode: ["text"], audio: true, }, { name: "ViduQ1", type: "video", modelName: "ViduQ1", durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], mode: ["singleImage", "startEndRequired", "text"], audio: true, }, { name: "ViduQ1 classic", type: "video", modelName: "viduQ1-classic", durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], mode: ["singleImage", "startEndRequired"], audio: true, }, { name: "Vidu2.0", type: "video", modelName: "vidu2.0", durationResolutionMap: [{ duration: [4, 8], resolution: ["360p", "720p", "1080p"] }], mode: ["singleImage", "startEndRequired"], audio: true, }, { name: "viduq1 for image", type: "image", modelName: "viduq1", mode: ["text"], }, { name: "viduq2 for image", type: "image", modelName: "viduq2", mode: ["text", "singleImage", "multiReference"], }, ], }; exports.vendor = vendor; // ==================== 适配器函数 ==================== // 文本请求函数 const textRequest: (textModel: TextModel) => { url: string; model: string } = (textModel) => { throw new Error("当前供应商仅支持视频大模型,谢谢!"); }; exports.textRequest = textRequest; //图片请求函数 interface ImageConfig { prompt: string; //图片提示词 imageBase64: string[]; //输入的图片提示词 size: "1K" | "2K" | "4K"; // 图片尺寸 aspectRatio: `${number}:${number}`; // 长宽比 } const imageRequest = async (imageConfig: ImageConfig, imageModel: ImageModel) => { if (!vendor.inputValues.apiKey) throw new Error("缺少API Key"); const apiKey = vendor.inputValues.apiKey.replace("Token ", ""); const size = imageConfig.size === "1K" ? "2K" : imageConfig.size; const sizeMap: Record> = { "16:9": { "1k": "1920x1080", "2K": "2848x1600", "4K": "4096x2304", }, "9:16": { "1k": "1920x1080", "2K": "1600x2848", "4K": "2304x4096", }, }; const body: Record = { model: imageModel.modelName, prompt: imageConfig.prompt, aspect_ratio: sizeMap[imageConfig.aspectRatio][size], seed: 0, resolution: size, ...(imageConfig.imageBase64 && { image: imageConfig.imageBase64 }), }; const createImageUrl = vendor.inputValues.baseUrl + "/reference2image"; const response = await fetch(createImageUrl, { method: "POST", headers: { Authorization: `Token ${apiKey}`, "Content-Type": "application/json" }, body: JSON.stringify(body), }); if (!response.ok) { const errorText = await response.text(); // 获取错误信息 console.error("请求失败,状态码:", response.status, ", 错误信息:", errorText); throw new Error(`请求失败,状态码: ${response.status}, 错误信息: ${errorText}`); } const data = await response.json(); const res = await checkTaskResult(data.task_id); if (!res.data) { throw new Error("图片未能生成"); } const list = JSON.parse(JSON.stringify(res.data)); return list[0].url; }; exports.imageRequest = imageRequest; interface VideoConfig { duration: number; resolution: string; aspectRatio: "16:9" | "9:16"; prompt: string; imageBase64?: string[]; audio?: boolean; mode: | "singleImage" // 单图 | "multiImage" // 多图模式 | "gridImage" // 网格单图(传入一张图片,但该图片是网格图) | "startEndRequired" // 首尾帧(两张都得有) | "endFrameOptional" // 首尾帧(尾帧可选) | "startFrameOptional" // 首尾帧(首帧可选) | "text" // 文本生视频 | ("video" | "image" | "audio" | "text")[]; // 混合参考 } // 构建 各个平台的metadata参数 const buildViduMetadata = (videoConfig: VideoConfig) => ({ aspect_ratio: videoConfig.aspectRatio, audio: videoConfig.audio ?? false, off_peak: false, }); type MetadataBuilder = (config: VideoConfig) => Record; const METADATA_BUILDERS: Array<[string, MetadataBuilder]> = [["vidu", buildViduMetadata]]; const buildModelMetadata = (modelName: string, videoConfig: VideoConfig) => { const lowerName = modelName.toLowerCase(); const match = METADATA_BUILDERS.find(([key]) => lowerName.includes(key)); return match ? match[1](videoConfig) : {}; }; // 检查生成物结果 const checkTaskResult = async (taskId: string) => { const queryUrl = vendor.inputValues.baseUrl + "/tasks/{id}/creations"; const apiKey = vendor.inputValues.apiKey; const res = await pollTask(async () => { const queryResponse = await fetch(queryUrl.replace("{id}", taskId), { method: "GET", headers: { Authorization: `Token ${apiKey}`, "Content-Type": "application/json" }, }); if (!queryResponse.ok) { const errorText = await queryResponse.text(); // 获取错误信息 console.error("请求失败,状态码:", queryResponse.status, ", 错误信息:", errorText); throw new Error(`请求失败,状态码: ${queryResponse.status}, 错误信息: ${errorText}`); } const queryData = await queryResponse.json(); const status = queryData?.state ?? queryData?.data?.state; const fail_reason = queryData?.data?.err_code ?? queryData?.data; switch (status) { case "completed": case "SUCCESS": case "success": return { completed: true, data: queryData.creations }; case "FAILURE": case "failed": return { completed: false, error: fail_reason || "生成失败" }; default: return { completed: false }; } }); if (res.error) throw new Error(res.error); return res; }; const videoRequest = async (videoConfig: VideoConfig, videoModel: VideoModel) => { if (!vendor.inputValues.apiKey) throw new Error("缺少API Key"); const apiKey = vendor.inputValues.apiKey.replace("Token ", ""); // 构建每个模型对应的附加参数 const metadata = buildModelMetadata(videoModel.modelName, videoConfig); //公共请求参数 const publicBody = { model: videoModel.modelName, ...(videoConfig.imageBase64 && videoConfig.imageBase64.length ? { images: videoConfig.imageBase64 } : {}), prompt: videoConfig.prompt, size: videoConfig.resolution, duration: videoConfig.duration, metadata: metadata, }; const requestUrl = vendor.inputValues.baseUrl + "/start-end2video"; const response = await fetch(requestUrl, { method: "POST", headers: { Authorization: `Token ${apiKey}`, "Content-Type": "application/json" }, body: JSON.stringify(publicBody), }); if (!response.ok) { const errorText = await response.text(); // 获取错误信息 console.error("请求失败,状态码:", response.status, ", 错误信息:", errorText); throw new Error(`请求失败,状态码: ${response.status}, 错误信息: ${errorText}`); } const data = await response.json(); const taskId = data.id; const result = await checkTaskResult(taskId); return result.data; }; exports.videoRequest = videoRequest; interface TTSConfig { text: string; voice: string; speechRate: number; pitchRate: number; volume: number; } const ttsRequest = async (ttsConfig: TTSConfig, ttsModel: TTSModel) => { throw new Error("Vidu 暂不支持语音合成(TTS)"); };