diff --git a/.codebuddy/plans/replace-video-api-to-seedance_d39cb864.md b/.codebuddy/plans/replace-video-api-to-seedance_d39cb864.md new file mode 100644 index 0000000..4264991 --- /dev/null +++ b/.codebuddy/plans/replace-video-api-to-seedance_d39cb864.md @@ -0,0 +1,102 @@ +--- +name: replace-video-api-to-seedance +overview: 将 VideoExplanation.vue 中视频生成部分从阿里云万相(wan2.2-kf2v-flash)切换到火山引擎 Seedance 1.5 Pro (doubao-seedance-1-5-pro-251215)，修改 API 调用、请求体结构和轮询逻辑。 +todos: + - id: replace-video-api-config + content: 修改 VideoExplanation.vue 的 API 配置常量：删除 VIDEO_API_KEY/VIDEO_API_URL/VIDEO_TASK_URL/VIDEO_MODEL，替换为 Seedance 的 VIDEO_API_URL 和 VIDEO_TASK_URL + status: completed + - id: rewrite-generate-single-clip + content: 重写 generateSingleClip 函数，适配 Seedance 的 content 数组请求格式和响应字段结构 + status: completed + dependencies: + - replace-video-api-config +--- + +## Product Overview + +将 VideoExplanation.vue 中视频生成部分从阿里云万相模型（wan2.2-kf2v-flash）切换为豆包 Seedance 1.5 Pro 模型（doubao-seedance-1-5-pro-251215），通过 Ark 平台 API 实现首尾帧图片生视频功能。 + +## Core Features + +- 替换视频生成 API 端点、请求结构和鉴权方式（从 DashScope 切换到 Ark） +- 适配 Seedance 的 content 数组格式（text + first_frame image_url + last_frame image_url） +- 适配 Seedance 的异步任务查询接口和响应字段（id/status/content.video_url） +- 复用已有的 /ark-api 代理和 DOUBAO_KEY 密钥，无需新增代理或配置 + +## Tech Stack + +- 前端框架: Vue 3 (Composition API) +- HTTP 请求: axios（已在项目中使用） +- API 代理: Vite proxy（/ark-api 已配置指向 ark.cn-beijing.volces.com） + +## Implementation Approach + +将视频生成 API 从 DashScope 万相模型切换到 Seedance 1.5 Pro，核心变化如下： + +### API 对比 + +| 维度 | 万相 (当前) | Seedance 1.5 Pro (目标) | +| --- | --- | --- | +| 创建任务 | POST `/dashscope-api/api/v1/services/aigc/image2video/video-synthesis` | POST `/ark-api/api/v3/contents/generations/tasks` | +| 请求体 | `{ model, input: { first_frame_url, last_frame_url, prompt }, parameters: { resolution, prompt_extend, watermark } }` | `{ model, content: [{ type:"text", text }, { type:"image_url", image_url:{url}, role:"first_frame" }, { type:"image_url", image_url:{url}, role:"last_frame" }], ratio, duration, watermark }` | +| 异步标识 | Header: `X-DashScope-Async: enable` | 无需异步头，创建即返回任务 ID | +| 查询任务 | GET `/dashscope-api/api/v1/tasks/{taskId}` | GET `/ark-api/api/v3/contents/generations/tasks/{id}` | +| 响应字段 | `output.task_id` / `output.task_status` / `output.video_url` | `id` / `status` / `content.video_url` | +| 状态值 | `SUCCEEDED` / `FAILED` | `succeeded` / `failed` | +| 鉴权 | 独立 VIDEO_API_KEY | DOUBAO_KEY（Ark 平台统一鉴权） | +| 时长范围 | 无限制 | 4~12 秒（当前 CLIP_DURATION=5 符合） | + + +### 关键技术决策 + +1. **复用 DOUBAO_KEY**: Seedance 同属 Ark 平台，直接复用 config/index.js 中已有的 DOUBAO_KEY，无需新增配置 +2. **复用 /ark-api 代理**: vite.config.js 已有 `/ark-api` 代理指向 `ark.cn-beijing.volces.com`，无需修改代理配置 +3. **保留首尾帧逻辑**: 当前分镜图片生成流程（首帧+尾帧）与 Seedance 的 first_frame/last_frame 能力完全匹配，图片生成部分不需改动 + +## Implementation Notes + +- 仅修改 VideoExplanation.vue 中的视频生成相关代码，不改动脚本生成和图片生成逻辑 +- 修改范围集中在两个区域：API 配置常量（第42-50行）和 generateSingleClip 函数（第251-319行） +- Seedance 任务轮询间隔保持 15 秒不变，最大轮询次数 24 次足够（最长等待 6 分钟） +- 删除 VIDEO_API_KEY 常量，统一使用 DOUBAO_KEY + +## Directory Structure + +``` +src/views/VideoExplanation.vue # [MODIFY] 替换视频生成 API：修改 API 配置常量、重写 generateSingleClip 函数 +``` + +## Key Code Structures + +### generateSingleClip 函数签名（不变） + +```typescript +const generateSingleClip = async (shot, firstFrameUrl, lastFrameUrl): Promise +``` + +### Seedance 创建任务请求体结构 + +```typescript +{ + model: "doubao-seedance-1-5-pro-251215", + content: [ + { type: "text", text: string }, + { type: "image_url", image_url: { url: string }, role: "first_frame" }, + { type: "image_url", image_url: { url: string }, role: "last_frame" } + ], + ratio: "16:9", + duration: 5, + watermark: false +} +``` + +### Seedance 查询响应结构 + +```typescript +{ + id: string, // 任务 ID + status: "succeeded" | "failed" | "queued" | "running", + content: { video_url: string }, + error: { code: string, message: string } +} +``` \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index d91c767..2d46492 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,8 @@ "name": "ai-demo", "version": "0.0.0", "dependencies": { + "@ffmpeg/ffmpeg": "^0.12.15", + "@ffmpeg/util": "^0.12.2", "axios": "^1.13.6", "vue": "^3.5.30", "vue-router": "^5.0.3" @@ -113,6 +115,36 @@ "tslib": "^2.4.0" } }, + "node_modules/@ffmpeg/ffmpeg": { + "version": "0.12.15", + "resolved": "https://registry.npmmirror.com/@ffmpeg/ffmpeg/-/ffmpeg-0.12.15.tgz", + "integrity": "sha512-1C8Obr4GsN3xw+/1Ww6PFM84wSQAGsdoTuTWPOj2OizsRDLT4CXTaVjPhkw6ARyDus1B9X/L2LiXHqYYsGnRFw==", + "license": "MIT", + "dependencies": { + "@ffmpeg/types": "^0.12.4" + }, + "engines": { + "node": ">=18.x" + } + }, + "node_modules/@ffmpeg/types": { + "version": "0.12.4", + "resolved": "https://registry.npmmirror.com/@ffmpeg/types/-/types-0.12.4.tgz", + "integrity": "sha512-k9vJQNBGTxE5AhYDtOYR5rO5fKsspbg51gbcwtbkw2lCdoIILzklulcjJfIDwrtn7XhDeF2M+THwJ2FGrLeV6A==", + "license": "MIT", + "engines": { + "node": ">=16.x" + } + }, + "node_modules/@ffmpeg/util": { + "version": "0.12.2", + "resolved": "https://registry.npmmirror.com/@ffmpeg/util/-/util-0.12.2.tgz", + "integrity": "sha512-ouyoW+4JB7WxjeZ2y6KpRvB+dLp7Cp4ro8z0HIVpZVCM7AwFlHa0c4R8Y/a4M3wMqATpYKhC7lSFHQ0T11MEDw==", + "license": "MIT", + "engines": { + "node": ">=18.x" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmmirror.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", diff --git a/package.json b/package.json index 21e41ca..0492bfa 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,8 @@ "preview": "vite preview" }, "dependencies": { + "@ffmpeg/ffmpeg": "^0.12.15", + "@ffmpeg/util": "^0.12.2", "axios": "^1.13.6", "vue": "^3.5.30", "vue-router": "^5.0.3" diff --git a/src/md/doubao_video.md b/src/md/doubao_video.md new file mode 100644 index 0000000..a92d90b --- /dev/null +++ b/src/md/doubao_video.md @@ -0,0 +1,3044 @@ +Seedance 模型具备出色的语义理解能力，可根据用户输入的文本、图片等内容，快速生成优质的视频片段。通过这篇教程，您可学习到如何调用 [Video Generation API](https://www.volcengine.com/docs/82379/1520758) 生成视频。 +:::warning +Seedance 2.0 模型目前仅支持 [控制台体验中心](https://console.volcengine.com/ark/region:ark+cn-beijing/experience/vision?modelId=doubao-seedance-2-0-260128&tab=GenVideo) 在免费额度内体验，暂不支持 API 调用，敬请期待。 +::: +:::tip +方舟平台的新用户？获取 API Key 及 开通模型等准备工作，请参见 [快速入门](/docs/82379/1399008)。 +::: + +# 效果预览 +访问[模型卡片](https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seedance-2-0)查看更多示例。 + + +|场景 |输入：提示词 |输入：图片、视频、音频 |输出 | +|---|---|---|---| +|首帧图生视频 |一辆地铁轰隆隆驶过，书页和女孩的头发飞扬，镜头开始环绕着女孩360度旋转，周围的背景从地铁站渐渐转变为一个中世纪的教堂，西式幻想风格的音乐渐入。夹在女孩书中的几页信纸随风飞扬，在女孩的周身打着旋，随风而动的纸张降落时，女孩身处的环境已经彻底变成中世纪教堂的模样 |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/46ecf776adf14c9aaf842ac7b06ee924~tplv-goo7wpa0wc-image.image =250x) ||\ +| | | | | +|首尾帧生视频 |360度环绕运镜 |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f8fc1008f23a4908b7c897e8b7eb87df~tplv-goo7wpa0wc-image.image =250x) ||\ +| | | | | + + +# 新手入门 +视频生成是一个异步过程： + +1. 成功调用 `POST /contents/generations/tasks` 接口后，API 将返回一个任务 ID 。 +2. 您可以轮询 `GET /contents/generations/tasks/{id}` 接口，直到任务状态变为 `succeeded`；或者使用 Webhook 自动接收视频生成任务的状态变化。 +3. 任务完成后，您可在 content.**video_url** 字段处，下载最终生成的 MP4 文件。 + + +## Step1: 创建视频生成任务 +通过 `POST /contents/generations/tasks` 创建视频生成任务。 + +```mixin-react +return ( + + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动，可以听到风声") + .build()); + // The URL of the first frame image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png") + .build()) + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .generateAudio(generateAudio) + .ratio(ratio) + .duration(duration) + .watermark(watermark) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + service.shutdownExecutor(); + } +} +\`\`\` + +`}> +); +``` + +请求成功后，系统将返回一个任务 ID。 +```JSON +{ + "id": "cgt-2025******-****" +} +``` + + +## Step2: 查询视频生成任务 +利用创建视频生成任务时返回的 ID ，您可以查询视频生成任务的详细状态与结果。此接口会返回任务的当前状态（如 `queued` 、`running` 、 `succeeded` 等）以及生成的视频相关信息（如视频下载链接、分辨率、时长等）。 +:::tip +因模型、API负载和视频输出规格的不同，视频生成的过程可能耗时较长。为高效管理这一过程，您可以通过轮询 API 接口（详见 [基础使用](/docs/82379/1366799#754e68e3) 和 [进阶使用](/docs/82379/1366799#e190e738) 部分的 SDK 示例）来请求状态更新，或通过 [使用 Webhook 通知](/docs/82379/1366799#724d67c3) 接收通知。 + +::: +```mixin-react +return ( + + + +); +``` + +当任务状态变为 succeeded 后，您可在 content.**video_url** 字段处，下载最终生成的视频文件。 +```JSON +{ + "id": "cgt-2025****", + "model": "doubao-seedance-1-5-pro-251215", + "status": "succeeded", + "content": { + // Video download URL (file format is MP4) + "video_url": "https://ark-content-generation-cn-beijing.tos-cn-beijing.volces.com/****" + }, + "usage": { + "completion_tokens": 246840, + "total_tokens": 246840 + }, + "created_at": 1765510475, + "updated_at": 1765510559, + "seed": 58944, + "resolution": "1080p", + "ratio": "16:9", + "duration": 5, + "framespersecond": 24, + "service_tier": "default", + "execution_expires_after": 172800 +} +``` + + +# 模型能力 + + +|模型名称 | |[Seedance 1.5 pro](https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seedance-1-5-pro&projectName=default) |[Seedance 1.0 pro](https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seedance-1-0-pro&projectName=default) |[Seedance 1.0 pro fast](https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seedance-1-0-pro-fast&projectName=default) |[Seedance 1.0 lite i2v](https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seedance-1-0-lite-i2v&projectName=default) |[Seedance-1.0 lite t2v](https://console.volcengine.com/ark/region:ark+cn-beijing/model/detail?Id=doubao-seedance-1-0-lite-t2v) | +|---|---|---|---|---|---|---| +|Model ID | |doubao\-seedance\-1\-5\-pro\-251215 |doubao\-seedance\-1\-0\-pro\-250528 |doubao\-seedance\-1\-0\-pro\-fast\-251015 |doubao\-seedance\-1\-0\-lite\-t2v\-250428 |doubao\-seedance\-1\-0\-lite\-i2v\-250428 | +|文生视频 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) | +|图生视频\-首帧 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) | +|图生视频\-首尾帧 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) | +|生成有声视频 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) | +|样片模式 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) | +|返回视频尾帧 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) | +|输出视频规格 |输出分辨率 |480p, 720p, 1080p |480p, 720p, 1080p |480p, 720p, 1080p |480p, 720p, 1080p |480p, 720p, 1080p | +| |输出宽高比 |21:9, 16:9, 4:3, 1:1, 3:4, 9:16 ||||| +| |输出时长 |4~12 秒 |2~12 秒 |2~12 秒 |2~12 秒 |2~12 秒 | +| |输出视频格式 |mp4 |mp4 |mp4 |mp4 |mp4 | +|离线推理 | |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) | +|在线推理限流 |RPM |600 |600 |600 |300 |300 | +| |并发数 |10 |10 |10 |5 |5 | +|离线推理限流 |TPD |5000亿 |5000亿 |5000亿 |2500亿 |2500亿 | + +  +  +  +  + +# 基础使用 + +## 文生视频 +根据用户输入的提示词生成视频，结果具有较大的随机性，可以用于激发创作灵感。 + + +|提示词 |输出 | +|---|---| +|写实风格，晴朗的蓝天之下，一大片白色的雏菊花田，镜头逐渐拉近，最终定格在一朵雏菊花的特写上，花瓣上有几颗晶莹的露珠 ||\ +| | | + + +```mixin-react +return ( + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("写实风格，晴朗的蓝天之下，一大片白色的雏菊花田，镜头逐渐拉近，最终定格在一朵雏菊花的特写上，花瓣上有几颗晶莹的露珠") + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .ratio(ratio) + .duration(duration) + .watermark(watermark) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 10 seconds...", status); + TimeUnit.SECONDS.sleep(10); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +## 图生视频\-基于首帧（`含音频`） +通过指定视频的首帧图片，模型能够基于该图片生成与之相关且画面连贯的视频内容。 +Seedance 1.5 pro 可通过设置参数 **generate_audio** 为 `true`，生成有声视频。 + + +|提示词 |首帧 |输出 | +|---|---|---| +|女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动，可以听到风声 |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/a28ec84ff9fc4287a0d98191020a3218~tplv-goo7wpa0wc-image.image =230x) ||\ +| | | | + + +```mixin-react +return ( + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动，可以听到风声") + .build()); + // The URL of the first frame image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png") + .build()) + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .generateAudio(generateAudio) + .ratio(ratio) + .duration(duration) + .watermark(watermark) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 10 seconds...", status); + TimeUnit.SECONDS.sleep(10); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +## 图生视频\-基于首尾帧（`含音频`） +通过指定视频的起始和结束图片，模型即可生成流畅衔接首、尾帧的视频，实现画面间自然、连贯的过渡效果。 +Seedance 1.5 pro 可通过设置参数 **generate_audio** 为 `true`，生成有声视频。 + + +|提示词 |首帧 |尾帧 |输出 | +|---|---|---|---| +|图中女孩对着镜头说“茄子”，360度环绕运镜 |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/649cb2057eae48d6a6eec872d912c75c~tplv-goo7wpa0wc-image.image =160x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/e39fd8e500a34bbdad50d06659c4ea6b~tplv-goo7wpa0wc-image.image =160x) ||\ +| | | | | + + +```mixin-react +return ( + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("图中女孩对着镜头说“茄子”，360度环绕运镜") + .build()); + // The URL of the first frame image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/seepro_first_frame.jpeg") + .build()) + .role("first_frame") + .build()); + + // The URL of the last frame image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/seepro_last_frame.jpeg") + .build()) + .role("last_frame") + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .generateAudio(generateAudio) + .ratio(ratio) + .duration(duration) + .watermark(watermark) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 10 seconds...", status); + TimeUnit.SECONDS.sleep(10); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +## 图生视频\-基于参考图 +模型能精准提取参考图片（支持输入1\-4张）中各类对象的关键特征，并依据这些特征在视频生成过程中高度还原对象的形态、色彩和纹理等细节，确保生成的视频与参考图的视觉风格一致。 + + +|提示词 |参考图1 |参考图2 |参考图3 |输出 | +|---|---|---|---|---| +|[图1]戴着眼镜穿着蓝色T恤的男生和[图2]的柯基小狗，坐在[图3]的草坪上，视频卡通风格 |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/2637ac87f1e64bd897bfc651fe7d0386~tplv-goo7wpa0wc-image.image =90x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/9450c9444b574112a9f228db9e81cdf4~tplv-goo7wpa0wc-image.image =90x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/574b8785f4b740ddaff791655e8633ba~tplv-goo7wpa0wc-image.image =90x) ||\ +| | | | | | + + +```mixin-react +return ( + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("[图1]戴着眼镜穿着蓝色T恤的男生和[图2]的柯基小狗，坐在[图3]的草坪上，视频卡通风格") + .build()); + // The URL of the first reference image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/seelite_ref_1.png") + .build()) + .role("reference_image") + .build()); + // The URL of the second reference image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/seelite_ref_2.png") + .build()) + .role("reference_image") + .build()); + // The URL of the third reference image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/seelite_ref_3.png") + .build()) + .role("reference_image") + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .ratio(ratio) + .duration(duration) + .watermark(watermark) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 10 seconds...", status); + TimeUnit.SECONDS.sleep(10); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +## 管理视频任务 + +### 查询视频生成任务列表 +该接口支持传入条件筛选参数，以查询符合条件的视频生成任务列表。 + +```mixin-react +return ( + + + +); +``` + + +### 删除或取消视频生成任务 +取消排队中的视频生成任务，或者删除视频生成任务记录。 + +```mixin-react +return ( + + + +); +``` + + +## 设置视频输出规格【New】 +支持通过 **resolution、ratio、duration、frames、seed、camera_fixed、watermark** 参数控制视频输出的规格。 +:::warning +不同模型，可能对应支持不同的参数与取值，详见下方表格。当输入的参数或取值不符合所选的模型时，该参数将被忽略或触发报错。 + +* **新方式：** 在 request body 中直接传入参数。此方式为**强校验，** 若参数填写错误，模型会返回错误提示。 +* **旧方式：** 在文本提示词后追加 \-\-[parameters]。此方式为**弱校验，** 若参数填写错误，模型将自动使用默认值且不会报错。 +::: +* **新方式（推荐）：在 request body 中直接传入参数** + +```JSON +... + // Strongly recommended + // Specify the aspect ratio of the generated video as 16:9, duration as 5 seconds, resolution as 720p, seed as 11, and include a watermark. The camera is not fixed. + "model": "doubao-seedance-1-5-pro-251215", + "content": [ + { + "type": "text", + "text": "小猫对着镜头打哈欠" + } + ], + // All parameters must be written in full; abbreviations are not supported + "resolution": "720p", + "ratio":"16:9", + "duration": 5, + // "frames": 29, Either duration or frames is required + "seed": 11, + "camera_fixed": false, + "watermark": true +... +``` + + +* **旧方式：在文本提示词后追加** **\-\-[parameters]** + +```JSON +... +// Specify the aspect ratio of the generated video as 16:9, duration as 5 seconds, resolution as 720p, seed as 11, and include a watermark. The camera is not fixed. +"content": [ + { + "type": "text", + "text": "小猫对着镜头打哈欠 --rs 720p --rt 16:9 --dur 5 --seed 11 --cf false --wm true" + // "text": "小猫对着镜头打哈欠 --resolution 720p --ratio 16:9 --duration 5 --seed 11 --camerafixed false --watermark true" + } + ] + ... +``` + + + +| |doubao\-seedance\-1\-5\-pro |doubao\-seedance\-1\-0\-pro|doubao\-seedance\-1\-0\-lite\-t2v|\ +| | |doubao\-seedance\-1\-0\-pro\-fast |doubao\-seedance\-1\-0\-lite\-i2v | +|---|---|---|---| +|resolution|* 480p|* 480p|* 480p|\ +|分辨率 |* 720p|* 720p|* 720p|\ +| |* 1080p |* 1080p |* 1080p`参考图场景不支持` | +|ratio|* 16:9|* 16:9|* 16:9|\ +|宽高比 |* 4:3|* 4:3|* 4:3|\ +| |* 1:1|* 1:1|* 1:1|\ +| |* 3:4|* 3:4|* 3:4|\ +| |* 9:16|* 9:16|* 9:16|\ +| |* 21:9|* 21:9|* 21:9|\ +| |* adaptive|* adaptive`文生视频场景不支持`|* adaptive`参考图和文生视频场景不支持`|\ +| ||||\ +| ||||\ +| |---|---|---|\ +| ||||\ +| ||||\ +| |480p 各画面比例的宽高像素值如下|480p 各画面比例的宽高像素值如下|480p 各画面比例的宽高像素值如下|\ +| ||||\ +| |* `16:9`：864×496|* `16:9`：864×480|* `16:9`：864×480|\ +| |* `4:3`：752×560|* `4:3`：736×544|* `4:3`：736×544|\ +| |* `1:1`：640×640|* `1:1`：640×640|* `1:1`：640×640|\ +| |* `3:4`：560×752|* `3:4`：544×736|* `3:4`：544×736|\ +| |* `9:16`：496×864|* `9:16`：480×864|* `9:16`：480×864|\ +| |* `21:9`：992×432|* `21:9`：960×416|* `21:9`：960×416|\ +| ||||\ +| ||||\ +| |---|---|---|\ +| ||||\ +| ||||\ +| |720p 各画面比例的宽高像素值如下|720p 各画面比例的宽高像素值如下|720p 各画面比例的宽高像素值如下|\ +| ||||\ +| |* `16:9`：1280×720|* `16:9`：1248×704|* `16:9`：1248×704|\ +| |* `4:3`：1112×834|* `4:3`：1120×832|* `4:3`：1120×832|\ +| |* `1:1`：960×960|* `1:1`：960×960|* `1:1`：960×960|\ +| |* `3:4`：834×1112|* `3:4`：832×1120|* `3:4`：832×1120|\ +| |* `9:16`：720×1280|* `9:16`：704×1248|* `9:16`：704×1248|\ +| |* `21:9`：1470×630|* `21:9`：1504×640|* `21:9`：1504×640|\ +| ||||\ +| ||||\ +| |---|---|---|\ +| ||||\ +| ||||\ +| |1080p 各画面比例的宽高像素值如下|1080p 各画面比例的宽高像素值如下|1080p 各画面比例的宽高像素值如下`参考图场景不支持`|\ +| ||||\ +| |* `16:9`：1920×1080|* `16:9`：1920×1088|* `16:9`：1920×1088|\ +| |* `4:3`：1664×1248|* `4:3`：1664×1248|* `4:3`：1664×1248|\ +| |* `1:1`：1440×1440|* `1:1`：1440×1440|* `1:1`：1440×1440|\ +| |* `3:4`：1248×1664|* `3:4`：1248×1664|* `3:4`：1248×1664|\ +| |* `9:16`：1080×1920|* `9:16`：1088×1920|* `9:16`：1088×1920|\ +| |* `21:9`：2206×946 |* `21:9`：2176×928 |* `21:9`：2176×928 | +|duration|4 ~12 秒 |2 ~12 秒 |2 ~12 秒 |\ +|生成视频时长（秒） | | | | +|frames|![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/f359753773c94d97885008ca1223c9bc~tplv-goo7wpa0wc-image.image =20x) |支持 [29, 289] 区间内所有满足 25 + 4n 格式的整数值，其中 n 为正整数。 |支持 [29, 289] 区间内所有满足 25 + 4n 格式的整数值，其中 n 为正整数。 |\ +|生成视频帧数 | | | | +|seed|![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |\ +|种子整数 | | | | +|camera_fixed|![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |\ +|是否固定摄像头 | | |`参考图场景不支持` | +|watermark|![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ee51ce32c1914aed81ff95080bb7db1d~tplv-goo7wpa0wc-image.image =20x) |\ +|是否包含水印 | | | | + + +## 提示词建议 + +* **提示词 = 主体 + 运动， 背景 + 运动，镜头 + 运动 ...** +* 用简洁准确的自然语言写出你想要的效果。 +* 如果有较为明确的效果预期，建议先用生图模型生成符合预期的图片，再用图生视频进行视频片段的生成。 +* 文生视频会有较大的结果随机性，可以用于激发创作灵感 +* 图生视频时请尽量上传高清高质量的图片，上传图片的质量对图生视频影响较大。 +* 当生成的视频不符合预期时，建议修改提示词，将抽象描述换成具象描述，并注意删除不重要的部分，将重要内容前置。 +* 更多提示词的使用技巧请参见 [Seedance-1.5-pro 提示词指南](/docs/82379/2168087)、[Seedance-1.0-pro&pro-fast 提示词指南](/docs/82379/1631633)、 [Seedance-1.0-lite 提示词指南](/docs/82379/1587797)。 + + +# 进阶使用 + +## 离线推理 +针对推理时延敏感度低（例如小时级响应）的场景，建议将 **service_tier** 设为 `flex`，一键切换至离线推理模式——价格仅为在线推理的 50%，显著降低业务成本。 +注意根据业务场景设置合适的超时时间，超过该时间后任务将自动终止。 + +```mixin-react +return ( + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动") + .build()); + // The URL of the first frame image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png") + .build()) + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .ratio(ratio) + .duration(duration) + .watermark(watermark) + .serviceTier(serviceTier) + .executionExpiresAfter(executionExpiresAfter) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 60 seconds...", status); + TimeUnit.SECONDS.sleep(60); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +## 样片模式 +获得一个符合预期的生产级别视频，通常需要多次抽卡，耗时耗力。Draft 样片模式是平台推出的中间产物可视化功能，开启该功能后，将生成一段预览视频，帮助用户 **低成本验证** 生成视频的场景结构、镜头调度、主体动作与 Prompt 意图等关键要素是否符合预期，快速调整方向。确认符合预期后，再基于 Draft 视频生成最终的高质量视频。 + + +|输入 |Draft 视频 |正式视频 | +|---|---|---| +|![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/ebb5217645b04cfc94209a6f7d36a523~tplv-goo7wpa0wc-image.image =240x) |||\ +|> 提示词：女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动，可以听到风声 |||\ +| |> 生成一段预览视频，低成本验证结果。 |> 复用 Draft 视频使用 **模型、提示词、输入图片、种子值、音频设置、视频宽高比、视频时长等** 生成正式视频，保证视频关键要素一致。 | + +本功能使用分为两步： + +### Step1: 生成 Draft 视频 + +1. 设置 `"draft": true`，调用`POST /contents/generations/tasks`接口创建 Draft 视频生成任务。 +2. 调用`GET /contents/generations/tasks/{id}`接口查询生成状态和结果，下载 Draft 视频，确认是否符合预期。 + +:::tip + +* 仅 Seedance 1.5 pro 支持该功能。 +* 仅支持 480p 分辨率（使用其他分辨率会报错），不支持返回尾帧功能，不支持离线推理功能。 +* Draft 视频的 token 单价不变，消耗的 token 更少。`Draft视频token用量 = 正常视频token用量 × 折算系数`，以 Seedance 1.5 pro 为例，有声视频的折算系数为 0.6，故生成一个 Draft 有声视频的价格是正常视频的 0.6 倍，显著降低了成本。 + + +::: +```mixin-react +return ( + + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("text") + .text("女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动") + .build()); + // The URL of the first frame image + contents.add(Content.builder() + .type("image_url") + .imageUrl(CreateContentGenerationTaskRequest.ImageUrl.builder() + .url("https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png") + .build()) + .build()); + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .seed(seed) + .duration(duration) + .draft(draft) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 10 seconds...", status); + TimeUnit.SECONDS.sleep(10); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +### Step2: 基于 Draft 视频生成正式视频 +如果确认 Draft 视频符合预期，可基于 Step1 返回的 Draft 视频任务 ID，再次调用`POST /contents/generations/tasks`接口，生成最终视频。 +:::tip + +* 平台将自动复用 Draft 视频使用的用户输入（ **model、** content.**text、** content.**image_url、generate_audio、seed、ratio、duration、camera_fixed** ），生成正式视频。 +* 其余参数支持指定，不指定将使用本模型的默认值。例如：指定正式视频的分辨率、是否包含水印、是否使用离线推理、是否返回尾帧等。 +* 基于 Draft 视频生成最终视频属于正常推理过程，按照正常视频消耗 token 量计费。 +* Draft 视频任务 ID 的有效期为 7 天（从 **created at** 时间戳开始计算），超时后将无法使用该 Draft 视频生成正式视频。 + + +::: +```mixin-react +return ( + + + contents = new ArrayList<>(); + + // Combination of text prompt and parameters + contents.add(Content.builder() + .type("draft_task") + .draftTask(CreateContentGenerationTaskRequest.DraftTask.builder() + .id("cgt-2026****-pzjqb") + .build()) + .build()); + + + // Create a video generation task + CreateContentGenerationTaskRequest createRequest = CreateContentGenerationTaskRequest.builder() + .model(model) + .content(contents) + .watermark(watermark) + .resolution(resolution) + .returnLastFrame(returnLastFrame) + .serviceTier(serviceTier) + .build(); + + CreateContentGenerationTaskResult createResult = service.createContentGenerationTask(createRequest); + System.out.println(createResult); + + // Get the details of the task + String taskId = createResult.getId(); + GetContentGenerationTaskRequest getRequest = GetContentGenerationTaskRequest.builder() + .taskId(taskId) + .build(); + + // Polling query section + System.out.println("----- polling task status -----"); + while (true) { + try { + GetContentGenerationTaskResponse getResponse = service.getContentGenerationTask(getRequest); + String status = getResponse.getStatus(); + if ("succeeded".equalsIgnoreCase(status)) { + System.out.println("----- task succeeded -----"); + System.out.println(getResponse); + break; + } else if ("failed".equalsIgnoreCase(status)) { + System.out.println("----- task failed -----"); + System.out.println("Error: " + getResponse.getStatus()); + break; + } else { + System.out.printf("Current status: %s, Retrying in 10 seconds...", status); + TimeUnit.SECONDS.sleep(10); + } + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + System.err.println("Polling interrupted"); + break; + } + } + } +} +\`\`\` + +`}> +); +``` + + +## 生成多个连续视频 +使用前一个生成视频的尾帧，作为后一个视频任务的首帧，循环生成多个连续的视频。 +后续您可以自行使用 FFmpeg 等工具，将生成的多个短视频拼接成一个完整长视频。 + + +|输出1 |输出2 |输出3 | +|---|---|---| +||||\ +||||\ +|> 女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动 |> 女孩和狐狸在草地上奔跑，阳光明媚，女孩的笑容灿烂，狐狸欢快地跳跃 |> 女孩和狐狸坐在树下休息，女孩轻轻抚摸狐狸的毛发，狐狸温顺地趴在女孩腿上 | + +```Python +import os +import time +# Install SDK: pip install 'volcengine-python-sdk[ark]' +from volcenginesdkarkruntime import Ark + +# Make sure that you have stored the API Key in the environment variable ARK_API_KEY +# Initialize the Ark client to read your API Key from an environment variable +client = Ark( + # This is the default path. You can configure it based on the service location + base_url="https://ark.cn-beijing.volces.com/api/v3", + # Get API Key：https://console.volcengine.com/ark/region:ark+cn-beijing/apikey + api_key=os.environ.get("ARK_API_KEY"), +) + +def generate_video_with_last_frame(prompt, initial_image_url=None): + """ + Generate video and return video URL and last frame URL + Parameters: + prompt: Text prompt for video generation + initial_image_url: Initial image URL (optional) + Returns: + video_url: Generated video URL + last_frame_url: URL of the last frame of the video + """ + print(f"----- Generating video: {prompt} -----") + + # Build content list + content = [{ + "text": prompt, + "type": "text" + }] + + # If initial image is provided, add to content + if initial_image_url: + content.append({ + "image_url": { + "url": initial_image_url + }, + "type": "image_url" + }) + + # Create video generation task + create_result = client.content_generation.tasks.create( + model="doubao-seedance-1-5-pro-251215", # Replace with Model ID + content=content, + return_last_frame=True, + ratio="adaptive", + duration=5, + watermark=False, + ) + + # Poll to check task status + task_id = create_result.id + while True: + get_result = client.content_generation.tasks.get(task_id=task_id) + status = get_result.status + + if get_result.status == "succeeded": + print("Video generation succeeded") + try: + if hasattr(get_result, 'content') and hasattr(get_result.content, 'video_url') and hasattr(get_result.content, 'last_frame_url'): + return get_result.content.video_url, get_result.content.last_frame_url + print("Failed to obtain video URL or last frame URL") + return None, None + except Exception as e: + print(f"Error occurred while obtaining video URL and last frame URL: {e}") + return None, None + elif status == "failed": + print(f"----- Video generation failed -----") + print(f"Error: {get_result.error}") + return None, None + else: + print(f"Current status: {status}, retrying in 10 seconds...") + time.sleep(10) + + + +if __name__ == "__main__": + # Define 3 video prompts + prompts = [ + "女孩抱着狐狸，女孩睁开眼，温柔地看向镜头，狐狸友善地抱着，镜头缓缓拉出，女孩的头发被风吹动", + "女孩和狐狸在草地上奔跑，阳光明媚，女孩的笑容灿烂，狐狸欢快地跳跃", + "女孩和狐狸坐在树下休息，女孩轻轻抚摸狐狸的毛发，狐狸温顺地趴在女孩腿上" + ] + + # Store generated video URLs + video_urls = [] + + # Initial image URL + initial_image_url = "https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png" + + # Generate 3 short videos + for i, prompt in enumerate(prompts): + print(f"Generating video {i+1}") + video_url, last_frame_url = generate_video_with_last_frame(prompt, initial_image_url) + + if video_url and last_frame_url: + video_urls.append(video_url) + print(f"Video {i+1} URL: {video_url}") + # Use the last frame of the current video as the first frame of the next video + initial_image_url = last_frame_url + else: + print(f"Video {i+1} generation failed, exiting program") + exit(1) + + print("All videos generated successfully!") + print("Generated video URL list:") + for i, url in enumerate(video_urls): + print(f"Video {i+1}: {url}") +``` + + +## 使用 Webhook 通知 +通过 **callback_url** 参数可以指定一个回调通知地址，当视频生成任务的状态发生变化时，方舟会向该地址发送一条 POST 请求，方便您及时获取任务最新情况。 请求内容结构与[查询任务API](https://www.volcengine.com/docs/82379/1521309)的返回体一致。 +```Bash +{ + "id": "cgt-2025****", + "model": "doubao-seedance-1-5-pro-251215", + "status": "running", # Possible status values: queued, running, succeeded, failed, expired + "created_at": 1765434920, + "updated_at": 1765434920, + "service_tier": "default", + "execution_expires_after": 172800 +} +``` + +您需要自行搭建一个公网可访问的 Web Server 来接收 Webhook 通知。以下是一个简单的 Web Server 代码示例，供您参考。 +```Python +# Building a Simple Web Server with Python Flask for Webhook Notification Processing + +from flask import Flask, request, jsonify +import sqlite3 +import logging +from datetime import datetime +import os + +# === Basic Configuration === +app = Flask(__name__) +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[logging.FileHandler('webhook.log'), logging.StreamHandler()] +) +# Database path +DB_PATH = 'video_tasks.db' + +# === Database Initialization === +def init_db(): + """Automatically create task table on first run, aligning fields with callback parameters""" + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + # Create table: task_id as primary key for idempotent updates + cursor.execute(''' + CREATE TABLE IF NOT EXISTS video_generation_tasks ( + task_id TEXT PRIMARY KEY, + model TEXT NOT NULL, + status TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + service_tier TEXT NOT NULL, + execution_expires_after INTEGER NOT NULL, + last_callback_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + ''') + conn.commit() + conn.close() + logging.info("Database initialized, table created/exists") + +# === Core Webhook Interface === +@app.route('/webhook/callback', methods=['POST']) +def video_task_callback(): + """Core interface for receiving Ark callback""" + try: + # 1. Parse callback request body (JSON format) + callback_data = request.get_json() + if not callback_data: + logging.error("Callback request body empty or non-JSON format") + return jsonify({"code": 400, "msg": "Invalid JSON data"}), 400 + + # 2. Validate required fields + required_fields = ['id', 'model', 'status', 'created_at', 'updated_at', 'service_tier', 'execution_expires_after'] + for field in required_fields: + if field not in callback_data: + logging.error(f"Callback data missing required field: {field}, data: {callback_data}") + return jsonify({"code": 400, "msg": f"Missing field: {field}"}), 400 + + # 3. Extract key information and log + task_id = callback_data['id'] + status = callback_data['status'] + model = callback_data['model'] + logging.info(f"Received task callback | Task ID: {task_id} | Status: {status} | Model: {model}") + print(f"[{datetime.now()}] Task {task_id} status updated to: {status}") # Console output + + # 4. Database operation + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + cursor.execute(''' + INSERT OR REPLACE INTO video_generation_tasks ( + task_id, model, status, created_at, updated_at, service_tier, execution_expires_after + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ''', ( + task_id, + model, + status, + callback_data['created_at'], + callback_data['updated_at'], + callback_data['service_tier'], + callback_data['execution_expires_after'] + )) + conn.commit() + conn.close() + logging.info(f"Task {task_id} database update successful") + + # 5. Return 200 response + return jsonify({"code": 200, "msg": "Callback received successfully", "task_id": task_id}), 200 + + except Exception as e: + # Catch all exceptions to avoid returning 5xx + logging.error(f"Callback processing failed: {str(e)}", exc_info=True) + return jsonify({"code": 200, "msg": "Callback received successfully (internal processing exception)"}), 200 + +# === Helper Interface (Optional, for querying task status) === +@app.route('/tasks/', methods=['GET']) +def get_task_status(task_id): + """Query latest status of specified task""" + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + cursor.execute('SELECT * FROM video_generation_tasks WHERE task_id = ?', (task_id,)) + task = cursor.fetchone() + conn.close() + if not task: + return jsonify({"code": 404, "msg": "Task not found"}), 404 + # Map field names for response + fields = ['task_id', 'model', 'status', 'created_at', 'updated_at', 'service_tier', 'execution_expires_after', 'last_callback_at'] + task_dict = dict(zip(fields, task)) + return jsonify({"code": 200, "data": task_dict}), 200 + +# === Service Startup === +if __name__ == '__main__': + # Initialize database + init_db() + # Start Flask service (bind to 0.0.0.0 for public access, port customizable) + # Test environment: debug=True; Production environment should disable debug and use gunicorn + app.run(host='0.0.0.0', port=8080, debug=False) +``` + + +# 使用限制 + +## 保存时间 +任务数据（如任务状态、视频URL等）仅保留24小时，超时后会被自动清除。请您务必及时保存生成的视频。 + +## 限流说明 +**模型限流** +**default（在线推理）** + +* RPM 限流：账号下同模型（区分模型版本）每分钟允许创建的任务数量上限。若超过该限制，创建视频生成任务时会报错。 +* 并发数限制：账号下同模型（区分模型版本）同一时刻在处理中的任务数量上限。超过此限制的任务将进入队列等待处理。 +* 不同模型的限制值不同，详见[视频生成能力](/docs/82379/1330310#2705b333)。 + +**flex（离线推理）** + +* TPD 限流：账号在一天内对同一模型（区分模型版本）的总调用 token 上限。超过此限制的调用请求将被拒绝。不同模型的 TPD 限流值不同，详见[视频生成能力](/docs/82379/1330310#2705b333)。 + + +## 图片裁剪规则 +**Seedance 系列模型的图生视频场景，支持设置生成视频的宽高比。** 当选择的视频宽高与您上传的图片宽高比不一致时，方舟会对您的图片进行裁剪，裁剪时会居中裁剪。详细规则如下： +:::tip +若要呈现出较好的视频效果，建议所指定的视频宽高比（ratio）与实际上传图片的宽高比尽可能接近。 + +::: +1. 输入参数： + * 原始图片宽度记为`W`（单位：像素），高度记为`H`（单位：像素）。 + * 目标比例记为`A:B`（例如，21:9），这表示裁剪后的宽度与高度之比应为 `A/B`（如 21/9≈2.333）。 +2. 比较宽高比： +* 计算原始图片的宽高比`Ratio_原始=W/H`。 +* 计算目标比例的比值`Ratio_目标=A/B`（例如，21:9 的 Ratio目标=21/9≈2.333)。 +* 根据比较结果，决定裁剪基准： + * 如果`Ratio_原始Ratio_目标`（即原始图片“太宽”或“横宽”），则以高度为基准裁剪。 + * 如果相等，则无需裁剪，直接使用全图。 +3. 裁剪尺寸计算（量化公式）： + * 以宽度为基准（适用于竖高图片）： + * 裁剪宽度`Crop_W=W`（使用整个原始宽度）。 + * 裁剪高度`Crop_H=(B/A)×W`（根据目标比例等比例计算高度）。 + * 裁剪区域的起始坐标（居中定位）： + * X 坐标（水平）：总是 0（因为宽度全用，从左侧开始）。 + * Y 坐标（垂直）：`(H−Crop_H)/2`（确保垂直居中，从顶部开始）。 + * 以高度为基准（适用于横宽图片）： + * 裁剪高度`Crop_H=H`（使用整个原始高度）。 + * 裁剪宽度`Crop_W=(A/B)×H`（根据目标比例等比例计算宽度）。 + * 裁剪区域的起始坐标（居中定位）： + * X 坐标（水平）：`(W−Crop_W)/2`（确保水平居中，从左侧开始）。 + * Y 坐标（垂直）：总是 0（因为高度全用，从顶部开始）。 +4. 裁剪结果： + * 最终裁剪出的图片尺寸为`Crop_W×Crop_H`，比例严格为`A:B`，且完全位于原始图片内部，无黑边。 + * 裁剪区域总是以原始图片中心为基准，因此内容居中。 +5. 裁剪示例： +> 以 Seedance 1.0 Pro 首帧图生视频功能为例 + + + +|输入的首帧图片 |指定的宽高比ratio |生成的视频结果 | +|---|---|---| +|16:9|21:9 ||\ +|![图片](https://p9-arcosite.byteimg.com/tos-cn-i-goo7wpa0wc/c66d7faff6104320a981b36149dc713f~tplv-goo7wpa0wc-image.image =1920x) | | | +|^^|16:9 ||\ +| | | | +|^^|4:3 ||\ +| | | | +|^^|1:1 ||\ +| | | | +|^^|3:4 ||\ +| | | | +|^^|9:16 ||\ +| | | | + + + diff --git a/src/views/VideoExplanation.vue b/src/views/VideoExplanation.vue index 27b034a..4e43fbb 100644 --- a/src/views/VideoExplanation.vue +++ b/src/views/VideoExplanation.vue @@ -2,11 +2,14 @@ import { ref, computed, onUnmounted, watch, nextTick } from "vue"; import { useRouter } from "vue-router"; import axios from "axios"; +import { FFmpeg } from "@ffmpeg/ffmpeg"; +import { fetchFile, toBlobURL } from "@ffmpeg/util"; import { DOUBAO_KEY } from "@/config/index.js"; // ── 状态 ── -const status = ref("idle"); // idle | generating | done | error -// 生成阶段: script（解析试题/生成脚本）| firstFrame（生成首帧）| lastFrame（生成尾帧）| video（生成视频） +// idle | generating | script-review | images-review | video-step | done | error +const status = ref("idle"); +// 生成阶段: script | firstFrame | lastFrame | video const phase = ref("script"); const textInput = ref(""); const errorMsg = ref(""); @@ -15,32 +18,42 @@ const progressText = ref(""); const cancelTokenSource = ref(null); // 脚本数据 -const scriptContent = ref(""); // 讲解文案 -const storyboard = ref([]); // 分镜脚本列表 [{ index, desc, visual }] +const scriptContent = ref(""); +const storyboard = ref([]); // [{ index, desc, visual }] // 多片段视频 -const CLIP_DURATION = 5; // 每段视频时长（秒），万相模型固定5秒 -const MAX_CLIPS = 16; // 最多片段数（上限约 80 秒） -const videoClips = ref([]); // 已生成的视频片段 URL 列表 -const currentClipIndex = ref(0); // 当前正在生成第几段 +const CLIP_DURATION = 5; +const MAX_CLIPS = 16; +const videoClips = ref([]); +const currentClipIndex = ref(0); + +// 配音音频 +const narrationAudioUrl = ref(""); // 分镜图片（首尾帧） const storyboardImages = ref([]); // [{ index, firstFrameUrl, lastFrameUrl }] -const videoPlayerRef = ref(null); // video 元素引用 -const currentPlayIndex = ref(0); // 当前播放到第几段 +const videoPlayerRef = ref(null); +const currentPlayIndex = ref(0); + +// video-step 预览 +const previewVideoRef = ref(null); + +// 是否为最后一段视频 +const isLastClip = computed( + () => currentClipIndex.value + 1 >= storyboardImages.value.length +); // ── API 配置 ── -// 脚本生成 API（GPT 兼容接口） const SCRIPT_API_KEY = "sk-aVLnOpoEsktYJh0A51aVnwqM3o5WF6JUf9icmWkOXMnZKvOM"; const SCRIPT_API_URL = "https://api.oaibest.com/v1/chat/completions"; const SCRIPT_MODEL = "gpt-4o"; -// 视频生成 API（阿里云 DashScope 万相首尾帧模型） -const VIDEO_API_KEY = "sk-74fa3459e6a84dda85135fcbb4cf0f29"; // 替换为阿里云百炼 API Key -const VIDEO_API_URL = - "/dashscope-api/api/v1/services/aigc/image2video/video-synthesis"; -const VIDEO_TASK_URL = "/dashscope-api/api/v1/tasks"; -const VIDEO_MODEL = "wan2.2-kf2v-flash"; +const VIDEO_API_URL = "/ark-api/api/v3/contents/generations/tasks"; +const VIDEO_MODEL = "doubao-seedance-1-5-pro-251215"; + +// ── TTS 配置（豆包中文语音）── +const TTS_API_URL = "/tts-api/api/v3/tts/unidirectional"; +const TTS_VOICE = "BV700_streaming"; // ── Router ── const router = useRouter(); @@ -52,23 +65,53 @@ const canGenerate = computed(() => { const goBack = () => router.back(); +// ── 统一错误处理 ── +const handleError = (err) => { + if (axios.isCancel(err)) return; + console.error("生成失败:", err); + errorMsg.value = + err?.response?.data?.error?.message || + err?.response?.data?.message || + err?.response?.data?.error || + err.message || + "生成失败，请稍后重试"; + status.value = "error"; +}; + // ── 阶段一：解析试题，生成讲解文案 + 分镜脚本 ── const generateScript = async () => { phase.value = "script"; progress.value = 5; progressText.value = "正在解析试题..."; - const systemPrompt = `你是一位专业的英语教师，擅长制作试题讲解视频。 -请根据用户提供的英语试题，完成以下两项任务，并严格按照 JSON 格式返回： + const systemPrompt = `你是一位专业的英语教师，擅长制作试题讲解视频的脚本和分镜设计。 +所有分镜画面必须严格遵循统一的视觉风格，确保整段视频镜头连贯、风格一致。 + +**统一视觉风格要求（每个分镜的 visual 字段必须体现）：** +- 风格：2D 扁平化教育动画，简洁现代的信息图风格 +- 配色：深蓝色渐变背景（#1a1a3e → #2d2b55），白色/浅蓝文字，粉色/青色强调色 +- 字体：无衬线粗体标题 + 常规正文，文字全部使用英文，文字居中对齐 +- 布局：核心信息居中展示，四周留白，底部可放置页码或进度指示 +- 严禁：任何中文字符出现在画面上！所有文字必须翻译为英文 +- 严禁：任何真人、卡通人物、人物剪影、手指指引等拟人化元素 +- 元素：仅使用几何图形、箭头、色块、图标、表格、下划线、高亮标记等抽象视觉元素 + +请根据用户提供的英语试题，严格按照 JSON 格式返回： { "script": "完整的讲解文案（中文，包含题目分析、解题思路、答案解析）", "storyboard": [ - { "index": 1, "desc": "分镜描述", "visual": "画面/动画/图表/板书说明" }, + { "index": 1, "desc": "分镜描述（该段讲解的内容概述）", "visual": "画面设计描述，所有画面文字必须为英文（如 Question/Answer/Key Point）。必须包含：深蓝色渐变背景、核心英文文字内容、使用的视觉元素（箭头/色块/高亮）、动画效果描述" }, ... ] } -storyboard 每个分镜对应一个知识点或解题步骤，visual 字段描述该分镜的视觉呈现方式（如：板书写出语法结构、动画展示句子成分、图表对比选项等）。 -**请根据试题的复杂度自行决定分镜数量（建议 3~12 个），每个分镜对应约 15 秒视频内容。简单题目可少分镜，复杂题目适当增加分镜，确保讲解充分。**`; + +**分镜 visual 字段编写规范：** +1. 每个分镜的 visual 必须以"深蓝色渐变背景"开头，确保背景统一 +2. 描述具体的视觉元素和动画效果，避免模糊描述 +3. 相邻分镜之间应有视觉延续感（如颜色过渡、元素呼应） +4. 每个分镜聚焦一个核心知识点，视觉元素不要过于复杂 + +**请根据试题的复杂度自行决定分镜数量（建议 3~8 个，简单题 3~4 个，复杂题 5~8 个），确保讲解充分且每个分镜信息量适中。**`; const response = await axios.post( SCRIPT_API_URL, @@ -101,17 +144,25 @@ storyboard 每个分镜对应一个知识点或解题步骤，visual 字段描 throw new Error("脚本生成失败，请重试"); } - progress.value = 35; - progressText.value = "脚本生成完成，准备生成视频..."; + progress.value = 100; + progressText.value = "脚本生成完成！"; }; // ── 阶段二：调用豆包 Seedream 生成分镜首尾帧图片 ── const IMAGE_API_URL = "/ark-api/api/v3/images/generations"; const IMAGE_MODEL = "doubao-seedream-5-0-260128"; +const VISUAL_STYLE_PREFIX = + "2D扁平化教育动画风格，深蓝色渐变背景（深蓝到暗紫），白色和浅蓝色文字，粉色和青色作为强调色，无衬线粗体字体，简洁现代的信息图布局，核心内容居中，四周留白，无任何真人、人物、卡通形象，仅使用几何图形、箭头、色块、图标、表格、高亮标记等抽象元素"; + const generateImageForShot = async (shot, frameType) => { const frameLabel = frameType === "first" ? "首帧" : "尾帧"; - const prompt = shot.visual; + const frameGuidance = + frameType === "first" + ? "画面处于初始状态：展示本分镜要讲解的问题或知识点的引入阶段，关键信息尚未完全展开，留有动画过渡的空间。画面右下角标注序号" + : "画面处于完成状态：展示本分镜的结论、答案或知识点的最终呈现，与首帧有自然的视觉延续，关键信息完整呈现且突出显示。画面右下角标注序号"; + + const prompt = `${VISUAL_STYLE_PREFIX}。${shot.visual}。${frameGuidance}`; try { const createResponse = await axios.post( @@ -155,7 +206,6 @@ const generateImageForShot = async (shot, frameType) => { return imgUrl; } catch (err) { - // 打印完整错误信息以便排查 console.error("[Seedream 生图失败]", { status: err?.response?.status, statusText: err?.response?.statusText, @@ -179,14 +229,12 @@ const generateImagesFromStoryboard = async () => { const shot = shots[i]; currentClipIndex.value = i + 1; - // 生成首帧 - progress.value = 15 + Math.round((i / totalClips) * 25); + progress.value = Math.round((i / totalClips) * 100); progressText.value = `正在生成第 ${i + 1}/${totalClips} 段首帧图片...`; const firstFrameUrl = await generateImageForShot(shot, "first"); - // 生成尾帧 - progress.value = 15 + Math.round(((i + 0.5) / totalClips) * 25); + progress.value = Math.round(((i + 0.5) / totalClips) * 100); progressText.value = `正在生成第 ${i + 1}/${totalClips} 段尾帧图片...`; const lastFrameUrl = await generateImageForShot(shot, "last"); @@ -202,50 +250,177 @@ const generateImagesFromStoryboard = async () => { throw new Error("所有分镜图片生成失败"); } - progress.value = 40; - progressText.value = "图片生成完成，准备生成视频..."; + progress.value = 100; + progressText.value = "图片生成完成！"; }; -// ── 阶段三：根据首尾帧图片调用万相模型生成视频 ── +// ── TTS：生成中文讲解音频 ── +const generateNarrationAudio = async (text) => { + try { + const response = await axios.post( + TTS_API_URL, + { + model: "chat-tts-240830", + input: text, + voice: TTS_VOICE, + stream: false, + response_format: "url", + speed: 1.0, + }, + { + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${DOUBAO_KEY}`, + }, + cancelToken: cancelTokenSource.value.token, + timeout: 60000, + } + ); + + const data = response.data; + if (data?.error) { + throw new Error(`TTS 生成失败：${data.error.message || data.error.code}`); + } + + // 返回音频 URL + return data?.data?.audio_url || null; + } catch (err) { + console.error("[TTS 生成失败]", { + status: err?.response?.status, + data: err?.response?.data, + }); + throw err; + } +}; + +// ── FFmpeg：合并视频片段和音频 ── +let ffmpeg = null; +const initFFmpeg = async () => { + if (ffmpeg) return ffmpeg; + ffmpeg = new FFmpeg(); + ffmpeg.on("log", ({ message }) => { + console.log("[FFmpeg]", message); + }); + ffmpeg.on("progress", ({ progress }) => { + console.log("[FFmpeg progress]", Math.round(progress * 100) + "%"); + }); + const baseURL = "https://unpkg.com/@ffmpeg/core@0.12.6/dist/esm"; + await ffmpeg.load({ + coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"), + wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, "application/wasm"), + }); + return ffmpeg; +}; + +const mergeVideoAndAudio = async (videoUrls, audioUrl) => { + progressText.value = "正在合并视频和音频..."; + progress.value = 80; + + const ff = await initFFmpeg(); + + // 写入视频文件 + for (let i = 0; i < videoUrls.length; i++) { + progressText.value = `正在处理第 ${i + 1}/${videoUrls.length} 段视频...`; + const videoData = await fetchFile(videoUrls[i]); + await ff.writeFile(`input_video_${i}.mp4`, videoData); + } + + // 写入音频文件 + progressText.value = "正在处理音频..."; + const audioData = await fetchFile(audioUrl); + await ff.writeFile("input_audio.mp3", audioData); + + // 创建文件列表用于concat + const concatList = videoUrls.map((_, i) => `file 'input_video_${i}.mp4'`).join("\n"); + await ff.writeFile("concat.txt", concatList); + + // 合并视频片段 + progressText.value = "正在拼接视频片段..."; + progress.value = 85; + await ff.exec([ + "-f", "concat", "-safe", "0", + "-i", "concat.txt", + "-c", "copy", + "merged_video.mp4" + ]); + + // 合并视频和音频 + progressText.value = "正在合成最终视频..."; + progress.value = 90; + await ff.exec([ + "-i", "merged_video.mp4", + "-i", "input_audio.mp3", + "-c:v", "copy", + "-c:a", "aac", + "-shortest", + "final_output.mp4" + ]); + + // 读取最终文件 + const outputData = await ff.readFile("final_output.mp4"); + const blob = new Blob([outputData.buffer], { type: "video/mp4" }); + const url = URL.createObjectURL(blob); + + // 清理临时文件 + for (let i = 0; i < videoUrls.length; i++) { + try { await ff.deleteFile(`input_video_${i}.mp4`); } catch {} + } + try { await ff.deleteFile("input_audio.mp3"); } catch {} + try { await ff.deleteFile("concat.txt"); } catch {} + try { await ff.deleteFile("merged_video.mp4"); } catch {} + try { await ff.deleteFile("final_output.mp4"); } catch {} + + return url; +}; + +// ── 阶段三：根据首尾帧图片调用 Seedance 生成视频 ── const generateSingleClip = async (shot, firstFrameUrl, lastFrameUrl) => { - const videoPrompt = `英语试题讲解视频片段（第 ${shot.index} 段，约5秒），教学风格，纯动画板书形式，不出现任何真人或人物形象。 -整体讲解文案：${scriptContent.value} -当前分镜内容：${shot.desc} -视觉要求：${shot.visual}`; + const videoPrompt = `2D flat animation style, dark blue gradient background, white and light blue text, pink and cyan accent colors, clean modern infographic layout, centered content with ample margins. +Core requirements: Smooth transition from first frame to last frame, maintain visual continuity, stable camera without shaking, all visual elements (colors, fonts, icon styles) consistent across shots. +STRICTLY FORBIDDEN: any real people, characters, fingers, cartoon images, Chinese characters, any text in languages other than English. +ONLY use: geometric shapes, arrows, color blocks, icons, charts, highlight markers and other abstract educational elements. +Animation transitions: text fade-in line by line, color blocks slide in smoothly, arrow stroke animations, highlight area gradual appearance, natural and smooth transitions. +IMPORTANT: All text rendered in the video must be in ENGLISH only. Do not include any Chinese characters in visual text.`; const createResponse = await axios.post( VIDEO_API_URL, { model: VIDEO_MODEL, - input: { - first_frame_url: firstFrameUrl, - last_frame_url: lastFrameUrl, - prompt: videoPrompt, - }, - parameters: { - resolution: "720P", - prompt_extend: true, - watermark: false, - }, + content: [ + { type: "text", text: videoPrompt }, + { + type: "image_url", + image_url: { url: firstFrameUrl }, + role: "first_frame", + }, + { + type: "image_url", + image_url: { url: lastFrameUrl }, + role: "last_frame", + }, + ], + generate_audio: false, + ratio: "adaptive", + duration: CLIP_DURATION, + watermark: false, }, { headers: { "Content-Type": "application/json", - Authorization: `Bearer ${VIDEO_API_KEY}`, - "X-DashScope-Async": "enable", + Authorization: `Bearer ${DOUBAO_KEY}`, }, cancelToken: cancelTokenSource.value.token, - timeout: 30000, + timeout: 120000, } ); - const taskId = createResponse.data?.output?.task_id; + const taskId = createResponse.data?.id; if (!taskId) { + console.error("[Seedance 视频响应]", createResponse.data); throw new Error(`分镜 ${shot.index} 创建视频任务失败`); } - // 轮询任务状态（建议间隔 15 秒） - const maxPollAttempts = 24; // 最多 6 分钟 + const maxPollAttempts = 24; let pollCount = 0; while (pollCount < maxPollAttempts) { @@ -254,22 +429,22 @@ const generateSingleClip = async (shot, firstFrameUrl, lastFrameUrl) => { await new Promise((resolve) => setTimeout(resolve, 15000)); pollCount++; - const statusResponse = await axios.get(`${VIDEO_TASK_URL}/${taskId}`, { - headers: { Authorization: `Bearer ${VIDEO_API_KEY}` }, + const statusResponse = await axios.get(`${VIDEO_API_URL}/${taskId}`, { + headers: { Authorization: `Bearer ${DOUBAO_KEY}` }, cancelToken: cancelTokenSource.value.token, - timeout: 10000, + timeout: 30000, }); - const taskStatus = statusResponse.data?.output?.task_status; + const taskStatus = statusResponse.data?.status; - if (taskStatus === "SUCCEEDED") { - return statusResponse.data?.output?.video_url || null; + if (taskStatus === "succeeded") { + return statusResponse.data?.content?.video_url || null; } - if (taskStatus === "FAILED") { + if (taskStatus === "failed") { throw new Error( `分镜 ${shot.index} 视频生成失败：${ - statusResponse.data?.output?.message || "未知错误" + statusResponse.data?.error?.message || statusResponse.data?.error || "未知错误" }` ); } @@ -278,51 +453,66 @@ const generateSingleClip = async (shot, firstFrameUrl, lastFrameUrl) => { throw new Error(`分镜 ${shot.index} 视频生成超时`); }; -const generateVideoFromScript = async () => { - // 阶段二：生成分镜首尾帧图片 - await generateImagesFromStoryboard(); +// 生成 currentClipIndex 指向的视频片段 +const generateNextVideoClip = async () => { + const i = currentClipIndex.value; + const shot = storyboard.value[i]; + const imageData = storyboardImages.value[i]; + const total = storyboardImages.value.length; - if (status.value !== "generating") return; + progress.value = Math.round(((i + 0.5) / total) * 100); + progressText.value = `正在生成第 ${i + 1}/${total} 段视频...`; - // 阶段三：根据首尾帧图片生成视频 - phase.value = "video"; - videoClips.value = []; + const url = await generateSingleClip( + shot, + imageData.firstFrameUrl, + imageData.lastFrameUrl + ); - const totalClips = storyboardImages.value.length; - - for (let i = 0; i < totalClips; i++) { - if (status.value !== "generating") return; - - const shot = storyboard.value[i]; - const imageData = storyboardImages.value[i]; - currentClipIndex.value = i + 1; - const baseProgress = 65; - const clipProgress = Math.round((i / totalClips) * 33); - progress.value = baseProgress + clipProgress; - progressText.value = `正在生成第 ${i + 1}/${totalClips} 段视频...`; - - const url = await generateSingleClip( - shot, - imageData.firstFrameUrl, - imageData.lastFrameUrl - ); - if (url) { + if (url) { + if (videoClips.value.length > i) { + videoClips.value[i] = url; + } else { videoClips.value.push(url); } } - - if (videoClips.value.length === 0) { - throw new Error("所有视频片段生成失败"); - } - - progress.value = 99; - progressText.value = "视频片段全部生成完成！"; }; -// ── 主入口 ── +// ══════════════════════════════════════ +// ── 测试视频生成（跳过脚本/图片步骤）── +// ══════════════════════════════════════ +const testVideo = async () => { + if (status.value === "generating") return; + status.value = "generating"; + errorMsg.value = ""; + videoClips.value = []; + progress.value = 0; + progressText.value = "测试视频生成中..."; + cancelTokenSource.value = axios.CancelToken.source(); + + // 使用公开测试图片 + const testFirstFrame = "https://ark-project.tos-cn-beijing.volces.com/doc_image/i2v_foxrgirl.png"; + const testLastFrame = "https://ark-project.tos-cn-beijing.volces.com/doc_image/seepro_last_frame.jpeg"; + + try { + const url = await generateSingleClip({ index: 1 }, testFirstFrame, testLastFrame); + if (!url) throw new Error("未获取到视频地址"); + videoClips.value = [url]; + progress.value = 100; + progressText.value = "测试视频生成完成！"; + status.value = "done"; + } catch (err) { + handleError(err); + } +}; + +// ══════════════════════════════════════ +// ── 分步流程控制 ── +// ══════════════════════════════════════ + +// Step 1: 仅生成脚本 → 进入 script-review const generateVideo = async () => { if (!canGenerate.value) return; - status.value = "generating"; errorMsg.value = ""; videoClips.value = []; @@ -331,53 +521,204 @@ const generateVideo = async () => { storyboardImages.value = []; currentClipIndex.value = 0; progress.value = 0; - progressText.value = "正在连接 AI 模型..."; - cancelTokenSource.value = axios.CancelToken.source(); try { - // 阶段一：解析试题，生成讲解文案 + 分镜脚本 await generateScript(); - if (status.value !== "generating") return; - - // 阶段二：生成分镜首尾帧图片 - // 阶段三：按分镜逐段生成视频 - await generateVideoFromScript(); - - if (status.value !== "generating") return; - - progress.value = 100; - progressText.value = "视频生成完成！"; - status.value = "done"; + status.value = "script-review"; } catch (err) { - if (axios.isCancel(err)) return; - console.error("生成失败:", err); - errorMsg.value = - err?.response?.data?.error?.message || - err?.response?.data?.message || - err?.response?.data?.error || - err.message || - "生成失败，请稍后重试"; - status.value = "error"; + handleError(err); } }; +// 重新生成脚本 +const regenerateScript = async () => { + status.value = "generating"; + scriptContent.value = ""; + storyboard.value = []; + storyboardImages.value = []; + videoClips.value = []; + currentClipIndex.value = 0; + progress.value = 0; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + await generateScript(); + if (status.value !== "generating") return; + status.value = "script-review"; + } catch (err) { + handleError(err); + } +}; + +// 确认脚本 → 生成图片 → 进入 images-review +const confirmScript = async () => { + status.value = "generating"; + storyboardImages.value = []; + videoClips.value = []; + currentClipIndex.value = 0; + phase.value = "firstFrame"; + progress.value = 0; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + await generateImagesFromStoryboard(); + if (status.value !== "generating") return; + status.value = "images-review"; + } catch (err) { + handleError(err); + } +}; + +// 重新生成图片 +const regenerateImages = async () => { + status.value = "generating"; + storyboardImages.value = []; + videoClips.value = []; + currentClipIndex.value = 0; + phase.value = "firstFrame"; + progress.value = 0; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + await generateImagesFromStoryboard(); + if (status.value !== "generating") return; + status.value = "images-review"; + } catch (err) { + handleError(err); + } +}; + +// 返回脚本审核 +const goBackToScriptReview = () => { + storyboardImages.value = []; + videoClips.value = []; + currentClipIndex.value = 0; + status.value = "script-review"; +}; + +// 确认图片 → 生成第一段视频 → 进入 video-step +const confirmImages = async () => { + status.value = "generating"; + videoClips.value = []; + currentClipIndex.value = 0; + phase.value = "video"; + progress.value = 0; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + await generateNextVideoClip(); + if (status.value !== "generating") return; + progress.value = Math.round( + ((currentClipIndex.value + 1) / storyboardImages.value.length) * 100 + ); + status.value = "video-step"; + } catch (err) { + handleError(err); + } +}; + +// 继续生成下一段视频 +const continueNextClip = async () => { + currentClipIndex.value++; + if (currentClipIndex.value >= storyboardImages.value.length) { + // 所有视频片段已生成，开始合成配音和合并 + progress.value = 90; + progressText.value = "正在生成中文配音..."; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + // 生成 TTS 中文配音 + const audioUrl = await generateNarrationAudio(scriptContent.value); + if (!audioUrl) throw new Error("配音生成失败"); + + // 合并视频和音频 + const finalVideoUrl = await mergeVideoAndAudio(videoClips.value, audioUrl); + + narrationAudioUrl.value = finalVideoUrl; + progress.value = 100; + progressText.value = "视频生成完成！"; + status.value = "done"; + } catch (err) { + handleError(err); + } + return; + } + status.value = "generating"; + phase.value = "video"; + progressText.value = `正在生成第 ${currentClipIndex.value + 1}/${storyboardImages.value.length} 段视频...`; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + await generateNextVideoClip(); + if (status.value !== "generating") return; + progress.value = Math.round( + ((currentClipIndex.value + 1) / storyboardImages.value.length) * 100 + ); + status.value = "video-step"; + } catch (err) { + handleError(err); + } +}; + +// 重新生成当前片段 +const regenerateCurrentClip = async () => { + narrationAudioUrl.value = ""; // 清空合并视频，重新合并 + status.value = "generating"; + phase.value = "video"; + const i = currentClipIndex.value; + progressText.value = `正在重新生成第 ${i + 1}/${storyboardImages.value.length} 段视频...`; + cancelTokenSource.value = axios.CancelToken.source(); + + try { + await generateNextVideoClip(); + if (status.value !== "generating") return; + progress.value = Math.round( + ((i + 1) / storyboardImages.value.length) * 100 + ); + status.value = "video-step"; + } catch (err) { + handleError(err); + } +}; + +// 返回图片审核 +const goBackToImagesReview = () => { + videoClips.value = []; + narrationAudioUrl.value = ""; + status.value = "images-review"; +}; + +// 取消当前生成 const cancelGeneration = () => { if (cancelTokenSource.value) { cancelTokenSource.value.cancel("用户取消生成"); } - status.value = "idle"; - progress.value = 0; + if (phase.value === "script") { + status.value = scriptContent.value ? "script-review" : "idle"; + } else if (phase.value === "firstFrame" || phase.value === "lastFrame") { + status.value = + storyboardImages.value.length > 0 ? "images-review" : "script-review"; + } else if (phase.value === "video") { + status.value = + videoClips.value.length > 0 ? "video-step" : "images-review"; + } else { + status.value = "idle"; + } progressText.value = ""; - currentClipIndex.value = 0; }; +// 重置全部 const resetAll = () => { + if (cancelTokenSource.value) { + cancelTokenSource.value.cancel("重置"); + } status.value = "idle"; phase.value = "script"; textInput.value = ""; videoClips.value = []; + narrationAudioUrl.value = ""; errorMsg.value = ""; progress.value = 0; progressText.value = ""; @@ -387,14 +728,22 @@ const resetAll = () => { currentClipIndex.value = 0; }; -// ── 多片段视频顺序播放 ── +// ── 多片段视频顺序播放（done 状态） ── const setupPlayer = () => { - if (!videoPlayerRef.value || videoClips.value.length === 0) return; + if (!videoPlayerRef.value) return; + // 优先使用合并后的带配音视频 + if (narrationAudioUrl.value) { + videoPlayerRef.value.src = narrationAudioUrl.value; + videoPlayerRef.value.play().catch(() => {}); + return; + } + + // 否则使用原始片段顺序播放（兼容旧状态） + if (videoClips.value.length === 0) return; const video = videoPlayerRef.value; currentPlayIndex.value = 0; video.src = videoClips.value[0]; - video.addEventListener("ended", () => { currentPlayIndex.value++; if (currentPlayIndex.value < videoClips.value.length) { @@ -404,12 +753,19 @@ const setupPlayer = () => { }); }; -// 状态变为 done 时初始化播放器 watch(status, async (newVal) => { if (newVal === "done" && videoClips.value.length > 0) { await nextTick(); setupPlayer(); } + if (newVal === "video-step") { + await nextTick(); + if (previewVideoRef.value && videoClips.value.length > 0) { + previewVideoRef.value.src = + videoClips.value[currentClipIndex.value]; + previewVideoRef.value.play().catch(() => {}); + } + } }); onUnmounted(() => { @@ -419,7 +775,6 @@ onUnmounted(() => { }); -