From 3f0a323e0a5400eb63c5d015bfd1697e80f6c52e Mon Sep 17 00:00:00 2001 From: ojbkyuyue Date: Wed, 16 Sep 2026 04:22:48 +0800 Subject: [PATCH] =?UTF-8?q?feat(alibaba):=20=E6=8E=A5=E5=85=A5=E9=98=BF?= =?UTF-8?q?=E9=87=8C=20DashScope=20wan3.0-video=20=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E7=94=9F=E6=88=90=E9=93=BE=E8=B7=AF=EF=BC=88=E7=AB=AF=E5=88=B0?= =?UTF-8?q?=E7=AB=AF=E5=B7=B2=E5=87=BA=E7=89=87=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - alibaba-video: fragment/surface 对齐当前 Distribution API，改用 createExactModelPrimaryGenerationFragment + sealGenerationRequestDraft，修复 @hypit/elaborator 不导出 createAssembledGenerationFragment 导致 CLI 加载失败 - alibaba-video: 端口表移除单端口 atMostOneOf（该约束至少需 2 个端口） - provider-wanx: parameters.duration 改为整数（DashScope 要求），新增 firstNumber 读取 number 端口值 - alibaba-wanx: 新增 demo.svrun 作为 Run Source（build 需吃 .svrun，.svml 是 Author Source） - 默认模型 wan3.0-video；本地实测 Build complete 并已导出视频 --- alibaba-wanx/.gitignore | 1 + alibaba-wanx/README.md | 75 +++++++ alibaba-wanx/demo.svml | 9 + alibaba-wanx/demo.svrun | 6 + alibaba-wanx/hypit.runtime.json | 17 ++ packages/alibaba-video/README.md | 99 +++++++++ packages/alibaba-video/package.json | 22 ++ packages/alibaba-video/src/activation.ts | 46 +++++ packages/alibaba-video/src/fragment.ts | 23 +++ packages/alibaba-video/src/index.ts | 250 +++++++++++++++++++++++ packages/alibaba-video/src/surface.ts | 238 +++++++++++++++++++++ packages/provider-wanx/package.json | 11 + packages/provider-wanx/src/activation.ts | 31 +++ packages/provider-wanx/src/index.ts | 1 + packages/provider-wanx/src/provider.ts | 193 +++++++++++++++++ pnpm-lock.yaml | 30 +++ 16 files changed, 1052 insertions(+) create mode 100644 alibaba-wanx/.gitignore create mode 100644 alibaba-wanx/README.md create mode 100644 alibaba-wanx/demo.svml create mode 100644 alibaba-wanx/demo.svrun create mode 100644 alibaba-wanx/hypit.runtime.json create mode 100644 packages/alibaba-video/README.md create mode 100644 packages/alibaba-video/package.json create mode 100644 packages/alibaba-video/src/activation.ts create mode 100644 packages/alibaba-video/src/fragment.ts create mode 100644 packages/alibaba-video/src/index.ts create mode 100644 packages/alibaba-video/src/surface.ts create mode 100644 packages/provider-wanx/package.json create mode 100644 packages/provider-wanx/src/activation.ts create mode 100644 packages/provider-wanx/src/index.ts create mode 100644 packages/provider-wanx/src/provider.ts diff --git a/alibaba-wanx/.gitignore b/alibaba-wanx/.gitignore new file mode 100644 index 00000000..26416673 --- /dev/null +++ b/alibaba-wanx/.gitignore @@ -0,0 +1 @@ +*.mp4 diff --git a/alibaba-wanx/README.md b/alibaba-wanx/README.md new file mode 100644 index 00000000..e3c878da --- /dev/null +++ b/alibaba-wanx/README.md @@ -0,0 +1,75 @@ +# 本地跑通 Alibaba（Wanx / DashScope）视频生成 + +本目录把用户 fork 的 `feature/alibaba-video` 分支在本地拼成可运行形态： + +- `packages/alibaba-video/` —— 模型 + SVML 表面（`` / ``）， + 声明能力 `alibaba-qwen-vvg`。分支原文件逐字放入；**仅 `src/activation.ts` 被改写** + （原分支写的是裸 `export function activate(context)`，但 hypit 0.1.9 的包加载器 + `packages/package-loader-node/src/loader.ts` 要求默认导出是 `hypit.node-package@1` + 贡献对象，否则报 “activation has no default package export”。已对齐到 `@hypit/seedance` 的写法）。 +- `packages/provider-wanx/` —— 分支缺的 **Endpoint**：真正调用 DashScope 的 Wanx 视频 API。 +- `hypit.runtime.json` —— 把能力 `@hypit/alibaba-video@1#alibaba-qwen-vvg` 绑定到本地端点 `wanx.personal`。 +- `demo.svml` —— 最小可跑样例。 + +## 本地运行步骤（网络/构建由你本机执行） + +```bash +cd E:/tools/hypit-alibaba + +# 1) 安装工作区依赖（联网一步；会用 pnpm workspace 把上面两个包软链进 node_modules） +pnpm install + +# 2) 写入 DashScope / 阿里云 API Key（存进本机 OS 凭据库，key = wanx.personal） +hypit auth login wanx.personal --runtime alibaba-wanx/hypit.runtime.json +# 提示输入时粘贴你的 DASHSCOPE_API_KEY + +# 3) 真正生成视频（--follow 会一直等到出片） +hypit build alibaba-wanx/demo.svml --runtime alibaba-wanx/hypit.runtime.json --follow +``` + +出片后视频产物在 `.hypit/execution/...` 下，CLI 会打印结果路径。 + +## 模型名（已按本地控制台对齐） + +分支把能力叫 `alibaba-qwen-vvg`（“Qwen VVG”），但阿里云**实际对外可调用**的视频生成模型名 +**不固定**（控制台可见的 video 模型会随账号/区域变）。本地测试统一用 `wan3.0-video` +（用户控制台里唯一可用的 video 模型），已写入三处默认值： + +- `provider.ts` 的 `DEFAULT_MODEL_T2V` = `DEFAULT_MODEL_I2V` = `"wan3.0-video"` +- `activation.ts` 里 `config.model` 兜底默认 `"wan3.0-video"` +- `hypit.runtime.json` 里 `endpoints.wanx.personal.config.model` = `"wan3.0-video"` + +> ⚠️ 跑通前注意：wan3.0 的 **请求参数 schema** 可能和 wanx2.1 不同 +> （本 Provider 目前按 wanx2.1 提交 `parameters: { size, duration }`）。 +> 如果本地 `hypit build --follow` 报参数错误（如 4xx / `InvalidParameter`），把报错 +> 贴给我，我按 wan3.0 的真实入参改 `provider.ts` 的 `start()` 即可——这是运行时行为， +> 需要你本机实际打一次才知道。 + +- 想临时换模型：改 `hypit.runtime.json` 的 `config.model` 即可，不必动代码。 +- 文生视频走 `model`；当 SVML 带 `reference-image` 时自动切到 `modelI2V`（当前同 `wan3.0-video`）。 + +## 改 SVML 自测 + +`demo.svml` 用 ``。图生视频写法（参考分支 README）： + +```markup + +``` + +属性范围（模型端口限定）：`prompt` ≤200 字、`duration` 5–10、`resolution` 480p/720p/1080p、 +`aspect-ratio` 1:1/16:9/9:16/4:3/3:4、`referenceImage` 可选图片。 + +## 排错 + +- 报 “installed packages do not provide @hypit/alibaba-video@1#alibaba-qwen-vvg”： + 多半是 `pnpm install` 没跑 / 没成功，导致 `node_modules/@hypit/alibaba-video` 软链不存在。 +- 报能力 unsupported：检查 `demo.svml` 的 `prompt` 是否引用了一个存在的 ``。 +- DashScope 返回 4xx：确认 API Key 与模型名（Wanx）对该账号可用。 diff --git a/alibaba-wanx/demo.svml b/alibaba-wanx/demo.svml new file mode 100644 index 00000000..88bbe84c --- /dev/null +++ b/alibaba-wanx/demo.svml @@ -0,0 +1,9 @@ + + + + + + + A serene mountain lake at sunrise, soft golden light, cinematic wide shot. + + diff --git a/alibaba-wanx/demo.svrun b/alibaba-wanx/demo.svrun new file mode 100644 index 00000000..054bf125 --- /dev/null +++ b/alibaba-wanx/demo.svrun @@ -0,0 +1,6 @@ + + + + + + diff --git a/alibaba-wanx/hypit.runtime.json b/alibaba-wanx/hypit.runtime.json new file mode 100644 index 00000000..6e28ae3c --- /dev/null +++ b/alibaba-wanx/hypit.runtime.json @@ -0,0 +1,17 @@ +{ + "format": "hypit.runtime-local@1", + "dataRoot": ".hypit/execution", + "credentials": { "os": { "use": "@hypit/credential-store-os" } }, + "endpoints": { + "wanx.personal": { + "use": "@example/provider-wanx", + "config": { + "baseUrl": "https://dashscope.aliyuncs.com", + "apiKey": { "store": "os", "key": "wanx.personal" }, + "model": "wan3.0-video", + "pollIntervalMs": 5000 + } + } + }, + "bindings": { "@hypit/alibaba-video@1#alibaba-qwen-vvg": "wanx.personal" } +} diff --git a/packages/alibaba-video/README.md b/packages/alibaba-video/README.md new file mode 100644 index 00000000..c82babb3 --- /dev/null +++ b/packages/alibaba-video/README.md @@ -0,0 +1,99 @@ +# @hypit/alibaba-video + +Alibaba Video Generation Model Support for Hypit + +## Overview + +This package provides integration with Alibaba's video generation models, specifically the **Qwen Video Generation (Qwen VVG)** model. It enables AI-powered video generation with text prompts and optional reference images. + +## Features + +- **Text-to-Video Generation**: Generate videos from text prompts +- **Reference-Based Generation**: Use reference images to guide the generation style +- **Configurable Resolution**: Support for 480p, 720p, and 1080p output +- **Multiple Aspect Ratios**: Support for 1:1, 16:9, 9:16, 4:3, 3:4 formats +- **Variable Duration**: Generate videos with 5-10 second duration + +## Supported Models + +### Alibaba Qwen Video Generation (alibaba-qwen-vvg) + +- **Input**: Text prompt (max 200 characters) +- **Resolution**: 480p, 720p, 1080p +- **Duration**: 5-10 seconds +- **Aspect Ratios**: 1:1, 16:9, 9:16, 4:3, 3:4 +- **Optional Reference**: Image for style guidance + +## Usage + +### Basic Text-to-Video + +```markup + +``` + +### Video Generation with Reference Image + +```markup + +``` + +## Configuration + +### Prompt +- Maximum 200 characters +- Should describe the desired video content clearly + +### Duration +- Range: 5-10 seconds +- Must be a whole number + +### Resolution +- `480p`: Lower quality, faster generation +- `720p`: Standard quality (default) +- `1080p`: High quality, longer generation time + +### Aspect Ratio +- `16:9`: Landscape (default) +- `9:16`: Portrait/vertical +- `1:1`: Square +- `4:3`: Standard +- `3:4`: Vertical standard + +## Architecture + +The package follows Hypit's standard model integration pattern: + +- **Generation Ports**: Defines input/output structure +- **Markup Surfaces**: Provides XML/markup interface for video generation +- **Fragment Builders**: Creates generation fragments with proper request sealing +- **Surface Decoders**: Converts markup into generation requests + +## Dependencies + +- `@hypit/artifact`: Artifact type definitions +- `@hypit/generation`: Generation request/port infrastructure +- `@hypit/model-kit`: Model module definition utilities +- `@hypit/markup`: Markup surface definitions +- `@hypit/elaborator`: Fragment building and activation context +- `@hypit/text`: Text type definitions +- `@hypit/protocol`: Core protocol definitions + +## License + +See LICENSE file in the repository root. diff --git a/packages/alibaba-video/package.json b/packages/alibaba-video/package.json new file mode 100644 index 00000000..b968bfb7 --- /dev/null +++ b/packages/alibaba-video/package.json @@ -0,0 +1,22 @@ +{ + "name": "@hypit/alibaba-video", + "version": "0.0.0-dev", + "license": "SEE LICENSE IN LICENSE", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "hypit": { + "activation": "./src/activation.ts" + }, + "dependencies": { + "@hypit/artifact": "workspace:*", + "@hypit/elaborator": "workspace:*", + "@hypit/generation": "workspace:*", + "@hypit/model-kit": "workspace:*", + "@hypit/protocol": "workspace:*", + "@hypit/text": "workspace:*", + "@hypit/markup": "workspace:*" + } +} diff --git a/packages/alibaba-video/src/activation.ts b/packages/alibaba-video/src/activation.ts new file mode 100644 index 00000000..78b68a4e --- /dev/null +++ b/packages/alibaba-video/src/activation.ts @@ -0,0 +1,46 @@ +/** + * Alibaba Video Generation Model Activation + * + * Registers the model and its markup surfaces with the Hypit runtime. + * + * NOTE: the upstream `feature/alibaba-video` branch shipped this file as a bare + * `export function activate(context) { ... }`, but Hypit 0.1.9's package loader + * (`packages/package-loader-node/src/loader.ts`, `importContribution`) requires the + * activation module's DEFAULT export to be a `hypit.node-package@1` contribution that + * declares `modules`, `components` and `hostFacets`. A plain `activate` function is + * rejected ("activation has no default package export" / "unsupported package format"). + * This rewrite mirrors the proven `@hypit/seedance` model package so the model loads. + */ + +import { createMarkupSurfaceHostFacet } from "@hypit/markup"; + +import { + decodeAlibabaVideoReferenceVideoSurface, + decodeAlibabaVideoTextVideoSurface, + alibabaVideoComponent, + alibabaVideoDefinition, + alibabaVideoManifest, + alibabaVideoModuleRef, + alibabaVideoMarkupSurfaces, +} from "./index.js"; + +export const hypitPackage = { + format: "hypit.node-package@1" as const, + modules: [{ manifest: alibabaVideoManifest }], + components: [alibabaVideoComponent], + hostFacets: [ + alibabaVideoDefinition.hostFacet, + createMarkupSurfaceHostFacet({ + module: alibabaVideoModuleRef, + declaration: alibabaVideoMarkupSurfaces.find((item) => item.name === "text-video")!, + handler: decodeAlibabaVideoTextVideoSurface, + }), + createMarkupSurfaceHostFacet({ + module: alibabaVideoModuleRef, + declaration: alibabaVideoMarkupSurfaces.find((item) => item.name === "reference-video")!, + handler: decodeAlibabaVideoReferenceVideoSurface, + }), + ], +}; + +export default hypitPackage; diff --git a/packages/alibaba-video/src/fragment.ts b/packages/alibaba-video/src/fragment.ts new file mode 100644 index 00000000..90576bad --- /dev/null +++ b/packages/alibaba-video/src/fragment.ts @@ -0,0 +1,23 @@ +/** + * Alibaba Video Generation Fragment Builder + * Mirrors the current Distribution API (same shape as @hypit/seedance). + */ + +import { createExactModelPrimaryGenerationFragment } from "@hypit/model-kit"; +import type { + ExactModelEndpoint, + ExactModelMediaInput, + ExactModelTextInput, +} from "@hypit/model-kit"; + +/** + * Builds the primary generation fragment for an Alibaba video endpoint. + * `mediaInputs` carries reference-image bindings; `textInputs` carries the prompt. + */ +export function createAlibabaVideoAssembledGenerationFragment( + endpoint: ExactModelEndpoint, + mediaInputs: readonly ExactModelMediaInput[] = [], + textInputs: readonly ExactModelTextInput[] = [], +) { + return createExactModelPrimaryGenerationFragment(endpoint, mediaInputs, textInputs); +} diff --git a/packages/alibaba-video/src/index.ts b/packages/alibaba-video/src/index.ts new file mode 100644 index 00000000..d82abef5 --- /dev/null +++ b/packages/alibaba-video/src/index.ts @@ -0,0 +1,250 @@ +import { artifactTypes } from "@hypit/artifact"; +import { + sealGenerationPortRequest, + sealGenerationPortTable, +} from "@hypit/generation"; +import type { + GenerationPortTable, + GenerationPortValue, + GenerationRequest, +} from "@hypit/generation"; +import type { + SurfaceAttributeVocabulary, + SurfacePortVocabulary, + SurfaceVocabulary, +} from "@hypit/markup"; +import { defineExactModelModule } from "@hypit/model-kit"; +import { textTypes } from "@hypit/text"; + +/** + * Alibaba Video Generation Models + * Supports: + * - alibaba-qwen-vvg: Qwen Video Generation model + */ +export const alibabaVideoModuleRef = { + name: "@hypit/alibaba-video", + version: "1", +} as const; + +export const alibabaVideoModels = [ + "alibaba-qwen-vvg", +] as const; +export type AlibabaVideoModel = typeof alibabaVideoModels[number]; + +const ASPECT_RATIOS = [ + "1:1", + "16:9", + "9:16", + "4:3", + "3:4", +] as const; + +const RESOLUTIONS = [ + "480p", + "720p", + "1080p", +] as const; + +/** + * Alibaba Video Generation model ports configuration. + * Defines the input/output structure for video generation requests. + */ +function alibabaVideoPortTable( + model: AlibabaVideoModel +): GenerationPortTable { + return sealGenerationPortTable({ + model, + result: "video", + ports: [ + { + name: "prompt", + value: { kind: "text", maxChars: 200 }, + minItems: 1, + maxItems: 1, + }, + { + name: "duration", + value: { kind: "number", integer: true, minimum: 5, maximum: 10 }, + minItems: 1, + maxItems: 1, + }, + { + name: "resolution", + value: { + kind: "enum", + values: [...RESOLUTIONS], + }, + minItems: 0, + maxItems: 1, + }, + { + name: "aspectRatio", + value: { kind: "enum", values: [...ASPECT_RATIOS] }, + minItems: 0, + maxItems: 1, + }, + { + name: "referenceImage", + value: { kind: "media", accepts: ["image"] }, + minItems: 0, + maxItems: 1, + }, + ], + requires: [], + }); +} + +export const alibabaVideoPorts: Readonly< + Record +> = { + "alibaba-qwen-vvg": alibabaVideoPortTable("alibaba-qwen-vvg"), +}; + +export type AlibabaVideoPortMap = Readonly< + Record +>; + +export function sealAlibabaVideoRequest( + model: AlibabaVideoModel, + ports: AlibabaVideoPortMap +): GenerationRequest { + return sealGenerationPortRequest(alibabaVideoPorts[model], ports); +} + +const alibabaVideoBaseDefinition = defineExactModelModule({ + module: alibabaVideoModuleRef, + endpoints: [ + { + key: "qwen-vvg", + requestTypeName: "AlibabaQwenVVGRequest", + producerName: "request-alibaba-qwen-vvg", + ports: alibabaVideoPorts["alibaba-qwen-vvg"], + }, + ], +}); + +export const alibabaVideoEndpoints = alibabaVideoBaseDefinition.endpoints; +export const alibabaVideoComponent = alibabaVideoBaseDefinition.component; + +const alibabaVideoCommonAttributes: readonly SurfaceAttributeVocabulary[] = [ + { + name: "id", + kind: "identifier", + required: true, + summary: "Names this generation and prefixes the binding it publishes.", + }, + { + name: "model", + kind: "literal", + required: true, + summary: "Chooses the exact Alibaba video model that renders the video.", + values: ["qwen-vvg", "alibaba-qwen-vvg"], + }, + { + name: "prompt", + kind: "reference", + required: true, + summary: "The Text description for the video generation (max 200 characters).", + accepts: [textTypes.text], + }, + { + name: "duration", + kind: "literal", + required: true, + summary: "Sets the length of the video in seconds (5-10 seconds).", + }, + { + name: "resolution", + kind: "literal", + required: false, + summary: "Chooses the output resolution of the video.", + values: [...RESOLUTIONS], + }, + { + name: "aspect-ratio", + kind: "literal", + required: false, + summary: "Chooses the aspect ratio of the generated video.", + values: [...ASPECT_RATIOS], + }, +]; + +const alibabaVideoPort: readonly SurfacePortVocabulary[] = [ + { + name: "video", + type: artifactTypes.blob, + summary: "The generated video Artifact.", + }, +]; + +const alibabaVideoSettingNotes: readonly string[] = [ + "`duration` must be between 5 and 10 seconds.", + "`resolution` defaults to `720p`.", + "`aspect-ratio` defaults to `16:9`.", + "Prompt text is limited to 200 characters maximum.", +]; + +export const alibabaVideoMarkupSurfaces = [ + { + name: "text-video", + tag: "TextVideo", + mode: "structured", + outputs: Object.values(alibabaVideoEndpoints).flatMap((endpoint) => [ + endpoint.draftType, + ...Object.values(endpoint.mediaBindings).map((binding) => binding.type), + ]), + vocabulary: { + summary: + "Generates one video with Alibaba Qwen video model from a Text prompt alone.", + attributes: alibabaVideoCommonAttributes, + ports: alibabaVideoPort, + example: + '', + notes: [ + ...alibabaVideoSettingNotes, + "The element accepts no children and no text content.", + ], + } as SurfaceVocabulary, + }, + { + name: "reference-video", + tag: "ReferenceVideo", + mode: "structured", + outputs: Object.values(alibabaVideoEndpoints).flatMap((endpoint) => [ + endpoint.draftType, + ...Object.values(endpoint.mediaBindings).map((binding) => binding.type), + ]), + vocabulary: { + summary: + "Generates one video with Alibaba Qwen model from a Text prompt and an optional reference image.", + attributes: [ + ...alibabaVideoCommonAttributes, + { + name: "reference-image", + kind: "reference", + required: false, + summary: + "The image Artifact to use as a style or content reference for video generation.", + accepts: [artifactTypes.blob], + }, + ], + ports: alibabaVideoPort, + example: + '', + notes: [ + ...alibabaVideoSettingNotes, + "Reference image is optional and provides style guidance.", + "The element accepts no children and no text content.", + ], + } as SurfaceVocabulary, + }, +] as const; + +export const alibabaVideoManifest = alibabaVideoBaseDefinition.manifest; +export const alibabaVideoDefinition = alibabaVideoBaseDefinition; + +export { createAlibabaVideoAssembledGenerationFragment } from "./fragment.js"; +export { + decodeAlibabaVideoTextVideoSurface, + decodeAlibabaVideoReferenceVideoSurface, +} from "./surface.js"; diff --git a/packages/alibaba-video/src/surface.ts b/packages/alibaba-video/src/surface.ts new file mode 100644 index 00000000..2c343cfe --- /dev/null +++ b/packages/alibaba-video/src/surface.ts @@ -0,0 +1,238 @@ +/** + * Alibaba Video Generation Surface Decoders + * Decodes markup surfaces into generation fragments using the current + * Distribution StructuredSurfaceHandler contract (same as @hypit/seedance). + */ + +import { artifactTypes } from "@hypit/artifact"; +import { + generationPort, + sealGenerationMediaBinding, + sealGenerationRequestDraft, +} from "@hypit/generation"; +import type { GenerationMediaPort, GenerationMediaRole, GenerationPortTable } from "@hypit/generation"; +import { exactModelMediaInputNames, exactModelTextInputName } from "@hypit/model-kit"; +import type { ExactModelEndpoint, ExactModelMediaInput } from "@hypit/model-kit"; +import type { CanonicalValue } from "@hypit/protocol"; +import { textTypes, verifyText } from "@hypit/text"; +import type { + MarkupAttributeValue, + StructuredElement, + StructuredSurfaceHandler, + SurfaceResolvedReference, +} from "@hypit/markup"; + +import { createAlibabaVideoAssembledGenerationFragment } from "./fragment.js"; +import { alibabaVideoEndpoints, alibabaVideoPorts } from "./index.js"; +import type { AlibabaVideoModel, AlibabaVideoPortMap } from "./index.js"; + +type MediaInput = { + readonly port: "referenceImage"; + readonly role: GenerationMediaRole; + readonly source: SurfaceResolvedReference; +}; + +function stringAttribute(element: StructuredElement, name: string): string { + const value = element.attributes[name]; + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${element.name}.${name} must be a non-empty string`); + } + return value.trim(); +} + +function optionalStringAttribute(element: StructuredElement, name: string): string | undefined { + const value = element.attributes[name]; + if (value === undefined) return undefined; + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${element.name}.${name} must be a non-empty string`); + } + return value.trim(); +} + +function referencePath(element: StructuredElement, name: string): string { + const value: MarkupAttributeValue | undefined = element.attributes[name]; + if (typeof value !== "object" || value.kind !== "reference" || value.path.length === 0) { + throw new Error(`${element.name}.${name} must be a whole-value reference`); + } + return value.path; +} + +function resolved( + element: StructuredElement, + name: string, + resolveReference: (path: string) => SurfaceResolvedReference | undefined, +): SurfaceResolvedReference { + const path = referencePath(element, name); + const value = resolveReference(path); + if (value === undefined) throw new Error(`${element.name}.${name} cannot resolve ${path}`); + return value; +} + +function sameType( + left: SurfaceResolvedReference["type"], + right: SurfaceResolvedReference["type"], +): boolean { + return left.module.name === right.module.name + && left.module.version === right.module.version + && left.name === right.name; +} + +function prompt(reference: SurfaceResolvedReference, subject: string): void { + if (!sameType(reference.type, textTypes.text)) throw new Error(`${subject} must reference Text`); + const value = reference.record?.value; + if (value !== undefined) { + if (value.kind !== "inline") throw new Error(`${subject} has an invalid authored Text value`); + verifyText(value.value); + } +} + +function mediaReference( + reference: SurfaceResolvedReference, + role: GenerationMediaRole, + subject: string, +): SurfaceResolvedReference { + if (!sameType(reference.type, artifactTypes.blob)) throw new Error(`${subject} must reference a BlobArtifact`); + const value = reference.record?.value; + if (value !== undefined && (value.kind !== "blob" || !value.mediaType.startsWith(`${role}/`))) { + throw new Error(`${subject} must reference ${role} media`); + } + return reference; +} + +function enumeratedPort(table: GenerationPortTable, name: string): readonly (string | number)[] { + const port = generationPort(table, name); + if (port.value.kind !== "enum") throw new Error(`alibaba-qwen-vvg port ${name} is not enumerated`); + return port.value.values; +} + +function generationSettings(element: StructuredElement, model: AlibabaVideoModel): AlibabaVideoPortMap { + const table = alibabaVideoPorts[model]; + const durationSec = Number(optionalStringAttribute(element, "duration") ?? "5"); + if (!Number.isSafeInteger(durationSec)) throw new Error(`${element.name}.duration must be an integer`); + const resolutions = enumeratedPort(table, "resolution"); + const resolution = optionalStringAttribute(element, "resolution") ?? "720p"; + if (!resolutions.includes(resolution)) { + throw new Error(`${element.name}.resolution must be ${resolutions.join(", ")}`); + } + const aspectRatios = enumeratedPort(table, "aspectRatio"); + const aspectRatio = optionalStringAttribute(element, "aspect-ratio") ?? "16:9"; + if (!aspectRatios.includes(aspectRatio)) { + throw new Error(`${element.name}.aspect-ratio must be ${aspectRatios.join(", ")}`); + } + return { + duration: [durationSec], + resolution: [resolution], + aspectRatio: [aspectRatio], + } as AlibabaVideoPortMap; +} + +function assembleMedia( + id: string, + endpoint: ExactModelEndpoint, + values: readonly MediaInput[], + range: StructuredElement["range"], +) { + const records: Array<{ + readonly id: string; + readonly type: SurfaceResolvedReference["type"]; + readonly value: { readonly kind: "inline"; readonly value: CanonicalValue }; + readonly range: StructuredElement["range"]; + }> = []; + const inputs: Record = {}; + const mediaInputs = values.map((value, index): ExactModelMediaInput => { + const name = `media-${String(index + 1).padStart(4, "0")}`; + const binding = endpoint.mediaBindings[value.port]; + if (binding === undefined) throw new Error(`${endpoint.ports.model} has no media port ${value.port}`); + const port = generationPort(endpoint.ports, value.port); + if (port.value.kind !== "media") throw new Error(`${endpoint.ports.model} port ${value.port} is not media`); + const bindingId = `${id}.${name}.binding`; + records.push({ + id: bindingId, + type: binding.type, + value: { + kind: "inline", + value: sealGenerationMediaBinding(port as GenerationMediaPort, { role: value.role }) as unknown as CanonicalValue, + }, + range, + }); + const names = exactModelMediaInputNames(name); + inputs[names.binding] = { kind: "record", id: bindingId }; + inputs[names.artifact] = value.source.ref; + return { name, port: value.port }; + }); + return { mediaInputs, records, inputs }; +} + +function generationOutput(args: { + readonly element: StructuredElement; + readonly endpoint: ExactModelEndpoint; + readonly model: AlibabaVideoModel; + readonly promptSource: SurfaceResolvedReference; + readonly media: readonly MediaInput[]; + readonly resolveReference: (path: string) => SurfaceResolvedReference | undefined; +}) { + const { element, endpoint, model, promptSource } = args; + const id = stringAttribute(element, "id"); + const assembled = args.media.length > 0 + ? assembleMedia(id, endpoint, args.media, element.range) + : { mediaInputs: [] as ExactModelMediaInput[], records: [], inputs: {} }; + const draft = sealGenerationRequestDraft(alibabaVideoPorts[model], generationSettings(element, model)); + const fragment = createAlibabaVideoAssembledGenerationFragment( + endpoint, + assembled.mediaInputs, + [{ name: "prompt", port: "prompt" }], + ); + const draftId = `${id}.draft`; + return { + records: [{ + id: draftId, + type: endpoint.draftType, + value: { kind: "inline" as const, value: draft as unknown as CanonicalValue }, + range: element.range, + }, ...assembled.records], + components: [{ + id, + fragment: fragment.id, + inputs: { + draft: { kind: "record" as const, id: draftId }, + [exactModelTextInputName("prompt")]: promptSource.ref, + ...assembled.inputs, + }, + outputs: { video: `${id}.video` }, + range: element.range, + }], + fragments: [fragment], + }; +} + +function modelSelection(element: StructuredElement): { endpoint: ExactModelEndpoint; model: AlibabaVideoModel } { + const requested = stringAttribute(element, "model"); + if (requested === "qwen-vvg" || requested === "alibaba-qwen-vvg") { + const endpoint = alibabaVideoEndpoints["qwen-vvg"]; + if (!endpoint) throw new Error(`${element.name}.model qwen-vvg endpoint is not defined`); + return { endpoint, model: "alibaba-qwen-vvg" }; + } + throw new Error(`${element.name}.model must be qwen-vvg or alibaba-qwen-vvg`); +} + +export const decodeAlibabaVideoTextVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => { + const selected = modelSelection(element); + const promptSource = resolved(element, "prompt", resolveReference); + prompt(promptSource, `${element.name}.prompt`); + return generationOutput({ element, ...selected, promptSource, media: [], resolveReference }); +}; + +export const decodeAlibabaVideoReferenceVideoSurface: StructuredSurfaceHandler = ({ element, resolveReference }) => { + const selected = modelSelection(element); + const promptSource = resolved(element, "prompt", resolveReference); + prompt(promptSource, `${element.name}.prompt`); + const media: MediaInput[] = []; + if (element.attributes["reference-image"] !== undefined) { + media.push({ + port: "referenceImage", + role: "image", + source: mediaReference(resolved(element, "reference-image", resolveReference), "image", `${element.name}.reference-image`), + }); + } + return generationOutput({ element, ...selected, promptSource, media, resolveReference }); +}; diff --git a/packages/provider-wanx/package.json b/packages/provider-wanx/package.json new file mode 100644 index 00000000..d6245f61 --- /dev/null +++ b/packages/provider-wanx/package.json @@ -0,0 +1,11 @@ +{ + "name": "@example/provider-wanx", + "version": "0.0.0", + "private": true, + "type": "module", + "hypit": { "activation": "./src/activation.ts" }, + "exports": { ".": "./src/index.ts" }, + "dependencies": { + "@hypit/hypit": "workspace:*" + } +} diff --git a/packages/provider-wanx/src/activation.ts b/packages/provider-wanx/src/activation.ts new file mode 100644 index 00000000..a67fccb1 --- /dev/null +++ b/packages/provider-wanx/src/activation.ts @@ -0,0 +1,31 @@ +import { + createRuntimeEndpointAdapterFacet, runtimeConfigCredentialRef, runtimeConfigExact, + runtimeConfigObject, runtimeConfigPositiveInteger, runtimeConfigString, +} from "@hypit/hypit/runtime-kit"; +import { createWanxProvider, providerModule } from "./provider.js"; + +export default { + format: "hypit.node-package@1" as const, + hostFacets: [createRuntimeEndpointAdapterFacet({ + use: providerModule.name, + activate(context) { + const config = runtimeConfigObject(context.config, "Wanx video service"); + runtimeConfigExact(config, ["baseUrl", "apiKey", "model", "concurrency", "pollIntervalMs"], "Wanx video service"); + const baseUrl = runtimeConfigString(config.baseUrl, "Wanx baseUrl") ?? "https://dashscope.aliyuncs.com"; + const apiKey = runtimeConfigCredentialRef(config.apiKey, "Wanx apiKey"); + const model = runtimeConfigString(config.model, "Wanx model") ?? "wan3.0-video"; + if (!apiKey || !context.pool) throw new Error("Wanx video service requires apiKey and pool"); + return { + endpoint: createWanxProvider({ + instance: context.instance, + pool: context.pool, + baseUrl, + apiKey, + model, + concurrency: runtimeConfigPositiveInteger(config.concurrency, "concurrency") ?? 1, + pollIntervalMs: runtimeConfigPositiveInteger(config.pollIntervalMs, "pollIntervalMs") ?? 5_000, + }), + }; + }, + })], +}; diff --git a/packages/provider-wanx/src/index.ts b/packages/provider-wanx/src/index.ts new file mode 100644 index 00000000..5f3b6afd --- /dev/null +++ b/packages/provider-wanx/src/index.ts @@ -0,0 +1 @@ +export { createWanxProvider, providerModule } from "./provider.js"; diff --git a/packages/provider-wanx/src/provider.ts b/packages/provider-wanx/src/provider.ts new file mode 100644 index 00000000..a57410bd --- /dev/null +++ b/packages/provider-wanx/src/provider.ts @@ -0,0 +1,193 @@ +import { canonicalize, defineEndpointPackage, wakeAfter } from "@hypit/hypit/endpoint-kit"; +import type { AsyncEndpoint, BlobRef, CredentialRef, EndpointRequest, ResourceStore } from "@hypit/hypit/endpoint-kit"; +import { generationTypes, sealGeneratedVideoSet } from "@hypit/hypit/generation"; +import type { GenerationPortValue } from "@hypit/hypit/generation"; + +/** + * Capability implemented by this Provider. + * + * Matches the video Model declared by the user's fork branch `feature/alibaba-video` + * (package `@hypit/alibaba-video`, model / capability `alibaba-qwen-vvg`). + * That branch ships the Model + SVML surfaces (``); this Provider + * is the missing Endpoint that actually calls DashScope. No name guessing needed. + */ +export const CAPABILITY_NAME = "alibaba-qwen-vvg"; +export const capability = { + module: { name: "@hypit/alibaba-video", version: "1" }, + name: CAPABILITY_NAME, +} as const; + +export const providerModule = { name: "@example/provider-wanx", version: "1" } as const; + +// 阿里云控制台里唯一可用的 video 模型（用户实测），t2v / i2v 暂统一用同一个。 +const DEFAULT_MODEL_T2V = "wan3.0-video"; +const DEFAULT_MODEL_I2V = "wan3.0-video"; +const DASHSCOPE_BASE = "https://dashscope.aliyuncs.com"; + +/** A request's ports arrive as a plain map of port name -> values. */ +type PortMap = Record; + +function text(value: unknown): string { + if (typeof value !== "string" || value.length === 0) throw new Error("Expected nonempty text"); + return value; +} +function firstText(ports: PortMap, name: string): string | undefined { + const v = ports[name]?.[0]; + return typeof v === "string" ? v : undefined; +} +function firstNumber(ports: PortMap, name: string): number | undefined { + const v = ports[name]?.[0]; + return typeof v === "number" ? v : undefined; +} +/** A media port value carries `{ role, artifact: BlobRef, fields? }`. */ +function isMediaValue(v: unknown): v is { role: string; artifact: BlobRef } { + return ( + typeof v === "object" && v !== null && + "artifact" in v! && (v as any).artifact?.kind === "blob" && + typeof (v as any).artifact.resource === "string" + ); +} +/** Resolve the optional `referenceImage` port into a base64 data URI. */ +async function resolveReferenceImage(ports: PortMap, resources: ResourceStore): Promise { + const v = ports["referenceImage"]?.[0]; + if (isMediaValue(v) && (v.role === "image" || v.role === "video")) { + const bytes = await resources.get(v.artifact.resource); + if (bytes === undefined) throw new Error("Reference image is unavailable in the resource store"); + let bin = ""; + const chunk = 0x8000; + const u = new Uint8Array(bytes); + for (let i = 0; i < u.length; i += chunk) bin += String.fromCharCode(...u.subarray(i, i + chunk)); + return `data:${v.artifact.mediaType || "image/png"};base64,${btoa(bin)}`; + } + return undefined; +} + +/** Wanx only accepts a few sizes; pick the supported one for the requested aspect ratio. */ +function wanxSize(aspectRatio?: string): string { + switch (aspectRatio) { + case "9:16": return "720*1280"; + case "1:1": return "1024*1024"; + case "4:3": return "1280*720"; + case "3:4": return "720*1280"; + case "16:9": + default: return "1280*720"; + } +} +/** Wanx durations are 5 or 10 (integers); clamp the branch's 5–10 range. */ +function wanxDuration(raw?: number): number { + const n = Math.round(raw ?? 5); + return n >= 8 ? 10 : 5; +} + +function support(request: EndpointRequest) { + const ports = (request.constraints as unknown as { ports?: PortMap }).ports ?? {}; + if (!firstText(ports, "prompt")) { + return { status: "unsupported" as const, reason: "This Provider requires a `prompt` text port" }; + } + return { status: "supported" as const }; +} + +export function createWanxProvider(options: { + instance: string; pool: string; baseUrl?: string; apiKey: CredentialRef; + model?: string; modelI2V?: string; concurrency?: number; pollIntervalMs?: number; fetch?: typeof globalThis.fetch; +}) { + const base = (options.baseUrl ?? DASHSCOPE_BASE).replace(/\/$/u, ""); + const fetcher = options.fetch ?? globalThis.fetch; + const modelT2V = options.model ?? DEFAULT_MODEL_T2V; + const modelI2V = options.modelI2V ?? DEFAULT_MODEL_I2V; + const interval = options.pollIntervalMs ?? 5_000; + const key = (credentials: Readonly>) => text(credentials.apiKey?.secret); + + async function wanxJson(path: string, secret: string, init: RequestInit = {}) { + const response = await fetcher(`${base}${path}`, { + ...init, + headers: { + ...init.headers, + authorization: `Bearer ${secret}`, + "content-type": "application/json", + }, + signal: AbortSignal.timeout(60_000), + }); + const body = await response.text(); + if (!response.ok) { + throw new Error(`Wanx ${init.method ?? "GET"} ${path} returned HTTP ${response.status}: ${body.slice(0, 500)}`); + } + try { return JSON.parse(body); } catch { return {}; } + } + + const endpoint: AsyncEndpoint = { + async start(context) { + const supported = support(context.need); + if (supported.status === "unsupported") throw new Error(supported.reason); + const secret = key(context.credentials); + const ports = (context.need.constraints as unknown as { ports: PortMap }).ports; + + const prompt = firstText(ports, "prompt"); + if (!prompt) throw new Error("Missing prompt"); + const imageUri = await resolveReferenceImage(ports, context.resources); + const model = imageUri ? modelI2V : modelT2V; + + const size = wanxSize(firstText(ports, "aspectRatio")); + const duration = wanxDuration(firstNumber(ports, "duration")); + + const input: Record = { prompt }; + if (imageUri) input.image_url = imageUri; // image-to-video when a reference is supplied + + await context.reportProgress?.({ phase: `Submitting Wanx ${model} request` }); + const task = await wanxJson("/api/v1/services/aigc/video-generation/video-synthesis", secret, { + method: "POST", + headers: { "X-DashScope-Async": "enable" }, + body: JSON.stringify({ model, input, parameters: { size, duration } }), + }); + const taskId = text(task?.output?.task_id ?? task?.task_id); + const handle = { taskId }; + await context.checkpoint?.({ handle, receipt: { id: taskId } }); + return { ...wakeAfter(handle, interval), receipt: { id: taskId } }; + }, + + async poll(context) { + const taskId = text((context.handle as { taskId?: string }).taskId); + const secret = key(context.credentials); + const task = await wanxJson(`/api/v1/tasks/${encodeURIComponent(taskId)}`, secret); + const status: string = task?.output?.task_status ?? task?.task_status ?? "PENDING"; + if (status === "PENDING" || status === "RUNNING") { + return wakeAfter({ taskId }, interval, Date.now(), { phase: status }); + } + if (status === "FAILED") { + const msg = String(task?.output?.message ?? task?.message ?? "Wanx task failed"); + return { status: "failed" as const, receipt: { id: taskId }, failure: { code: "WANX_FAILED", message: `Wanx task ${taskId} failed: ${msg}` } }; + } + if (status !== "SUCCEEDED") throw new Error(`Wanx returned unknown task status: ${status}`); + const videoUrl: string | undefined = + task?.output?.video_url ?? task?.output?.results?.[0]?.url ?? task?.output?.url; + if (!videoUrl) throw new Error(`Wanx task ${taskId} succeeded but returned no video URL`); + return { status: "ready" as const, handle: { taskId, url: videoUrl } }; + }, + + async collect(context) { + const url = text((context.handle as { url?: string }).url); + await context.reportProgress?.({ phase: "Receiving generated video" }); + const response = await fetcher(url, { signal: AbortSignal.timeout(120_000) }); + if (!response.ok) throw new Error(`Video download returned HTTP ${response.status}`); + const mediaType = response.headers.get("content-type")?.split(";")[0]?.trim() ?? "video/mp4"; + if (!mediaType.startsWith("video/")) throw new Error(`Wanx returned a non-video result: ${mediaType}`); + const artifact = await context.resources.put(new Uint8Array(await response.arrayBuffer()), mediaType); + return { status: "completed" as const, result: { value: { + kind: "inline", value: canonicalize(sealGeneratedVideoSet({ videos: [artifact] })), + } } }; + }, + }; + + return defineEndpointPackage({ + module: providerModule, + facet: "videos", + instance: options.instance, + pool: options.pool, + credentials: { apiKey: options.apiKey }, + credentialInputs: { apiKey: { label: "DashScope / Alibaba API key (DASHSCOPE_API_KEY)" } }, + defaultConcurrency: options.concurrency ?? 1, + actionLimits: { submit: { concurrency: 1 }, poll: { concurrency: 4 }, collect: { concurrency: 1 } }, + pricing: { kind: "page", url: "https://help.aliyun.com/zh/model-studio/wanx-video/" }, + capabilities: [{ capability, returns: generationTypes.videoSet, lifecycle: "asynchronous", supports: support, endpoint }], + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7879ee08..3266f03e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -406,6 +406,30 @@ importers: specifier: 5.9.3 version: 5.9.3 + packages/alibaba-video: + dependencies: + '@hypit/artifact': + specifier: workspace:* + version: link:../artifact + '@hypit/elaborator': + specifier: workspace:* + version: link:../elaborator + '@hypit/generation': + specifier: workspace:* + version: link:../generation + '@hypit/markup': + specifier: workspace:* + version: link:../markup + '@hypit/model-kit': + specifier: workspace:* + version: link:../model-kit + '@hypit/protocol': + specifier: workspace:* + version: link:../protocol + '@hypit/text': + specifier: workspace:* + version: link:../text + packages/artifact: dependencies: '@hypit/protocol': @@ -2318,6 +2342,12 @@ importers: specifier: workspace:* version: link:../driver-node + packages/provider-wanx: + dependencies: + '@hypit/hypit': + specifier: workspace:* + version: link:../.. + packages/provider-whisperx-local: dependencies: '@hypit/endpoint-kit':