From b2a5b80e388d65e163dd61e3ff7754171689af1f Mon Sep 17 00:00:00 2001 From: Lucas Clark Date: Wed, 16 Sep 2026 01:57:57 +0800 Subject: [PATCH] feat: add Alibaba video generation model support\n\nAdd support for Alibaba Qwen Video Generation (VVG) model with the following features:\n- Text-to-video generation from prompts\n- Optional reference image for style guidance\n- Configurable resolution (480p, 720p, 1080p)\n- Multiple aspect ratios (1:1, 16:9, 9:16, 4:3, 3:4)\n- 5-10 second video duration\n\nImplements standard Hypit model integration pattern:\n- Generation ports for request/response structure\n- Markup surfaces for XML-based video generation\n- Fragment builders for generation request creation\n- Surface decoders for markup parsing\n- Activation context for model registration --- packages/alibaba-video/README.md | 99 +++++++++ packages/alibaba-video/package.json | 22 ++ packages/alibaba-video/src/activation.ts | 24 +++ packages/alibaba-video/src/fragment.ts | 38 ++++ packages/alibaba-video/src/index.ts | 255 +++++++++++++++++++++++ packages/alibaba-video/src/surface.ts | 41 ++++ 6 files changed, 479 insertions(+) create mode 100644 packages/alibaba-video/README.md create mode 100644 packages/alibaba-video/package.json create mode 100644 packages/alibaba-video/src/activation.ts create mode 100644 packages/alibaba-video/src/fragment.ts create mode 100644 packages/alibaba-video/src/index.ts create mode 100644 packages/alibaba-video/src/surface.ts diff --git a/packages/alibaba-video/README.md b/packages/alibaba-video/README.md new file mode 100644 index 00000000..c82babb3 --- /dev/null +++ b/packages/alibaba-video/README.md @@ -0,0 +1,99 @@ +# @hypit/alibaba-video + +Alibaba Video Generation Model Support for Hypit + +## Overview + +This package provides integration with Alibaba's video generation models, specifically the **Qwen Video Generation (Qwen VVG)** model. It enables AI-powered video generation with text prompts and optional reference images. + +## Features + +- **Text-to-Video Generation**: Generate videos from text prompts +- **Reference-Based Generation**: Use reference images to guide the generation style +- **Configurable Resolution**: Support for 480p, 720p, and 1080p output +- **Multiple Aspect Ratios**: Support for 1:1, 16:9, 9:16, 4:3, 3:4 formats +- **Variable Duration**: Generate videos with 5-10 second duration + +## Supported Models + +### Alibaba Qwen Video Generation (alibaba-qwen-vvg) + +- **Input**: Text prompt (max 200 characters) +- **Resolution**: 480p, 720p, 1080p +- **Duration**: 5-10 seconds +- **Aspect Ratios**: 1:1, 16:9, 9:16, 4:3, 3:4 +- **Optional Reference**: Image for style guidance + +## Usage + +### Basic Text-to-Video + +```markup + +``` + +### Video Generation with Reference Image + +```markup + +``` + +## Configuration + +### Prompt +- Maximum 200 characters +- Should describe the desired video content clearly + +### Duration +- Range: 5-10 seconds +- Must be a whole number + +### Resolution +- `480p`: Lower quality, faster generation +- `720p`: Standard quality (default) +- `1080p`: High quality, longer generation time + +### Aspect Ratio +- `16:9`: Landscape (default) +- `9:16`: Portrait/vertical +- `1:1`: Square +- `4:3`: Standard +- `3:4`: Vertical standard + +## Architecture + +The package follows Hypit's standard model integration pattern: + +- **Generation Ports**: Defines input/output structure +- **Markup Surfaces**: Provides XML/markup interface for video generation +- **Fragment Builders**: Creates generation fragments with proper request sealing +- **Surface Decoders**: Converts markup into generation requests + +## Dependencies + +- `@hypit/artifact`: Artifact type definitions +- `@hypit/generation`: Generation request/port infrastructure +- `@hypit/model-kit`: Model module definition utilities +- `@hypit/markup`: Markup surface definitions +- `@hypit/elaborator`: Fragment building and activation context +- `@hypit/text`: Text type definitions +- `@hypit/protocol`: Core protocol definitions + +## License + +See LICENSE file in the repository root. diff --git a/packages/alibaba-video/package.json b/packages/alibaba-video/package.json new file mode 100644 index 00000000..b968bfb7 --- /dev/null +++ b/packages/alibaba-video/package.json @@ -0,0 +1,22 @@ +{ + "name": "@hypit/alibaba-video", + "version": "0.0.0-dev", + "license": "SEE LICENSE IN LICENSE", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "hypit": { + "activation": "./src/activation.ts" + }, + "dependencies": { + "@hypit/artifact": "workspace:*", + "@hypit/elaborator": "workspace:*", + "@hypit/generation": "workspace:*", + "@hypit/model-kit": "workspace:*", + "@hypit/protocol": "workspace:*", + "@hypit/text": "workspace:*", + "@hypit/markup": "workspace:*" + } +} diff --git a/packages/alibaba-video/src/activation.ts b/packages/alibaba-video/src/activation.ts new file mode 100644 index 00000000..098605f3 --- /dev/null +++ b/packages/alibaba-video/src/activation.ts @@ -0,0 +1,24 @@ +/** + * Alibaba Video Generation Model Activation + * Registers the model and surfaces with the Hypit system + */ + +import type { ActivationContext } from "@hypit/elaborator"; +import { + alibabaVideoComponent, + alibabaVideoManifest, + alibabaVideoMarkupSurfaces, +} from "./index.js"; + +export function activate(context: ActivationContext) { + // Register the model component + context.component(alibabaVideoComponent); + + // Register the manifest + context.manifest(alibabaVideoManifest); + + // Register all markup surfaces + for (const surface of alibabaVideoMarkupSurfaces) { + context.surface(surface); + } +} diff --git a/packages/alibaba-video/src/fragment.ts b/packages/alibaba-video/src/fragment.ts new file mode 100644 index 00000000..46305571 --- /dev/null +++ b/packages/alibaba-video/src/fragment.ts @@ -0,0 +1,38 @@ +/** + * Alibaba Video Generation Fragment Builders + * Provides functions to create generation fragments for video generation requests + */ + +import type { AssembledGenerationFragment, GenerationFragment } from "@hypit/elaborator"; +import { + createAssembledGenerationFragment, + createGenerationFragment, +} from "@hypit/elaborator"; +import { + alibabaVideoEndpoints, + sealAlibabaVideoRequest, + type AlibabaVideoPortMap, +} from "./index.js"; + +/** + * Creates an assembled generation fragment for Alibaba video generation + * with resolved media bindings and draft type information + */ +export function createAlibabaVideoAssembledGenerationFragment( + ports: AlibabaVideoPortMap +): AssembledGenerationFragment { + const endpoint = alibabaVideoEndpoints.qwenVvg!; + const request = sealAlibabaVideoRequest("alibaba-qwen-vvg", ports); + return createAssembledGenerationFragment(endpoint, request); +} + +/** + * Creates a generation fragment for Alibaba video generation + */ +export function createAlibabaVideoGenerationFragment( + ports: AlibabaVideoPortMap +): GenerationFragment { + const endpoint = alibabaVideoEndpoints.qwenVvg!; + const request = sealAlibabaVideoRequest("alibaba-qwen-vvg", ports); + return createGenerationFragment(endpoint, request); +} diff --git a/packages/alibaba-video/src/index.ts b/packages/alibaba-video/src/index.ts new file mode 100644 index 00000000..034e568c --- /dev/null +++ b/packages/alibaba-video/src/index.ts @@ -0,0 +1,255 @@ +import { artifactTypes } from "@hypit/artifact"; +import { + sealGenerationPortRequest, + sealGenerationPortTable, +} from "@hypit/generation"; +import type { + GenerationPortTable, + GenerationPortValue, + GenerationRequest, +} from "@hypit/generation"; +import type { + SurfaceAttributeVocabulary, + SurfacePortVocabulary, + SurfaceVocabulary, +} from "@hypit/markup"; +import { defineExactModelModule } from "@hypit/model-kit"; +import { textTypes } from "@hypit/text"; + +/** + * Alibaba Video Generation Models + * Supports: + * - alibaba-qwen-vvg: Qwen Video Generation model + */ +export const alibabaVideoModuleRef = { + name: "@hypit/alibaba-video", + version: "1", +} as const; + +export const alibabaVideoModels = [ + "alibaba-qwen-vvg", +] as const; +export type AlibabaVideoModel = typeof alibabaVideoModels[number]; + +const ASPECT_RATIOS = [ + "1:1", + "16:9", + "9:16", + "4:3", + "3:4", +] as const; + +const RESOLUTIONS = [ + "480p", + "720p", + "1080p", +] as const; + +/** + * Alibaba Video Generation model ports configuration. + * Defines the input/output structure for video generation requests. + */ +function alibabaVideoPortTable( + model: AlibabaVideoModel +): GenerationPortTable { + return sealGenerationPortTable({ + model, + result: "video", + ports: [ + { + name: "prompt", + value: { kind: "text", maxChars: 200 }, + minItems: 1, + maxItems: 1, + }, + { + name: "duration", + value: { kind: "number", integer: true, minimum: 5, maximum: 10 }, + minItems: 1, + maxItems: 1, + }, + { + name: "resolution", + value: { + kind: "enum", + values: [...RESOLUTIONS], + }, + minItems: 0, + maxItems: 1, + }, + { + name: "aspectRatio", + value: { kind: "enum", values: [...ASPECT_RATIOS] }, + minItems: 0, + maxItems: 1, + }, + { + name: "referenceImage", + value: { kind: "media", accepts: ["image"] }, + minItems: 0, + maxItems: 1, + }, + ], + requires: [ + { kind: "atMostOneOf", ports: ["referenceImage"] }, + ], + }); +} + +export const alibabaVideoPorts: Readonly< + Record +> = { + "alibaba-qwen-vvg": alibabaVideoPortTable("alibaba-qwen-vvg"), +}; + +export type AlibabaVideoPortMap = Readonly< + Record +>; + +export function sealAlibabaVideoRequest( + model: AlibabaVideoModel, + ports: AlibabaVideoPortMap +): GenerationRequest { + return sealGenerationPortRequest(alibabaVideoPorts[model], ports); +} + +const alibabaVideoBaseDefinition = defineExactModelModule({ + module: alibabaVideoModuleRef, + endpoints: [ + { + key: "qwen-vvg", + requestTypeName: "AlibabaQwenVVGRequest", + producerName: "request-alibaba-qwen-vvg", + ports: alibabaVideoPorts["alibaba-qwen-vvg"], + }, + ], +}); + +export const alibabaVideoEndpoints = alibabaVideoBaseDefinition.endpoints; +export const alibabaVideoComponent = alibabaVideoBaseDefinition.component; + +const alibabaVideoCommonAttributes: readonly SurfaceAttributeVocabulary[] = [ + { + name: "id", + kind: "identifier", + required: true, + summary: "Names this generation and prefixes the binding it publishes.", + }, + { + name: "model", + kind: "literal", + required: true, + summary: "Chooses the exact Alibaba video model that renders the video.", + values: ["qwen-vvg", "alibaba-qwen-vvg"], + }, + { + name: "prompt", + kind: "reference", + required: true, + summary: "The Text description for the video generation (max 200 characters).", + accepts: [textTypes.text], + }, + { + name: "duration", + kind: "literal", + required: true, + summary: "Sets the length of the video in seconds (5-10 seconds).", + }, + { + name: "resolution", + kind: "literal", + required: false, + summary: "Chooses the output resolution of the video.", + values: [...RESOLUTIONS], + }, + { + name: "aspect-ratio", + kind: "literal", + required: false, + summary: "Chooses the aspect ratio of the generated video.", + values: [...ASPECT_RATIOS], + }, +]; + +const alibabaVideoPort: readonly SurfacePortVocabulary[] = [ + { + name: "video", + type: artifactTypes.blob, + summary: "The generated video Artifact.", + }, +]; + +const alibabaVideoSettingNotes: readonly string[] = [ + "`duration` must be between 5 and 10 seconds.", + "`resolution` defaults to `720p`.", + "`aspect-ratio` defaults to `16:9`.", + "Prompt text is limited to 200 characters maximum.", +]; + +export const alibabaVideoMarkupSurfaces = [ + { + name: "text-video", + tag: "TextVideo", + mode: "structured", + outputs: Object.values(alibabaVideoEndpoints).flatMap((endpoint) => [ + endpoint.draftType, + ...Object.values(endpoint.mediaBindings).map((binding) => binding.type), + ]), + vocabulary: { + summary: + "Generates one video with Alibaba Qwen video model from a Text prompt alone.", + attributes: alibabaVideoCommonAttributes, + ports: alibabaVideoPort, + example: + '', + notes: [ + ...alibabaVideoSettingNotes, + "The element accepts no children and no text content.", + ], + } as SurfaceVocabulary, + }, + { + name: "reference-video", + tag: "ReferenceVideo", + mode: "structured", + outputs: Object.values(alibabaVideoEndpoints).flatMap((endpoint) => [ + endpoint.draftType, + ...Object.values(endpoint.mediaBindings).map((binding) => binding.type), + ]), + vocabulary: { + summary: + "Generates one video with Alibaba Qwen model from a Text prompt and an optional reference image.", + attributes: [ + ...alibabaVideoCommonAttributes, + { + name: "reference-image", + kind: "reference", + required: false, + summary: + "The image Artifact to use as a style or content reference for video generation.", + accepts: [artifactTypes.blob], + }, + ], + ports: alibabaVideoPort, + example: + '', + notes: [ + ...alibabaVideoSettingNotes, + "Reference image is optional and provides style guidance.", + "The element accepts no children and no text content.", + ], + } as SurfaceVocabulary, + }, +] as const; + +export const alibabaVideoManifest = alibabaVideoBaseDefinition.manifest; +export const alibabaVideoDefinition = alibabaVideoBaseDefinition; + +export { + createAlibabaVideoAssembledGenerationFragment, + createAlibabaVideoGenerationFragment, +} from "./fragment.js"; +export { + decodeAlibabaVideoTextVideoSurface, + decodeAlibabaVideoReferenceVideoSurface, +} from "./surface.js"; diff --git a/packages/alibaba-video/src/surface.ts b/packages/alibaba-video/src/surface.ts new file mode 100644 index 00000000..e7c879ef --- /dev/null +++ b/packages/alibaba-video/src/surface.ts @@ -0,0 +1,41 @@ +/** + * Alibaba Video Generation Surface Decoders + * Decodes markup surfaces into generation requests and port configurations + */ + +import type { Surface } from "@hypit/markup"; +import { createAlibabaVideoGenerationFragment } from "./fragment.js"; +import type { AlibabaVideoPortMap } from "./index.js"; + +/** + * Decodes Alibaba TextVideo surface markup into a generation fragment + */ +export function decodeAlibabaVideoTextVideoSurface( + surface: Surface +): ReturnType { + const ports: AlibabaVideoPortMap = { + prompt: surface.getAttribute("prompt") as any, + duration: surface.getAttribute("duration") as any, + resolution: surface.getAttribute("resolution") as any, + aspectRatio: surface.getAttribute("aspect-ratio") as any, + }; + + return createAlibabaVideoGenerationFragment(ports); +} + +/** + * Decodes Alibaba ReferenceVideo surface markup into a generation fragment + */ +export function decodeAlibabaVideoReferenceVideoSurface( + surface: Surface +): ReturnType { + const ports: AlibabaVideoPortMap = { + prompt: surface.getAttribute("prompt") as any, + duration: surface.getAttribute("duration") as any, + resolution: surface.getAttribute("resolution") as any, + aspectRatio: surface.getAttribute("aspect-ratio") as any, + referenceImage: surface.getAttribute("reference-image") as any, + }; + + return createAlibabaVideoGenerationFragment(ports); +}