From f072eb6162eec43fbe9e56ad82e222918108e49e Mon Sep 17 00:00:00 2001 From: Mohit Varikuti Date: Thu, 25 Jun 2026 09:47:28 -0700 Subject: [PATCH] Add TwelveLabs Pegasus as a hosted captioning backend Adds an opt-in pegasus-1.5 model preset that uses TwelveLabs Pegasus to caption videos server-side instead of a local VLM. The model loader gains a third 'pegasus' strategy (backend/twelvelabs_provider.py) that uploads the source video as a TwelveLabs asset, waits for it to be ready, and calls analyze() with the user's prompt. No GPU or local weights are required. The key is read from TWELVELABS_API_KEY in the environment and is never persisted. Defaults are unchanged (Qwen3-VL stays the default preset) and the local strategies are untouched. Docs and the frontend preset type are updated to match. --- README.md | 10 ++ backend/config.py | 5 + backend/model_loader.py | 31 +++- backend/model_presets.py | 27 +++- backend/processing.py | 2 + backend/schemas.py | 2 + backend/tests/test_twelvelabs_provider.py | 102 +++++++++++++ backend/twelvelabs_provider.py | 176 ++++++++++++++++++++++ documentation/ARCHITECTURE.md | 9 +- documentation/CONFIGURATION.md | 29 ++++ frontend/src/types/settings.ts | 2 + requirements.txt | 4 + 12 files changed, 393 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_twelvelabs_provider.py create mode 100644 backend/twelvelabs_provider.py diff --git a/README.md b/README.md index bc533eb..c7c08b7 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,16 @@ Edit `backend/config.py` to adjust: | `MAX_TOKENS` | 512 | Max caption length | | `TEMPERATURE` | 0.3 | Generation creativity | +### Hosted captioning with TwelveLabs Pegasus (optional) + +Besides the local VLMs, you can use [TwelveLabs](https://twelvelabs.io) Pegasus +as a hosted captioning backend — no GPU required, the video is analysed +server-side. Set `TWELVELABS_API_KEY` in your environment and pick the +**TwelveLabs Pegasus 1.5 (hosted)** preset in Settings. Get a free API key at +https://twelvelabs.io (generous free tier). See +[`documentation/CONFIGURATION.md`](documentation/CONFIGURATION.md#twelvelabs-pegasus-hosted) +for details. + ## Multi-GPU Processing On systems with multiple CUDA GPUs, the suite automatically detects available devices and enables parallel processing: diff --git a/backend/config.py b/backend/config.py index d55d4c9..39418d2 100644 --- a/backend/config.py +++ b/backend/config.py @@ -4,6 +4,7 @@ """ import json +import os from pathlib import Path from typing import Optional @@ -144,6 +145,10 @@ def set_include_images(include: bool) -> None: DEFAULT_PRESET_ID = "qwen3-vl-8b" MODEL_ID = "Qwen/Qwen3-VL-8B-Instruct" +# TwelveLabs Pegasus (hosted) preset. The API key is read from the environment +# and never persisted to disk. Get a free key at https://twelvelabs.io. +TWELVELABS_API_KEY = os.environ.get("TWELVELABS_API_KEY") + # Device: "cuda", "cpu", or "auto" DEVICE = "cuda" diff --git a/backend/model_loader.py b/backend/model_loader.py index 7c722cb..fa846e6 100644 --- a/backend/model_loader.py +++ b/backend/model_loader.py @@ -451,6 +451,25 @@ def load_model( total_start = time.time() + # Hosted presets (e.g. Pegasus) have no local weights and no torch device — + # build their client and return before any download/torch setup. + if preset["loader"] == "pegasus": + from backend.twelvelabs_provider import load_pegasus + + strategy_info = load_pegasus(preset) + model_info = { + **strategy_info, + "model_id": effective_model_id, + "model_path": None, + "device": None, + "dtype": None, + "preset": preset, + "preset_id": _find_preset_id(preset), + } + _MODEL_CACHE[cache_key] = model_info + print(f"[Model Loader] Hosted model ready: {preset['label']}") + return model_info + sage_enabled = enable_sage_attention() if use_sage_attention else False model_path = download_model(effective_model_id, config.MODELS_DIR) @@ -510,8 +529,14 @@ def generate_caption( max_tokens: int = None, temperature: float = None, video_fps: float = None, + video_path=None, ) -> Tuple[str, Dict[str, Any]]: - """Generate a caption via the strategy selected at load time.""" + """Generate a caption via the strategy selected at load time. + + `video_path` is the source media file; local strategies ignore it (they + consume pre-extracted `images`), but hosted strategies like Pegasus analyse + the file directly and require it. + """ max_tokens = max_tokens or config.MAX_TOKENS temperature = temperature if temperature is not None else config.TEMPERATURE preset = model_info["preset"] @@ -520,6 +545,10 @@ def generate_caption( return _generate_image_text_to_text(model_info, images, prompt, max_tokens, temperature) elif preset["loader"] == "gemma4": return _generate_gemma4(model_info, images, prompt, max_tokens, temperature, preset, video_fps=video_fps) + elif preset["loader"] == "pegasus": + from backend.twelvelabs_provider import generate_pegasus + + return generate_pegasus(model_info, images, prompt, max_tokens, temperature, video_path=video_path) else: raise ValueError(f"Unknown loader strategy: {preset['loader']}") diff --git a/backend/model_presets.py b/backend/model_presets.py index 741364c..77f4c8b 100644 --- a/backend/model_presets.py +++ b/backend/model_presets.py @@ -18,9 +18,10 @@ # model_id HuggingFace repo id # label Human-readable label for the UI dropdown # description One-line help text -# loader "image_text_to_text" | "gemma4" +# loader "image_text_to_text" | "gemma4" | "pegasus" # content_type "image_list" (one image block per frame) # | "video_block" (single video block w/ frame list) +# | "hosted_video" (source file analysed server-side) # supports_sage_attention Whether SageAttention is safe to use # supports_torch_compile Whether torch.compile is safe to use # supports_multi_gpu_shard If True, loads with device_map="auto" on one @@ -87,6 +88,26 @@ "enable_thinking": False, "gen_defaults": {"temperature": 1.0, "top_p": 0.95, "top_k": 64}, }, + "pegasus-1.5": { + "model_id": "twelvelabs/pegasus1.5", + "label": "TwelveLabs Pegasus 1.5 (hosted)", + "description": "Hosted video-understanding model. No GPU/VRAM needed; " + "analyses the source video server-side. Requires TWELVELABS_API_KEY.", + "loader": "pegasus", + "content_type": "hosted_video", + "supports_sage_attention": False, + "supports_torch_compile": False, + "supports_multi_gpu_shard": False, + "quantization": None, + "default_max_frames": 16, + "default_frame_size": 336, + "approx_vram_gb": 0, + "frame_size_divisor": None, + # Hosted-model marker: no local weights, requires an API key. + "is_hosted": True, + "requires_api_key": True, + "pegasus_model_name": "pegasus1.5", + }, } DEFAULT_PRESET = "qwen3-vl-8b" @@ -144,7 +165,9 @@ def list_presets_public() -> list: "quantization": p["quantization"], "supports_sage_attention": p["supports_sage_attention"], "supports_torch_compile": p["supports_torch_compile"], - "is_video_native": p["content_type"] == "video_block", + "is_video_native": p["content_type"] in ("video_block", "hosted_video"), + "is_hosted": p.get("is_hosted", False), + "requires_api_key": p.get("requires_api_key", False), }) return out diff --git a/backend/processing.py b/backend/processing.py index c787886..46f708a 100644 --- a/backend/processing.py +++ b/backend/processing.py @@ -396,6 +396,7 @@ async def process_single_video(worker_id: int, video_path: Path): max_tokens=settings.max_tokens, temperature=settings.temperature, video_fps=video_meta.get("fps"), + video_path=video_path, ) ) @@ -597,6 +598,7 @@ async def _process_videos_sequential( max_tokens=settings.max_tokens, temperature=settings.temperature, video_fps=video_meta.get("fps"), + video_path=video_path, ) ) diff --git a/backend/schemas.py b/backend/schemas.py index d653025..c24f9dc 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -187,6 +187,8 @@ class ModelPresetInfo(BaseModel): supports_sage_attention: bool supports_torch_compile: bool is_video_native: bool + is_hosted: bool = False + requires_api_key: bool = False class ModelPresetListResponse(BaseModel): diff --git a/backend/tests/test_twelvelabs_provider.py b/backend/tests/test_twelvelabs_provider.py new file mode 100644 index 0000000..1ed2fc3 --- /dev/null +++ b/backend/tests/test_twelvelabs_provider.py @@ -0,0 +1,102 @@ +""" +Tests for the TwelveLabs Pegasus captioning backend. + +The unit tests run with no network access. The live test is skipped unless +TWELVELABS_API_KEY is set (a free key is available at https://twelvelabs.io). +""" + +import os +import pytest + +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from backend import twelvelabs_provider as tl +from backend.model_presets import MODEL_PRESETS, get_preset, list_presets_public + + +class TestPegasusPreset: + """The Pegasus preset is registered and wired to the pegasus loader.""" + + def test_preset_registered(self): + preset = get_preset("pegasus-1.5") + assert preset is not None + assert preset["loader"] == "pegasus" + assert preset["is_hosted"] is True + assert preset["requires_api_key"] is True + assert preset["approx_vram_gb"] == 0 + + def test_preset_in_public_list(self): + ids = {p["id"] for p in list_presets_public()} + assert "pegasus-1.5" in ids + entry = next(p for p in list_presets_public() if p["id"] == "pegasus-1.5") + assert entry["is_hosted"] is True + assert entry["requires_api_key"] is True + + def test_default_preset_unchanged(self): + # Opt-in only: Pegasus must never become the default. + from backend.model_presets import DEFAULT_PRESET + assert DEFAULT_PRESET == "qwen3-vl-8b" + + +class TestBuildGenerateMeta: + """build_generate_meta is pure and matches the local-strategy meta shape.""" + + def test_meta_shape(self): + meta = tl.build_generate_meta("a caption", elapsed=2.0, num_frames=16) + for key in ( + "input_tokens", "output_tokens", "encode_time", + "generate_time", "total_time", "tokens_per_sec", "num_frames", + ): + assert key in meta + assert meta["num_frames"] == 16 + assert meta["generate_time"] == 2.0 + assert meta["total_time"] == 2.0 + + def test_uses_usage_tokens_when_provided(self): + meta = tl.build_generate_meta( + "hello world", elapsed=1.0, num_frames=4, usage_output_tokens=42 + ) + assert meta["output_tokens"] == 42 + assert meta["tokens_per_sec"] == 42.0 + + def test_estimates_tokens_without_usage(self): + meta = tl.build_generate_meta("x" * 40, elapsed=2.0, num_frames=1) + assert meta["output_tokens"] == 10 # ~4 chars/token + + def test_zero_elapsed_no_divide_by_zero(self): + meta = tl.build_generate_meta("text", elapsed=0.0, num_frames=1) + assert meta["tokens_per_sec"] == 0 + + +class TestPegasusGuards: + """Pegasus surfaces clear errors instead of silently misbehaving.""" + + def test_generate_requires_video_path(self): + model_info = {"client": object(), "pegasus_model_name": "pegasus1.5"} + with pytest.raises(ValueError, match="source video path"): + tl.generate_pegasus(model_info, images=[], prompt="x", + max_tokens=512, temperature=0.3, video_path=None) + + def test_load_requires_api_key(self, monkeypatch): + monkeypatch.delenv("TWELVELABS_API_KEY", raising=False) + with pytest.raises(RuntimeError, match="TWELVELABS_API_KEY"): + tl.load_pegasus(get_preset("pegasus-1.5")) + + +@pytest.mark.skipif( + not os.environ.get("TWELVELABS_API_KEY"), + reason="requires TWELVELABS_API_KEY (free key at https://twelvelabs.io)", +) +class TestPegasusLive: + """Live wiring check against the real API (skipped without a key).""" + + def test_load_builds_client(self): + info = tl.load_pegasus(get_preset("pegasus-1.5")) + assert info["client"] is not None + assert info["pegasus_model_name"] == "pegasus1.5" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/backend/twelvelabs_provider.py b/backend/twelvelabs_provider.py new file mode 100644 index 0000000..1e961f7 --- /dev/null +++ b/backend/twelvelabs_provider.py @@ -0,0 +1,176 @@ +""" +TwelveLabs Pegasus captioning backend. + +Pegasus is a hosted video-understanding model. Unlike the local VLM presets +(Qwen3-VL, Gemma 4) it does NOT run on the GPU and does NOT consume the +pre-extracted frames — it analyses the source video file server-side. To slot +into the existing loader/strategy seam it therefore: + + load_pegasus -> validates the API key and constructs a TwelveLabs client + (the only "load" step a hosted model needs) + generate_pegasus -> uploads the local video as an asset, waits for it to be + ready, then calls client.analyze(...) with the prompt + +The public model loader (`backend.model_loader`) dispatches here when a preset +declares `loader == "pegasus"`. This keeps `processing.py` unaware of which +backend is active, exactly like the local strategies. +""" + +import os +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +# Statuses returned by the TwelveLabs assets API. +ASSET_READY = "ready" +ASSET_FAILED = "failed" + +# How long to wait for an uploaded asset to finish processing before giving up. +_ASSET_POLL_INTERVAL_SEC = 3.0 +_ASSET_POLL_TIMEOUT_SEC = 600.0 + +# Pegasus requires max_tokens >= 512; the app's UI allows as low as 64, so +# clamp up to keep small UI values from being rejected by the API. +_PEGASUS_MIN_MAX_TOKENS = 512 + + +def get_api_key() -> Optional[str]: + """Read the TwelveLabs API key from the environment (never logged).""" + return os.environ.get("TWELVELABS_API_KEY") or None + + +def load_pegasus(preset: Dict[str, Any]) -> Dict[str, Any]: + """ + "Load" a Pegasus model: a hosted model has no weights to download, so this + just validates the API key and builds a reusable SDK client. + + Raises a clear error if the key is missing or the SDK isn't installed, so + the failure surfaces in the UI's model-load step rather than mid-caption. + """ + api_key = get_api_key() + if not api_key: + raise RuntimeError( + "TWELVELABS_API_KEY is not set. Export it before selecting the " + "Pegasus preset (get a free key at https://twelvelabs.io)." + ) + + try: + from twelvelabs import TwelveLabs + except ImportError as e: # pragma: no cover - exercised only without the dep + raise RuntimeError( + "The 'twelvelabs' package is required for the Pegasus preset. " + "Install it with: pip install 'twelvelabs>=1.2.8'" + ) from e + + client = TwelveLabs(api_key=api_key) + return { + "client": client, + "pegasus_model_name": preset.get("pegasus_model_name", "pegasus1.5"), + "sage_attention": False, + "torch_compiled": False, + "device_map": None, + } + + +def _wait_for_asset_ready(client: Any, asset_id: str) -> None: + """Poll an uploaded asset until it is ready (or raise on failure/timeout).""" + deadline = time.time() + _ASSET_POLL_TIMEOUT_SEC + while True: + asset = client.assets.retrieve(asset_id) + status = getattr(asset, "status", None) + if status == ASSET_READY: + return + if status == ASSET_FAILED: + raise RuntimeError(f"TwelveLabs asset {asset_id} failed to process") + if time.time() >= deadline: + raise TimeoutError( + f"TwelveLabs asset {asset_id} not ready after " + f"{_ASSET_POLL_TIMEOUT_SEC:.0f}s (status={status})" + ) + time.sleep(_ASSET_POLL_INTERVAL_SEC) + + +def _estimate_tokens(text: str) -> int: + """Rough token estimate (~4 chars/token) for parity with local strategies + when the API response omits a token count.""" + return max(1, len(text) // 4) + + +def build_generate_meta( + output_text: str, + elapsed: float, + num_frames: int, + usage_output_tokens: Optional[int] = None, +) -> Dict[str, Any]: + """ + Build the gen-meta dict the processing pipeline expects, matching the shape + returned by the local strategies. Pure/no-network so it is unit-testable. + """ + output_tokens = usage_output_tokens or _estimate_tokens(output_text) + return { + "input_tokens": 0, # hosted model: prompt tokenisation happens server-side + "output_tokens": output_tokens, + "encode_time": 0.0, + "generate_time": elapsed, + "total_time": elapsed, + "tokens_per_sec": output_tokens / elapsed if elapsed > 0 else 0, + "num_frames": num_frames, + } + + +def generate_pegasus( + model_info: Dict[str, Any], + images: List, + prompt: str, + max_tokens: int, + temperature: float, + video_path: Optional[Path] = None, +) -> Tuple[str, Dict[str, Any]]: + """ + Caption a video with Pegasus. + + Pegasus analyses the source video file directly, so `video_path` is + required and the pre-extracted `images` are used only to report a frame + count for UI parity. The video is uploaded as a TwelveLabs asset, polled + until ready, then analysed with the user's prompt. + """ + if video_path is None: + raise ValueError( + "Pegasus requires the source video path; it analyses the file " + "server-side rather than pre-extracted frames." + ) + + client = model_info["client"] + model_name = model_info["pegasus_model_name"] + from twelvelabs.types.video_context import VideoContext_AssetId + + start = time.time() + + # Upload the local file as an asset, then wait for server-side processing. + with open(video_path, "rb") as f: + asset = client.assets.create(method="direct", file=f) + _wait_for_asset_ready(client, asset.id) + + response = client.analyze( + model_name=model_name, + video=VideoContext_AssetId(type="asset_id", asset_id=asset.id), + prompt=prompt, + temperature=temperature, + max_tokens=max(max_tokens, _PEGASUS_MIN_MAX_TOKENS), + ) + + elapsed = time.time() - start + output_text = (response.data or "").strip() + + usage_tokens = None + usage = getattr(response, "usage", None) + if usage is not None: + usage_tokens = getattr(usage, "output_tokens", None) + + meta = build_generate_meta( + output_text, + elapsed=elapsed, + num_frames=len(images) if images else 0, + usage_output_tokens=usage_tokens, + ) + return output_text, meta diff --git a/documentation/ARCHITECTURE.md b/documentation/ARCHITECTURE.md index 4866c95..bb107f7 100644 --- a/documentation/ARCHITECTURE.md +++ b/documentation/ARCHITECTURE.md @@ -312,10 +312,12 @@ Composables: model: loader strategy, frame content-block format, quantization, capability flags (SageAttention, torch.compile, multi-GPU sharding), and recommended defaults. Adding a model is a new dict entry, not a new `if` branch. -- `backend/model_loader.py` is a thin dispatcher over two strategies: - `image_text_to_text` (Qwen-VL family, one image block per frame) and +- `backend/model_loader.py` is a thin dispatcher over three strategies: + `image_text_to_text` (Qwen-VL family, one image block per frame), `gemma4` (single video block, optional TorchAo int4, optional - `device_map="auto"` sharding). + `device_map="auto"` sharding), and `pegasus` (hosted TwelveLabs Pegasus — + no local weights or GPU; uploads the source video and analyses it + server-side, implemented in `backend/twelvelabs_provider.py`). - The settings POST handler in `backend/api.py` syncs `model_id` and enforces preset-declared capability flags whenever `model_preset` changes — a UI preset switch is a single action. @@ -339,6 +341,7 @@ Composables: | Model loading (dispatcher) | `backend/model_loader.py` | `load_model()` | | Image-text-to-text strategy | `backend/model_loader.py` | `_load_image_text_to_text`, `_generate_image_text_to_text` | | Gemma 4 strategy | `backend/model_loader.py` | `_load_gemma4`, `_generate_gemma4` | +| Pegasus (hosted) strategy | `backend/twelvelabs_provider.py` | `load_pegasus`, `generate_pegasus` | | Memory cleanup | `backend/model_loader.py` | `clear_cache()` | | Frame extraction | `backend/video_processor.py` | 80-150 | | Vue root component | `frontend/src/App.vue` | 1-464 | diff --git a/documentation/CONFIGURATION.md b/documentation/CONFIGURATION.md index 9acb4ac..6dbb6f7 100644 --- a/documentation/CONFIGURATION.md +++ b/documentation/CONFIGURATION.md @@ -54,6 +54,7 @@ sharding, quantization). | `qwen3-vl-8b` | `Qwen/Qwen3-VL-8B-Instruct` | ~16 GB | Default. Single-GPU, `torch.compile` enabled. | | `gemma-4-26b-a4b` | `google/gemma-4-26B-A4B-it` | ~52 GB | MoE, video-native. Shards across multiple GPUs (batch_size forced to 1). | | `gemma-4-26b-a4b-int4` | `google/gemma-4-26B-A4B-it` | ~15 GB | Same model, int4 via TorchAo. Fits on a single 24 GB card. | +| `pegasus-1.5` | `twelvelabs/pegasus1.5` | 0 GB (hosted) | TwelveLabs Pegasus. Runs server-side, no local GPU. Requires `TWELVELABS_API_KEY`. | To add a preset, append to `MODEL_PRESETS` in `backend/model_presets.py`. The loader chooses a strategy based on the preset's `loader` field: @@ -63,6 +64,34 @@ The loader chooses a strategy based on the preset's `loader` field: - `gemma4` uses `transformers.Gemma4ForConditionalGeneration`, optionally with TorchAo int4 quantization, and feeds a single `{"type": "video"}` content block carrying the frame list. +- `pegasus` uses the hosted TwelveLabs Pegasus model + (`backend/twelvelabs_provider.py`). It uploads the source video as a + TwelveLabs asset and analyses it server-side, so it needs no local weights, + no GPU, and ignores the pre-extracted frames. + +### TwelveLabs Pegasus (hosted) + +The `pegasus-1.5` preset is an opt-in hosted captioning backend. It is never +the default and does not affect the local presets. To use it: + +1. Get a free API key at https://twelvelabs.io (generous free tier). +2. Export it before launching the server: + + ```bash + export TWELVELABS_API_KEY=tlk_xxxxxxxxxxxxxxxx # Linux/Mac + set TWELVELABS_API_KEY=tlk_xxxxxxxxxxxxxxxx # Windows + ``` + +3. Select **TwelveLabs Pegasus 1.5 (hosted)** in Settings → Model. + +The key is read from the environment only (`backend/config.py:TWELVELABS_API_KEY`) +and is never written to `settings.json` or `user_config.json`. Because Pegasus +analyses the whole video server-side, `max_frames` / `frame_size` have no +effect, and `max_tokens` is clamped up to the model's minimum of 512. + +| Setting | Type | Default | Description | +|---------|------|---------|-------------| +| `TWELVELABS_API_KEY` | str/None | `None` (env) | API key for the Pegasus preset. Read from the environment, never persisted. | ### Inference Settings diff --git a/frontend/src/types/settings.ts b/frontend/src/types/settings.ts index 7282339..c2c71d1 100644 --- a/frontend/src/types/settings.ts +++ b/frontend/src/types/settings.ts @@ -74,6 +74,8 @@ export interface ModelPresetInfo { supports_sage_attention: boolean supports_torch_compile: boolean is_video_native: boolean + is_hosted: boolean + requires_api_key: boolean } export interface ModelPresetListResponse { diff --git a/requirements.txt b/requirements.txt index 6d9d4b3..e16b797 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,10 @@ opencv-python>=4.8.0 # HuggingFace huggingface_hub>=0.20.0 +# TwelveLabs Pegasus hosted captioning (optional — required only for the +# pegasus-1.5 preset; set TWELVELABS_API_KEY to use it) +twelvelabs>=1.2.8 + # Qwen VL specific qwen-vl-utils>=0.0.8 sentencepiece>=0.1.99