diff --git a/.claude/.codebase-info/.map-state.json b/.claude/.codebase-info/.map-state.json index 4257d781..52efcd40 100644 --- a/.claude/.codebase-info/.map-state.json +++ b/.claude/.codebase-info/.map-state.json @@ -1,31 +1,32 @@ { + "schemaVersion": 1, "tool": "codebase-mapper", - "version": "2.8.0", - "mappedAt": "2026-07-18", - "gitCommit": "73dbc8be54f4ca32e71d28f5d9a7648dd9628c9b", + "version": "2.14.4", + "mappedAt": "2026-08-11", + "gitCommit": "c80c7520a703018d36abc97f2199dcaf3d9c29a3", "documents": [ "architecture.md", - "tech-landscape.md", + "coding-style.md", + "communication.md", + "dependencies.md", "directory-structure.md", "entry-points.md", "modules.md", - "communication.md", - "dependencies.md", + "onboarding.md", "patterns.md", - "coding-style.md", - "onboarding.md" + "tech-landscape.md" ], "hashes": { - "INDEX.md": "17eaa0ba82f159465e126597aac349c6e4429330887b58742636734e15854313", - "architecture.md": "7c908f047e6b0f08824ccd7c3f674ad47bd547dde234d0bd424a6d2784c0cbb6", - "tech-landscape.md": "cad1a11e5e63013a10cf07657d248196d0e3e4e14203eba7b777c1c123628198", - "directory-structure.md": "7e3fc4c3c8b89b6627beabcb6ddebd74b0ac7094e52718926fc87a94c932df84", - "entry-points.md": "6bd4ccae3d975196e03fc2c4e86007eba89f1f6427232fd8adce78ec97d6b432", - "modules.md": "1238c76664edb1c1667fda23f39507bcab5b760b57a3a3728d7104b2cf8032e7", - "communication.md": "49741a7455b0f162881e1cfb2d797dc8e3832cc8782796bbcb8a10529f2c5a0b", + "INDEX.md": "50e7beb7784d3d01f8ce0e85b842889f5c033ca98cbb30981fd0b532c1495c8f", + "architecture.md": "8e6be8d2795231ea5919d7cdc23a201595f9fed9724b635e6cc686b2bf24581f", + "coding-style.md": "b63e8dff06d16a2d882f02624c62f490291317aec8b3032f6d79b3ee035e8567", + "communication.md": "6c73cad50151b73a6be63c8f0745d3f7a6653a861d37afce660c512ac30b8819", "dependencies.md": "b445a920c121e27006fec824add6865d01cf71212a517e5ff8b2058d3999037a", - "patterns.md": "12b96c2d4f8bc7507bcae39b2b385b9db4d0dec95ef6a2049958663aff021705", - "coding-style.md": "2f939544333b927cf8731f5bb906ce03cb3a46afa50c9735011ff8f7e4caca9e", - "onboarding.md": "3d28c671dc095a2aadb297bd7ab200c677779a99c7534e03e55a3d64c935f07e" + "directory-structure.md": "ca200817789c4d1fdc41d1683c37740c9837bf323b18881cb2351ef8a56c4926", + "entry-points.md": "62fd8ff478fc27c3658776626059bdd4101cab12a74a48cdd8787f9ca7b83f97", + "modules.md": "7a8772e5f31be0070003c9a7ce78eca5df95dfb6a1c9dd471a6be97df81084aa", + "onboarding.md": "3d28c671dc095a2aadb297bd7ab200c677779a99c7534e03e55a3d64c935f07e", + "patterns.md": "f3a339a7bc711c4586062139d135e1d79129653cf82fc430cd256028d25e2dcc", + "tech-landscape.md": "700b3ba7bb5e47ec14086e2b94ad65ad956c0bfbe1c463f96c39493143e0c912" } } diff --git a/.claude/.codebase-info/INDEX.md b/.claude/.codebase-info/INDEX.md index b99e8ac1..b96fc29a 100644 --- a/.claude/.codebase-info/INDEX.md +++ b/.claude/.codebase-info/INDEX.md @@ -1,12 +1,12 @@ # Codebase Map — Atomic Agents -*Last Updated: 2026-07-18* +*Last Updated: 2026-08-11* Atomic Agents is a lightweight, modular Python framework for building agentic AI applications as composable, schema-driven building blocks (built on Instructor + Pydantic). This repository is a `uv`-workspace **monorepo**: the core framework, a TUI tool installer, a tool library, and examples. -**Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling +**Stack:** Python ≥3.12 · Instructor · Pydantic v2 · LiteLLM · MCP · Textual · uv + Hatchling · Pyright · `VideoURL` multimodal content **Shape:** Monorepo — `atomic-agents/` (core lib) · `atomic-assembler/` (CLI) · `atomic-forge/` (tools) · `atomic-examples/` (examples) **Package:** `atomic-agents` v2.9.1 on PyPI · core import package is `atomic_agents` diff --git a/.claude/.codebase-info/architecture.md b/.claude/.codebase-info/architecture.md index d09f077a..9cc9cbd9 100644 --- a/.claude/.codebase-info/architecture.md +++ b/.claude/.codebase-info/architecture.md @@ -1,6 +1,6 @@ # Architecture -*Last Updated: 2026-07-05* +*Last Updated: 2026-08-11* ## Summary @@ -46,8 +46,8 @@ Tool install (atomic-assembler TUI): | Component | Path | Responsibility | |-----------|------|----------------| | Core agent | `atomic-agents/atomic_agents/agents/atomic_agent.py` | `AtomicAgent`, `AgentConfig`, run / stream / async methods | -| Base contracts | `atomic-agents/atomic_agents/base/` | `BaseIOSchema`, `BaseTool`, `BaseToolConfig`, `BaseResource`, `BasePrompt` | -| Context | `atomic-agents/atomic_agents/context/` | `SystemPromptGenerator`, `BaseDynamicContextProvider`, `BaseChatHistory` (pluggable memory contract) + `ChatHistory` | +| Base contracts | `atomic-agents/atomic_agents/base/` | `BaseIOSchema`, `BaseTool`, `BaseToolConfig`, `BaseResource`, `BasePrompt`, `VideoURL` | +| Context | `atomic-agents/atomic_agents/context/` | `SystemPromptGenerator`, `BaseDynamicContextProvider`, `BaseChatHistory` (pluggable memory contract) + `ChatHistory` with Instructor media and `VideoURL` | | Connectors | `atomic-agents/atomic_agents/connectors/mcp/` | Model Context Protocol tools / resources / prompts | | Utils | `atomic-agents/atomic_agents/utils/` | Token counting (LiteLLM), tool-message formatting | | Assembler (CLI) | `atomic-assembler/atomic_assembler/` | Textual TUI to fetch/install forge tools | @@ -60,8 +60,7 @@ Tool install (atomic-assembler TUI): 2. On `run(input)`, the `SystemPromptGenerator` assembles the system message from background / steps / output-instructions plus any registered **context providers** (evaluated live at call time). -3. The `ChatHistory` (typed `Message`s, multimodal-aware) is serialized into the provider message - list; oldest *turns* are trimmed to respect `max_context_tokens`. +3. The `ChatHistory` (typed `Message`s, multimodal-aware) is serialized into provider messages; `VideoURL` becomes an OpenAI-compatible `video_url` content part, while token counting uses a `[video content]` placeholder because LiteLLM cannot count video parts. Oldest *turns* are trimmed to respect `max_context_tokens`. 4. `client.chat.completions.create(response_model=output_schema)` performs the structured LLM call via Instructor. Streaming and async variants exist: `run_stream`, `run_async`, `run_async_stream`. 5. The validated output schema is appended to history and returned. Instructor **hooks** diff --git a/.claude/.codebase-info/coding-style.md b/.claude/.codebase-info/coding-style.md index 2d30cfd6..13091c0e 100644 --- a/.claude/.codebase-info/coding-style.md +++ b/.claude/.codebase-info/coding-style.md @@ -1,6 +1,6 @@ # Coding Style -*Last Updated: 2026-06-13* +*Last Updated: 2026-08-11* ## Tooling - **Formatter:** Black, line length **127** (`[tool.black]` in `pyproject.toml`; also a pre-commit hook). @@ -10,6 +10,7 @@ - **Pre-commit** (`.pre-commit-config.yaml`): trailing-whitespace, end-of-file-fixer, check-yaml, check-added-large-files, Black, Flake8. - **CI** runs `black --check` and `flake8` across all four subprojects (`code-quality.yml`). +- **Type checker:** Pyright, configured by `pyrightconfig.json` to use the uv-managed `.venv`. ## Conventions | Kind | Convention | Example | diff --git a/.claude/.codebase-info/communication.md b/.claude/.codebase-info/communication.md index 2c86b2ba..86aa7b84 100644 --- a/.claude/.codebase-info/communication.md +++ b/.claude/.codebase-info/communication.md @@ -1,6 +1,6 @@ # Communication & Integrations -*Last Updated: 2026-06-13* +*Last Updated: 2026-08-11* The framework exposes no HTTP API of its own; "communication" here means how it talks to LLM providers and external tools. @@ -11,13 +11,17 @@ providers and external tools. `client.chat.completions.create(response_model=OutputSchema)` (and `create_partial` for streaming). - Provider-agnostic: OpenAI, Anthropic, Google Gemini, MiniMax, and 100+ models via LiteLLM. Mode is configurable (`Mode.TOOLS` is the default). +- Multimodal content uses Instructor's Image/Audio/PDF types plus the framework's `VideoURL`; + `ChatHistory.get_history()` converts `VideoURL` to an OpenAI-compatible `video_url` dict, which + Instructor forwards unchanged to providers that support video inputs. - Provider quirks live in `AgentConfig`: `system_role`, `assistant_role` (use `"model"` for Gemini), `tool_result_role` (auto-detected). - Code: `atomic-agents/atomic_agents/agents/atomic_agent.py`. ## Token accounting (via LiteLLM) - `atomic-agents/atomic_agents/utils/token_counter.py` uses LiteLLM's `token_counter` for - provider-agnostic counts, which drive the context-window trimming in `AtomicAgent`. + provider-agnostic counts, which drive the context-window trimming in `AtomicAgent`. Video content + parts are represented by a `[video content]` text placeholder because LiteLLM cannot count them. ## MCP — Model Context Protocol - `atomic-agents/atomic_agents/connectors/mcp/` turns MCP server capabilities into agent diff --git a/.claude/.codebase-info/directory-structure.md b/.claude/.codebase-info/directory-structure.md index 219439e4..6f70644c 100644 --- a/.claude/.codebase-info/directory-structure.md +++ b/.claude/.codebase-info/directory-structure.md @@ -1,6 +1,6 @@ # Directory Structure -*Last Updated: 2026-07-18* +*Last Updated: 2026-08-11* ## Root Layout @@ -9,11 +9,11 @@ atomic-agents/ # repo root (uv workspace) ├── atomic-agents/ # CORE framework project (PyPI: atomic-agents) │ └── atomic_agents/ # import package │ ├── agents/ # AtomicAgent, AgentConfig -│ ├── base/ # BaseIOSchema, BaseTool, BaseResource, BasePrompt +│ ├── base/ # BaseIOSchema, BaseTool, BaseResource, BasePrompt, VideoURL │ ├── context/ # SystemPromptGenerator, BaseChatHistory/ChatHistory, context providers │ ├── connectors/mcp/ # Model Context Protocol integration │ └── utils/ # token counter, tool-message formatting -│ └── tests/ # pytest suite (agents/, base/, context/, connectors/, utils/) +│ └── tests/ # pytest suite (agents/, base/, context/, connectors/, utils/; VideoURL tests in base/) ├── atomic-assembler/ # Textual TUI (`atomic` command) to install forge tools │ └── atomic_assembler/ # main.py, app.py, screens/, widgets/, utils.py, constants.py ├── atomic-forge/ # library of standalone tools (NOT a package) @@ -27,6 +27,7 @@ atomic-agents/ # repo root (uv workspace) ├── guides/ # DEV_GUIDE.md and contributor guides ├── scripts/ # generate_llms_files.py (llms.txt index + llms-*.txt bundles) ├── pyproject.toml # package metadata, deps, [tool.black], uv workspace +├── pyrightconfig.json # Pyright configuration for the uv-managed `.venv` ├── context7.json # Context7 indexing config + v2 API rules for AI assistants ├── build_and_deploy.ps1 # version bump + uv build/publish ├── AGENTS.md # the project's own design philosophy (imported by CLAUDE.md) diff --git a/.claude/.codebase-info/entry-points.md b/.claude/.codebase-info/entry-points.md index e1129128..ebccf304 100644 --- a/.claude/.codebase-info/entry-points.md +++ b/.claude/.codebase-info/entry-points.md @@ -1,6 +1,6 @@ # Entry Points -*Last Updated: 2026-07-05* +*Last Updated: 2026-08-11* ## 1. Library API (the primary entry point) @@ -18,6 +18,8 @@ result = agent.run(BasicChatInputSchema(chat_message="Hello")) - **Module:** `atomic-agents/atomic_agents/agents/atomic_agent.py` - **Run methods:** `run`, `run_stream`, `run_async`, `run_async_stream`. - Custom agents define their own `BaseIOSchema` subclasses and pass them as `AtomicAgent[In, Out]`. +- **Public multimodal type:** import `VideoURL` from `atomic_agents` for video inputs; `ChatHistory` + serializes it as an OpenAI-compatible `video_url` content part before the provider call. ## 2. CLI — `atomic` diff --git a/.claude/.codebase-info/modules.md b/.claude/.codebase-info/modules.md index 05be8b89..40d733e0 100644 --- a/.claude/.codebase-info/modules.md +++ b/.claude/.codebase-info/modules.md @@ -1,6 +1,6 @@ # Key Modules -*Last Updated: 2026-07-05* +*Last Updated: 2026-08-11* ## Core framework — `atomic-agents/atomic_agents/` @@ -17,8 +17,9 @@ - **Purpose:** The typed contracts everything else implements. - **Key files:** `base_io_schema.py` (`BaseIOSchema` — Pydantic base; non-empty docstring enforced and used as the schema description), `base_tool.py` (`BaseTool[In, Out]`, `BaseToolConfig`), - `base_resource.py` (`BaseResource`), `base_prompt.py` (`BasePrompt`). -- **Exposes via package root:** `BaseIOSchema`, `BaseTool`, `BaseToolConfig`. + `base_resource.py` (`BaseResource`), `base_prompt.py` (`BasePrompt`), and `multimodal.py` + (`VideoURL` — OpenAI-compatible `video_url` content-part model for video inputs). +- **Exposes via package root:** `BaseIOSchema`, `BaseTool`, `BaseToolConfig`, `VideoURL`. ### context - **Location:** `atomic-agents/atomic_agents/context/` @@ -27,7 +28,8 @@ `base_chat_history.py` (`BaseChatHistory` — interface-only ABC declaring the memory contract `AtomicAgent` depends on; the pluggable seam for custom/persistent backends), `chat_history.py` (`ChatHistory`, `Message` — the built-in in-memory implementation of - `BaseChatHistory`: multimodal Image/Audio/PDF, turn grouping, `dump()`/`load()` serialization). + `BaseChatHistory`: multimodal Image/Audio/PDF plus `VideoURL`, turn grouping, `dump()`/`load()` + serialization). - **Note:** `AgentConfig.history` is typed to `BaseChatHistory`, so any conforming backend drops in. ### connectors/mcp diff --git a/.claude/.codebase-info/patterns.md b/.claude/.codebase-info/patterns.md index d1797297..a5879230 100644 --- a/.claude/.codebase-info/patterns.md +++ b/.claude/.codebase-info/patterns.md @@ -1,6 +1,6 @@ # Patterns & Conventions -*Last Updated: 2026-07-05* +*Last Updated: 2026-08-11* ## Atomicity Build with small, single-purpose, composable parts ("LEGO blocks"): each agent, tool, and context @@ -30,8 +30,10 @@ the next's input. See `atomic-examples/deep-research` and `orchestration-agent`. `copy()`, plus the `history`/`current_turn_id` attributes). It is the documented, dependency-free seam for plugging in custom/persistent backends; `AgentConfig.history` is typed to it. - `ChatHistory` is the built-in implementation: stores typed `Message`s grouped into turns (a - user+assistant pair shares a `turn_id`), is multimodal-aware (Image/Audio/PDF), and supports - `dump()`/`load()`. `AtomicAgent` trims the oldest whole turns to honor `max_context_tokens`. + user+assistant pair shares a `turn_id`), is multimodal-aware (Image/Audio/PDF plus `VideoURL`), + and supports `dump()`/`load()`. `VideoURL` is converted to an OpenAI-compatible `video_url` + content part for provider messages. `AtomicAgent` trims the oldest whole turns to honor + `max_context_tokens`, using a text placeholder when LiteLLM cannot count video parts. - Custom backend pattern: subclass `ChatHistory` and override `add_message`/`load` to persist (see the `persistent-memory` example and the "Writing a Custom Memory Backend" guide section). @@ -42,8 +44,8 @@ the next's input. See `atomic-examples/deep-research` and `orchestration-agent`. ## Testing - `pytest` (+ `pytest-asyncio` for async, `pytest-cov` for coverage), with `unittest.mock` for LLM clients. Core tests in `atomic-agents/tests/` mirror the package layout (`agents/`, `base/`, - `context/`, `connectors/mcp/`, `utils/`). Discovery (`pytest.ini`): files `test_*.py`, classes - `Test*`, functions `test_*`. + `context/`, `connectors/mcp/`, `utils/`), including `tests/base/test_multimodal.py` for `VideoURL`. + Discovery (`pytest.ini`): files `test_*.py`, classes `Test*`, functions `test_*`. ## Configuration - Runtime config is explicit via `AgentConfig` (client, model, history, roles, mode, diff --git a/.claude/.codebase-info/tech-landscape.md b/.claude/.codebase-info/tech-landscape.md index 7b7d928b..f6cb6347 100644 --- a/.claude/.codebase-info/tech-landscape.md +++ b/.claude/.codebase-info/tech-landscape.md @@ -1,6 +1,6 @@ # Technology Landscape -*Last Updated: 2026-07-18* +*Last Updated: 2026-08-11* ## Source-of-Truth Files @@ -11,6 +11,7 @@ | Legacy install shims | `setup.py`, `requirements.txt` | | Lint | `.flake8` | | Format + git hooks | `.pre-commit-config.yaml`, `[tool.black]` in `pyproject.toml` | +| Type checking | `pyrightconfig.json` | | Tests | `pytest.ini`, `.coveragerc` | | CI/CD | `.github/workflows/` | | Release | `build_and_deploy.ps1` | @@ -30,6 +31,7 @@ | HTTP / Git | requests, GitPython | assembler fetches forge from GitHub | | Build backend | Hatchling | `[build-system]` | | Workspace / lock / publish | uv | `uv sync` / `uv build` / `uv publish` | +| Static type checking | Pyright | configured for the workspace `.venv` via `pyrightconfig.json` | | Docs | Sphinx + MyST + RTD theme | + `sphinxcontrib-mermaid`, deployed to GitHub Pages | ## Infrastructure diff --git a/atomic-agents/atomic_agents/__init__.py b/atomic-agents/atomic_agents/__init__.py index 7e542961..c4ed2d23 100644 --- a/atomic-agents/atomic_agents/__init__.py +++ b/atomic-agents/atomic_agents/__init__.py @@ -4,7 +4,7 @@ # Core exports - base classes only from .agents.atomic_agent import AtomicAgent, AgentConfig, BasicChatInputSchema, BasicChatOutputSchema -from .base import BaseIOSchema, BaseTool, BaseToolConfig +from .base import BaseIOSchema, BaseTool, BaseToolConfig, VideoURL # Version info - read from pyproject.toml via package metadata from importlib.metadata import version as _version @@ -19,4 +19,5 @@ "BaseIOSchema", "BaseTool", "BaseToolConfig", + "VideoURL", ] diff --git a/atomic-agents/atomic_agents/agents/atomic_agent.py b/atomic-agents/atomic_agents/agents/atomic_agent.py index d2ccfe6e..10c77aa9 100644 --- a/atomic-agents/atomic_agents/agents/atomic_agent.py +++ b/atomic-agents/atomic_agents/agents/atomic_agent.py @@ -431,7 +431,8 @@ def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]: This method converts instructor multimodal objects (Image, Audio, PDF) to the OpenAI format that LiteLLM's token counter expects. Text content is also - converted to the proper multimodal text format when mixed with media. + converted to the proper multimodal text format when mixed with media. Content-part + dicts (e.g. video) become text placeholders because LiteLLM cannot count them. Returns: List[Dict[str, Any]]: History messages in LiteLLM-compatible format. @@ -462,6 +463,11 @@ def _serialize_history_for_token_count(self) -> List[Dict[str, Any]]: f"Using placeholder for estimation." ) serialized_content.append({"type": "text", "text": f"[{media_type.lower()} content]"}) + elif isinstance(item, dict): + # get_history() emits pre-lowered content-part dicts (e.g. video). + # LiteLLM's token counter only accepts text and image_url parts, + # so estimate them with a placeholder. + serialized_content.append({"type": "text", "text": f"[{item.get('type', 'unknown')} content]"}) else: # Unknown type - convert to string serialized_content.append({"type": "text", "text": str(item)}) diff --git a/atomic-agents/atomic_agents/base/__init__.py b/atomic-agents/atomic_agents/base/__init__.py index 9bbe9319..18d1d5a0 100644 --- a/atomic-agents/atomic_agents/base/__init__.py +++ b/atomic-agents/atomic_agents/base/__init__.py @@ -4,6 +4,7 @@ from .base_tool import BaseTool, BaseToolConfig from .base_resource import BaseResource, BaseResourceConfig from .base_prompt import BasePrompt, BasePromptConfig +from .multimodal import VideoURL __all__ = [ "BaseIOSchema", @@ -13,4 +14,5 @@ "BaseResourceConfig", "BasePrompt", "BasePromptConfig", + "VideoURL", ] diff --git a/atomic-agents/atomic_agents/base/multimodal.py b/atomic-agents/atomic_agents/base/multimodal.py new file mode 100644 index 00000000..e9e2df87 --- /dev/null +++ b/atomic-agents/atomic_agents/base/multimodal.py @@ -0,0 +1,35 @@ +"""Multimodal content types that Instructor does not provide.""" + +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field + + +class VideoURL(BaseModel): + """ + Video reference sent to the LLM as an OpenAI-compatible ``video_url`` content part. + + Instructor ships Image, Audio, and PDF types but no video type + (see https://github.com/567-labs/instructor/discussions/2520), so this class fills + the gap for providers that accept ``video_url`` content parts, such as MiniMax + and Qwen-VL. + + Attributes: + url (str): HTTP(S) or data: URL of the video. + fps (Optional[float]): Frame sampling rate, for providers that accept it. + detail (Optional[str]): Detail level, for providers that accept it. + """ + + url: str = Field(..., description="HTTP(S) or data: URL of the video.") + fps: Optional[float] = Field(default=None, description="Frame sampling rate, for providers that accept it.") + detail: Optional[str] = Field(default=None, description="Detail level, for providers that accept it.") + + def to_openai(self) -> Dict[str, Any]: + """ + Build the OpenAI-compatible content part for this video. + + Returns: + Dict[str, Any]: A ``{"type": "video_url", "video_url": {...}}`` content part, + omitting optional parameters that were not set. + """ + return {"type": "video_url", "video_url": self.model_dump(exclude_none=True)} diff --git a/atomic-agents/atomic_agents/context/chat_history.py b/atomic-agents/atomic_agents/context/chat_history.py index e4fbf91e..b679a74a 100644 --- a/atomic-agents/atomic_agents/context/chat_history.py +++ b/atomic-agents/atomic_agents/context/chat_history.py @@ -2,16 +2,17 @@ import uuid from enum import Enum from pathlib import Path -from typing import Dict, List, Optional, Type +from typing import Dict, List, Optional, Tuple, Type, Union from instructor.processing.multimodal import PDF, Image, Audio from pydantic import BaseModel, Field from atomic_agents.base.base_io_schema import BaseIOSchema +from atomic_agents.base.multimodal import VideoURL from atomic_agents.context.base_chat_history import BaseChatHistory -INSTRUCTOR_MULTIMODAL_TYPES = (Image, Audio, PDF) +MULTIMODAL_TYPES = (Image, Audio, PDF, VideoURL) class Message(BaseModel): @@ -109,10 +110,17 @@ def get_history(self) -> List[Dict]: if multimodal_objects: processed_content = [] + # Message.content is a BaseIOSchema, so extraction never excludes the whole object. + assert not isinstance(exclude_spec, bool) content_json = input_content.model_dump_json(exclude=exclude_spec) if content_json and content_json != "{}": processed_content.append(content_json) - processed_content.extend(multimodal_objects) + # Instructor forwards dict content parts to the provider unchanged but + # raises on object types it does not know, so VideoURL is converted here. + processed_content.extend( + multimodal_object.to_openai() if isinstance(multimodal_object, VideoURL) else multimodal_object + for multimodal_object in multimodal_objects + ) history.append({"role": message.role, "content": processed_content}) else: content_json = input_content.model_dump_json() @@ -121,13 +129,14 @@ def get_history(self) -> List[Dict]: return history @staticmethod - def _extract_multimodal_info(obj): + def _extract_multimodal_info(obj) -> Tuple[List[Union[Image, Audio, PDF, VideoURL]], Union[Dict, bool, None]]: """ Recursively extract multimodal objects and build a Pydantic-compatible exclude spec. - Walks the object tree to find all Instructor multimodal types (Image, Audio, PDF) - at any nesting depth, collecting them into a flat list and building an exclude - specification that can be passed to model_dump_json(exclude=...). + Walks the object tree to find all multimodal types (Instructor's Image, Audio, + and PDF, plus VideoURL) at any nesting depth, collecting them into a flat list + and building an exclude specification that can be passed to + model_dump_json(exclude=...). Args: obj: The object to inspect (BaseIOSchema, list, dict, or primitive). @@ -137,7 +146,7 @@ def _extract_multimodal_info(obj): - multimodal_objects: flat list of all multimodal objects found - exclude_spec: Pydantic exclude dict, True (exclude entirely), or None """ - if isinstance(obj, INSTRUCTOR_MULTIMODAL_TYPES): + if isinstance(obj, MULTIMODAL_TYPES): return [obj], True if hasattr(obj, "__class__") and hasattr(obj.__class__, "model_fields"): diff --git a/atomic-agents/tests/agents/test_atomic_agent.py b/atomic-agents/tests/agents/test_atomic_agent.py index c626791a..504cdd35 100644 --- a/atomic-agents/tests/agents/test_atomic_agent.py +++ b/atomic-agents/tests/agents/test_atomic_agent.py @@ -1120,6 +1120,18 @@ class MultimodalInputSchema(BaseIOSchema): assert image_entry["image_url"]["url"] == "https://example.com/test.png" +def test_serialize_history_for_token_count_video_placeholder(agent, mock_history): + video_part = {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}} + mock_history.get_history.return_value = [{"role": "user", "content": ['{"prompt":"Summarize the video"}', video_part]}] + + serialized = agent._serialize_history_for_token_count() + + assert serialized[0]["content"] == [ + {"type": "text", "text": '{"prompt":"Summarize the video"}'}, + {"type": "text", "text": "[video_url content]"}, + ] + + # --- Tests for tool_result_role and Gemini system message remapping (issue #221) --- diff --git a/atomic-agents/tests/base/test_multimodal.py b/atomic-agents/tests/base/test_multimodal.py new file mode 100644 index 00000000..28925841 --- /dev/null +++ b/atomic-agents/tests/base/test_multimodal.py @@ -0,0 +1,16 @@ +from atomic_agents import VideoURL + + +def test_to_openai_with_url_only(): + content_part = VideoURL(url="https://example.com/clip.mp4").to_openai() + + assert content_part == {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}} + + +def test_to_openai_with_sampling_parameters(): + content_part = VideoURL(url="https://example.com/clip.mp4", fps=1.0, detail="low").to_openai() + + assert content_part == { + "type": "video_url", + "video_url": {"url": "https://example.com/clip.mp4", "fps": 1.0, "detail": "low"}, + } diff --git a/atomic-agents/tests/context/test_chat_history.py b/atomic-agents/tests/context/test_chat_history.py index 4089aff2..df1c0bc6 100644 --- a/atomic-agents/tests/context/test_chat_history.py +++ b/atomic-agents/tests/context/test_chat_history.py @@ -6,7 +6,7 @@ from pathlib import Path from pydantic import Field from atomic_agents.context import ChatHistory, Message -from atomic_agents import BaseIOSchema +from atomic_agents import BaseIOSchema, VideoURL import instructor @@ -865,3 +865,40 @@ class DictImageSchema(BaseIOSchema): assert "image_map" not in json_part assert img_a in result[0]["content"] assert img_b in result[0]["content"] + + +class VideoMessageSchema(BaseIOSchema): + """Test schema with a video field""" + + prompt: str = Field(..., description="Instruction for the video") + video: VideoURL = Field(..., description="The video to analyze") + + +def test_get_history_video_url_becomes_content_part(history): + """VideoURL fields are separated from the JSON text as video_url content parts""" + content = VideoMessageSchema( + prompt="Summarize the video", + video=VideoURL(url="https://example.com/clip.mp4", fps=1.0), + ) + + history.add_message("user", content) + result = history.get_history() + + assert len(result) == 1 + assert json.loads(result[0]["content"][0]) == {"prompt": "Summarize the video"} + assert result[0]["content"][1] == { + "type": "video_url", + "video_url": {"url": "https://example.com/clip.mp4", "fps": 1.0}, + } + + +def test_dump_and_load_video_url(history): + """VideoURL round-trips through dump() and load() as a typed object""" + history.add_message("user", VideoMessageSchema(prompt="Summarize", video=VideoURL(url="https://example.com/clip.mp4"))) + + loaded_history = ChatHistory() + loaded_history.load(history.dump()) + + loaded_content = loaded_history.history[0].content + assert isinstance(loaded_content.video, VideoURL) + assert loaded_content.video.url == "https://example.com/clip.mp4" diff --git a/docs/guides/memory.md b/docs/guides/memory.md index d0bf563e..1fb32d4e 100644 --- a/docs/guides/memory.md +++ b/docs/guides/memory.md @@ -477,7 +477,8 @@ There's a full runnable example of this pattern (a dependency-free SQLite-backed ## Multimodal Content in History -ChatHistory supports images, PDFs, and audio through Instructor's multimodal types. +ChatHistory supports images, PDFs, and audio through Instructor's multimodal types, plus +video through the framework's own `VideoURL` type. ### Adding Multimodal Messages @@ -522,6 +523,29 @@ for message in history_data: json_content = message["content"] ``` +### Video + +Instructor has no video type yet, so Atomic Agents ships its own `VideoURL` for providers +that accept OpenAI-compatible `video_url` content parts (MiniMax, Qwen-VL, ...): + +```python +from atomic_agents import BaseIOSchema, VideoURL +from pydantic import Field + +class VideoAnalysisInput(BaseIOSchema): + """Input with a video for analysis""" + question: str = Field(..., description="Question about the video") + video: VideoURL = Field(..., description="Video to analyze") + +input_with_video = VideoAnalysisInput( + question="What happens in this clip?", + video=VideoURL(url="https://example.com/clip.mp4", fps=1.0), +) +``` + +`get_history()` emits the video as a `{"type": "video_url", ...}` content part, which +Instructor passes through to the provider unchanged. + ### Serialization with Multimodal ```{note}