diff --git a/docs/docs.json b/docs/docs.json index 741dadbb1..2dd23a6c7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -75,6 +75,7 @@ { "group": "Integrations", "pages": [ + "integrations/aimlapi", "integrations/langgraph-integration", "integrations/openenv-integration" ] diff --git a/docs/fundamentals/ruler.mdx b/docs/fundamentals/ruler.mdx index 9750a7df0..5aa4bb11a 100644 --- a/docs/fundamentals/ruler.mdx +++ b/docs/fundamentals/ruler.mdx @@ -167,6 +167,9 @@ Rank 3: Score 0.100 You can use any LLM supported by LiteLLM as the judge: ```python +# Using aimlapi.com — see the integration guide for setup and cost tracking +await ruler_score_group(group, "aiml/openai/gpt-5-5") + # Using o4-mini await ruler_score_group(group, "openai/o4-mini") @@ -177,6 +180,12 @@ await ruler_score_group(group, "anthropic/claude-sonnet-4-20250514") await ruler_score_group(group, "ollama/qwen3:32b") ``` + + Judging through a gateway such as [aimlapi.com](/integrations/aimlapi) gives + you one key for many judge models. Note that judge cost metrics need an extra + step for any provider prefix other than `openai` or `anthropic`. + + ### Extra LiteLLM Parameters You can pass additional parameters to LiteLLM for fine-tuning the judge behavior: diff --git a/docs/integrations/aimlapi.mdx b/docs/integrations/aimlapi.mdx new file mode 100644 index 000000000..8b4519b61 --- /dev/null +++ b/docs/integrations/aimlapi.mdx @@ -0,0 +1,129 @@ +--- +title: "aimlapi.com" +description: "Use AI/ML API as the LLM judge for RULER, through LiteLLM's aiml provider." +--- + +# aimlapi.com + +[AI/ML API](https://aimlapi.com) is an OpenAI-compatible gateway that serves a +few hundred chat models behind a single key and base URL. ART reaches it through +LiteLLM, which ships a first-class `aiml` provider, so nothing needs to be +installed or configured beyond an API key. + +This is useful when you want a strong judge model for +[RULER](/fundamentals/ruler) without holding a separate account with each model +vendor. + +## Setup + +Export your key. LiteLLM reads `AIML_API_KEY`: + +```bash +export AIML_API_KEY="..." +``` + + + The environment variable is `AIML_API_KEY`, not `AIMLAPI_API_KEY`. LiteLLM + chose the shorter spelling, and a wrong name surfaces only as a 401 on the + first judge call. + + +## Using AI/ML API as a RULER judge + +Prefix any AI/ML API model id with `aiml/` and pass it as the judge model: + +```python +import art +from art.rewards import ruler_score_group + +group = art.TrajectoryGroup([...]) + +judged_group = await ruler_score_group(group, "aiml/openai/gpt-5-5") +``` + +The same prefix works with the lower-level `ruler` function: + +```python +from art.rewards.ruler import ruler + +scores = await ruler(message_lists, judge_model="aiml/anthropic/claude-sonnet-4.6") +``` + +## Model ids + +AI/ML API ids are themselves namespaced (`openai/gpt-5-5`, +`anthropic/claude-sonnet-4.6`, `google/gemini-2.5-flash`), so a judge model +string carries two slashes: `aiml//`. Only the first segment is +consumed as the provider, and the rest is forwarded verbatim. + +Browse the catalog at `https://api.aimlapi.com/v1/models` (public, no key +required). Filter to models whose `type` is `openai/chat-completions` — the +catalog also contains image, video and audio models that cannot serve a judge +request. + + + Ids are also matched against each model's `aliases`, so a spelling that is + absent from the `id` field may still route. Add `?include=all` to the catalog + request to get `capabilities`, `modalities` and `pricing`, none of which + appear in the default response. + + +## Overriding the base URL + +The default base URL is `https://api.aimlapi.com/v1`. To point at a proxy or a +regional endpoint, set `AIML_API_BASE`, or pass it per call: + +```python +await ruler_score_group( + group, + "aiml/openai/gpt-5-5", + extra_litellm_params={"api_base": "https://your-proxy.example.com/v1"}, +) +``` + +## Cost tracking + +RULER records judge spend under `costs//judge/ruler`, but it can only +derive that number for providers it has a pricing path for. AI/ML API model ids +are not in LiteLLM's pricing map, so by default the judge runs correctly and the +cost metric is simply absent — `ruler` swallows the resulting `ValueError` on +purpose, because it supports local and custom LiteLLM models that have no +pricing. + +To record judge cost, register a cost extractor for the `aiml` provider on your +`MetricsBuilder`. Take the per-million rates from the `pricing` block of the +catalog entry for the model you are judging with: + +```python +PROMPT_PER_MILLION = ... # from the model's catalog pricing +COMPLETION_PER_MILLION = ... + + +def aiml_judge_cost(response) -> float | None: + usage = getattr(response, "usage", None) + if usage is None: + return None + prompt = float(getattr(usage, "prompt_tokens", 0) or 0) + completion = float(getattr(usage, "completion_tokens", 0) or 0) + return ( + prompt / 1_000_000 * PROMPT_PER_MILLION + + completion / 1_000_000 * COMPLETION_PER_MILLION + ) + + +metrics_builder.register_cost_extractor("aiml", aiml_judge_cost) +``` + + + `register_model_pricing` alone is not enough here. Cost is estimated from + token counts only for the `openai` and `anthropic` provider prefixes, so a + judge model reached through any other provider — `aiml`, `groq`, `together`, + `ollama` — needs the extractor above. This applies to AI/ML API only because + of the provider prefix; the responses themselves are OpenAI-shaped. + + + + On some models `completion_tokens` excludes reasoning tokens, so an extractor + built on `completion_tokens` alone can under-report spend for reasoning + models. Compare against `total_tokens` if that matters for your budget. + diff --git a/tests/unit/test_aimlapi_litellm_route.py b/tests/unit/test_aimlapi_litellm_route.py new file mode 100644 index 000000000..0c4c17d01 --- /dev/null +++ b/tests/unit/test_aimlapi_litellm_route.py @@ -0,0 +1,55 @@ +"""Guard the AI/ML API (`aiml`) judge route that RULER depends on. + +RULER delegates every judge call to LiteLLM (`art.rewards.ruler.ruler` -> +`litellm.acompletion`), so AI/ML API support is not ART code — it is whatever +the pinned LiteLLM resolves for the `aiml/` prefix. `pyproject.toml` pins +`litellm>=1.71.1,<=1.82.0`, and the route silently disappearing on a bump would +turn `judge_model="aiml/..."` into a confusing "LLM Provider NOT provided" +error at training time rather than at import time. These assertions are cheap +and require neither network access nor a key. +""" + +import litellm +from litellm.utils import get_llm_provider +import pytest + +from art.rewards.ruler import _judge_provider + +AIML_PROVIDER = "aiml" +AIML_API_BASE = "https://api.aimlapi.com/v1" + + +def test_aiml_is_a_known_litellm_provider() -> None: + assert AIML_PROVIDER in litellm.provider_list + assert AIML_PROVIDER in litellm.openai_compatible_providers + + +@pytest.mark.parametrize( + "judge_model, expected_model", + [ + ("aiml/openai/gpt-5-5", "openai/gpt-5-5"), + ("aiml/anthropic/claude-sonnet-4.6", "anthropic/claude-sonnet-4.6"), + ("aiml/google/gemini-2.5-flash", "google/gemini-2.5-flash"), + ], +) +def test_aiml_judge_model_resolves_to_the_aimlapi_chat_endpoint( + judge_model: str, expected_model: str, monkeypatch: pytest.MonkeyPatch +) -> None: + # AI/ML API model ids contain slashes of their own, so the prefix must be + # stripped exactly once and the remainder forwarded verbatim. + monkeypatch.setenv("AIML_API_KEY", "sentinel-not-a-real-key") + model, provider, api_key, api_base = get_llm_provider(model=judge_model) + + assert model == expected_model + assert provider == AIML_PROVIDER + assert api_base == AIML_API_BASE + # LiteLLM reads AIML_API_KEY, not the AIMLAPI_API_KEY used elsewhere in the + # ecosystem; getting this wrong surfaces only as a 401 on the first call. + assert api_key == "sentinel-not-a-real-key" + + +def test_ruler_attributes_aiml_judges_to_the_aiml_provider() -> None: + # `_judge_provider` splits on the first "/" only, which is what keeps + # multi-segment AI/ML API ids intact for cost attribution. + assert _judge_provider("aiml/openai/gpt-5-5") == AIML_PROVIDER + assert _judge_provider("AIML/openai/gpt-5-5") == AIML_PROVIDER