From 367181aef176ff6e003bd37916b2a100ec4f077c Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:43:57 +0500 Subject: [PATCH 1/8] refactor(providers): extract shared ollama payload builder --- src/anycode/providers/ollama.py | 42 +++++++++++++-------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index 82155c7..7880934 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -36,7 +36,7 @@ def _ensure_httpx() -> None: class OllamaAdapter: - """HTTP-based adapter for Ollama — uses the OpenAI-compatible chat endpoint.""" + """HTTP-based adapter for Ollama's native ``/api/chat`` endpoint.""" def __init__(self, base_url: str | None = None, model: str | None = None) -> None: _ensure_httpx() @@ -47,24 +47,25 @@ def __init__(self, base_url: str | None = None, model: str | None = None) -> Non def name(self) -> str: return "ollama" - async def chat( - self, - messages: list[LLMMessage], - options: LLMChatOptions, - ) -> LLMResponse: - model = self._default_model or options.model - oai_msgs = map_messages(messages, options.system_prompt) - + def _build_payload(self, messages: list[LLMMessage], options: LLMChatOptions, *, stream: bool) -> dict[str, Any]: payload: dict[str, Any] = { - "model": model, - "messages": oai_msgs, - "stream": False, + "model": self._default_model or options.model, + "messages": map_messages(messages, options.system_prompt), + "stream": stream, } - if options.tools: payload["tools"] = [map_tool_def(t) for t in options.tools] if options.temperature is not None: payload.setdefault("options", {})["temperature"] = options.temperature + return payload + + async def chat( + self, + messages: list[LLMMessage], + options: LLMChatOptions, + ) -> LLMResponse: + payload = self._build_payload(messages, options, stream=False) + model = payload["model"] async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT) as client: resp = await client.post(f"{self._base_url}/api/chat", json=payload) @@ -99,19 +100,8 @@ async def chat( ) async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> AsyncIterator[StreamEvent]: - model = self._default_model or options.model - oai_msgs = map_messages(messages, options.system_prompt) - - payload: dict[str, Any] = { - "model": model, - "messages": oai_msgs, - "stream": True, - } - - if options.tools: - payload["tools"] = [map_tool_def(t) for t in options.tools] - if options.temperature is not None: - payload.setdefault("options", {})["temperature"] = options.temperature + payload = self._build_payload(messages, options, stream=True) + model = payload["model"] full_text = "" tool_blocks: list[ToolUseBlock] = [] From 9e481d7926b4807df7accea71255cfafd6ce435d Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:45:28 +0500 Subject: [PATCH 2/8] fix(providers): send images to ollama as native base64 arrays --- src/anycode/providers/ollama.py | 30 +++++++++++++++++++++++++- tests/test_providers.py | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index 7880934..ba015be 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -35,6 +35,34 @@ def _ensure_httpx() -> None: raise ImportError('httpx is required for the Ollama provider. Install it with: pip install "anycode-py[ollama]"') +def _to_native_messages(oai_msgs: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Flatten OpenAI-style content-part lists into Ollama's native message shape. + + The native ``/api/chat`` endpoint takes ``content`` as a plain string plus an + optional per-message ``images`` array of base64 payloads; it does not accept + ``image_url`` parts. + """ + native: list[dict[str, Any]] = [] + for msg in oai_msgs: + content = msg.get("content") + if not isinstance(content, list): + native.append(msg) + continue + texts: list[str] = [] + images: list[str] = [] + for part in content: + if part.get("type") == "text": + texts.append(part.get("text", "")) + elif part.get("type") == "image_url": + url = part.get("image_url", {}).get("url", "") + images.append(url.split(",", 1)[1] if "," in url else url) + flattened = {**msg, "content": "".join(texts)} + if images: + flattened["images"] = images + native.append(flattened) + return native + + class OllamaAdapter: """HTTP-based adapter for Ollama's native ``/api/chat`` endpoint.""" @@ -50,7 +78,7 @@ def name(self) -> str: def _build_payload(self, messages: list[LLMMessage], options: LLMChatOptions, *, stream: bool) -> dict[str, Any]: payload: dict[str, Any] = { "model": self._default_model or options.model, - "messages": map_messages(messages, options.system_prompt), + "messages": _to_native_messages(map_messages(messages, options.system_prompt)), "stream": stream, } if options.tools: diff --git a/tests/test_providers.py b/tests/test_providers.py index 8a2e588..660d437 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -16,6 +16,8 @@ from anycode.providers.ollama import OllamaAdapter from anycode.providers.openai import OpenAIAdapter from anycode.types import ( + ImageBlock, + ImageSource, LLMChatOptions, LLMMessage, LLMResponse, @@ -254,7 +256,42 @@ async def test_system_prompt_passed(self) -> None: # --------------------------------------------------------------------------- +def _ollama_chat_client(response_data: dict) -> AsyncMock: + """Mock httpx.AsyncClient returning a single non-streaming chat response.""" + mock_resp = MagicMock() + mock_resp.json.return_value = response_data + mock_resp.raise_for_status = MagicMock() + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.post = AsyncMock(return_value=mock_resp) + return mock_client + + class TestOllamaAdapter: + async def test_chat_sends_native_base64_images(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "A cat."}}) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="llava") + messages = [ + LLMMessage( + role="user", + content=[ + TextBlock(text="What is in this image?"), + ImageBlock(source=ImageSource(media_type="image/png", data="aGVsbG8=")), + ], + ) + ] + await adapter.chat(messages, options) + + payload = mock_client.post.call_args.kwargs["json"] + sent = payload["messages"][0] + assert sent["content"] == "What is in this image?" + assert sent["images"] == ["aGVsbG8="] + assert "image_url" not in json.dumps(payload) + async def test_chat_returns_llmresponse(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") response_data = { From 3365932da3eadce11b5a7198e10bc6d6b2a7cffa Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:46:43 +0500 Subject: [PATCH 3/8] feat(providers): forward sampling options and keep_alive to ollama --- src/anycode/providers/ollama.py | 20 ++++++++++++++++++-- tests/test_providers.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index ba015be..5c0096d 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -66,10 +66,19 @@ def _to_native_messages(oai_msgs: list[dict[str, Any]]) -> list[dict[str, Any]]: class OllamaAdapter: """HTTP-based adapter for Ollama's native ``/api/chat`` endpoint.""" - def __init__(self, base_url: str | None = None, model: str | None = None) -> None: + def __init__( + self, + base_url: str | None = None, + model: str | None = None, + *, + keep_alive: str | float | None = None, + default_options: dict[str, Any] | None = None, + ) -> None: _ensure_httpx() self._base_url = (base_url or OLLAMA_DEFAULT_BASE_URL).rstrip("/") self._default_model = model + self._keep_alive = keep_alive + self._default_options = dict(default_options) if default_options else {} @property def name(self) -> str: @@ -83,8 +92,15 @@ def _build_payload(self, messages: list[LLMMessage], options: LLMChatOptions, *, } if options.tools: payload["tools"] = [map_tool_def(t) for t in options.tools] + model_options: dict[str, Any] = dict(self._default_options) + if options.max_tokens is not None: + model_options["num_predict"] = options.max_tokens if options.temperature is not None: - payload.setdefault("options", {})["temperature"] = options.temperature + model_options["temperature"] = options.temperature + if model_options: + payload["options"] = model_options + if self._keep_alive is not None: + payload["keep_alive"] = self._keep_alive return payload async def chat( diff --git a/tests/test_providers.py b/tests/test_providers.py index 660d437..6030339 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -292,6 +292,35 @@ async def test_chat_sends_native_base64_images(self) -> None: assert sent["images"] == ["aGVsbG8="] assert "image_url" not in json.dumps(payload) + async def test_chat_forwards_sampling_options_and_keep_alive(self) -> None: + adapter = OllamaAdapter( + base_url="http://localhost:11434", + keep_alive="5m", + default_options={"seed": 7, "top_p": 0.9, "num_predict": 1}, + ) + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "ok"}}) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="llama3.3:70b", max_tokens=256, temperature=0.2) + await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="Hi")])], options) + + payload = mock_client.post.call_args.kwargs["json"] + assert payload["keep_alive"] == "5m" + # Per-call options override constructor defaults; untouched defaults pass through. + assert payload["options"] == {"seed": 7, "top_p": 0.9, "num_predict": 256, "temperature": 0.2} + + async def test_chat_omits_options_and_keep_alive_by_default(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "ok"}}) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="llama3.3:70b") + await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="Hi")])], options) + + payload = mock_client.post.call_args.kwargs["json"] + assert "options" not in payload + assert "keep_alive" not in payload + async def test_chat_returns_llmresponse(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") response_data = { From 2516e41f6d389d4accf8312ae79ba88b457d251d Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:48:35 +0500 Subject: [PATCH 4/8] feat(providers): add ollama thinking support --- src/anycode/providers/ollama.py | 26 ++++++++++++ tests/test_providers.py | 70 +++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index 5c0096d..a8a747a 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -18,6 +18,7 @@ LLMStreamOptions, StreamEvent, TextBlock, + ThinkingBlock, TokenUsage, ToolUseBlock, ) @@ -29,6 +30,9 @@ OLLAMA_REQUEST_TIMEOUT = OLLAMA_REQUEST_TIMEOUT_S +# Ollama's `think` levels; `minimal` has no native equivalent and maps to "low". +_EFFORT_TO_THINK = {"minimal": "low", "low": "low", "medium": "medium", "high": "high"} + def _ensure_httpx() -> None: if httpx is None: @@ -72,12 +76,14 @@ def __init__( model: str | None = None, *, keep_alive: str | float | None = None, + think: bool | str | None = None, default_options: dict[str, Any] | None = None, ) -> None: _ensure_httpx() self._base_url = (base_url or OLLAMA_DEFAULT_BASE_URL).rstrip("/") self._default_model = model self._keep_alive = keep_alive + self._think = think self._default_options = dict(default_options) if default_options else {} @property @@ -101,8 +107,19 @@ def _build_payload(self, messages: list[LLMMessage], options: LLMChatOptions, *, payload["options"] = model_options if self._keep_alive is not None: payload["keep_alive"] = self._keep_alive + think = self._resolve_think(options) + if think is not None: + payload["think"] = think return payload + def _resolve_think(self, options: LLMChatOptions) -> bool | str | None: + if options.reasoning_effort is not None: + return _EFFORT_TO_THINK[options.reasoning_effort] + if options.thinking_budget_tokens is not None: + # Ollama has no budget concept; any budget request enables thinking. + return True + return self._think + async def chat( self, messages: list[LLMMessage], @@ -119,6 +136,8 @@ async def chat( content: list[ContentBlock] = [] msg_data = data.get("message", {}) + if msg_data.get("thinking"): + content.append(ThinkingBlock(thinking=msg_data["thinking"])) if msg_data.get("content"): content.append(TextBlock(text=msg_data["content"])) @@ -148,6 +167,7 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> model = payload["model"] full_text = "" + full_thinking = "" tool_blocks: list[ToolUseBlock] = [] input_tokens = 0 output_tokens = 0 @@ -170,6 +190,10 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> output_tokens = chunk["eval_count"] msg_data = chunk.get("message", {}) + if msg_data.get("thinking"): + thinking_text = msg_data["thinking"] + full_thinking += thinking_text + yield StreamEvent(type="thinking", data=thinking_text) if msg_data.get("content"): text = msg_data["content"] full_text += text @@ -186,6 +210,8 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> yield StreamEvent(type="tool_use", data=block) done_content: list[ContentBlock] = [] + if full_thinking: + done_content.append(ThinkingBlock(thinking=full_thinking)) if full_text: done_content.append(TextBlock(text=full_text)) done_content.extend(tool_blocks) diff --git a/tests/test_providers.py b/tests/test_providers.py index 6030339..0e5b28a 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -21,8 +21,10 @@ LLMChatOptions, LLMMessage, LLMResponse, + LLMStreamOptions, LLMToolDef, TextBlock, + ThinkingBlock, ToolResultBlock, ToolUseBlock, ) @@ -268,6 +270,26 @@ def _ollama_chat_client(response_data: dict) -> AsyncMock: return mock_client +def _ollama_stream_client(lines: list[str]) -> AsyncMock: + """Mock httpx.AsyncClient streaming NDJSON lines from /api/chat.""" + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + + async def aiter_lines(): + for line in lines: + yield line + + mock_resp.aiter_lines = aiter_lines + stream_cm = MagicMock() + stream_cm.__aenter__ = AsyncMock(return_value=mock_resp) + stream_cm.__aexit__ = AsyncMock(return_value=False) + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.stream = MagicMock(return_value=stream_cm) + return mock_client + + class TestOllamaAdapter: async def test_chat_sends_native_base64_images(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") @@ -321,6 +343,54 @@ async def test_chat_omits_options_and_keep_alive_by_default(self) -> None: assert "options" not in payload assert "keep_alive" not in payload + async def test_chat_maps_reasoning_effort_to_think_and_parses_thinking(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "4", "thinking": "2+2 is 4."}, "eval_count": 3}) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="qwen3", reasoning_effort="high") + result = await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="2+2?")])], options) + + payload = mock_client.post.call_args.kwargs["json"] + assert payload["think"] == "high" + assert isinstance(result.content[0], ThinkingBlock) + assert result.content[0].thinking == "2+2 is 4." + assert isinstance(result.content[1], TextBlock) + assert result.content[1].text == "4" + + async def test_think_resolution_order(self) -> None: + adapter = OllamaAdapter(think="max") + assert adapter._resolve_think(LLMChatOptions(model="m")) == "max" + assert adapter._resolve_think(LLMChatOptions(model="m", thinking_budget_tokens=1024)) is True + assert adapter._resolve_think(LLMChatOptions(model="m", reasoning_effort="minimal")) == "low" + assert OllamaAdapter()._resolve_think(LLMChatOptions(model="m")) is None + + async def test_stream_yields_thinking_events(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + lines = [ + json.dumps({"message": {"role": "assistant", "thinking": "Consider"}}), + json.dumps({"message": {"role": "assistant", "thinking": " carefully."}}), + json.dumps({"message": {"role": "assistant", "content": "Answer"}}), + json.dumps({"message": {"role": "assistant", "content": ""}, "done": True, "prompt_eval_count": 5, "eval_count": 9}), + ] + mock_client = _ollama_stream_client(lines) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMStreamOptions(model="qwen3") + events = [e async for e in adapter.stream([LLMMessage(role="user", content=[TextBlock(text="Q")])], options)] + + thinking_events = [e for e in events if e.type == "thinking"] + assert [e.data for e in thinking_events] == ["Consider", " carefully."] + done = events[-1] + assert done.type == "done" + blocks = done.data.content + assert isinstance(blocks[0], ThinkingBlock) + assert blocks[0].thinking == "Consider carefully." + assert isinstance(blocks[1], TextBlock) + assert blocks[1].text == "Answer" + assert done.data.usage.input_tokens == 5 + assert done.data.usage.output_tokens == 9 + async def test_chat_returns_llmresponse(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") response_data = { From aa5e11c9d351359c96a789482a76d782dbab16a5 Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:49:36 +0500 Subject: [PATCH 5/8] feat(providers): add ollama structured outputs --- src/anycode/providers/ollama.py | 14 ++++++++++++++ tests/test_providers.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index a8a747a..a44b9d3 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -120,12 +120,26 @@ def _resolve_think(self, options: LLMChatOptions) -> bool | str | None: return True return self._think + @staticmethod + def _to_native_format(response_format: dict[str, Any]) -> str | dict[str, Any]: + """Translate an OpenAI-style response_format into Ollama's ``format`` value.""" + kind = response_format.get("type") + if kind == "json_object": + return "json" + if kind == "json_schema": + return response_format.get("json_schema", {}).get("schema", {}) + return response_format + async def chat( self, messages: list[LLMMessage], options: LLMChatOptions, + *, + response_format: dict[str, Any] | None = None, ) -> LLMResponse: payload = self._build_payload(messages, options, stream=False) + if response_format: + payload["format"] = self._to_native_format(response_format) model = payload["model"] async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT) as client: diff --git a/tests/test_providers.py b/tests/test_providers.py index 0e5b28a..90308d6 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -391,6 +391,28 @@ async def test_stream_yields_thinking_events(self) -> None: assert done.data.usage.input_tokens == 5 assert done.data.usage.output_tokens == 9 + @pytest.mark.parametrize( + ("response_format", "expected"), + [ + ({"type": "json_object"}, "json"), + ( + {"type": "json_schema", "json_schema": {"name": "out", "schema": {"type": "object", "properties": {"x": {"type": "integer"}}}}}, + {"type": "object", "properties": {"x": {"type": "integer"}}}, + ), + ({"type": "object", "properties": {"y": {"type": "string"}}}, {"type": "object", "properties": {"y": {"type": "string"}}}), + ], + ) + async def test_chat_translates_response_format(self, response_format: dict, expected: object) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "{}"}}) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="llama3.3:70b") + await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="Hi")])], options, response_format=response_format) + + payload = mock_client.post.call_args.kwargs["json"] + assert payload["format"] == expected + async def test_chat_returns_llmresponse(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") response_data = { From f64b68d0b0bbb1f6bd5d2e5f3f2d7ae2aa642b55 Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:50:48 +0500 Subject: [PATCH 6/8] feat(providers): support ollama cloud api keys --- src/anycode/providers/adapter.py | 2 +- src/anycode/providers/ollama.py | 13 +++++++++++-- tests/test_providers.py | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/anycode/providers/adapter.py b/src/anycode/providers/adapter.py index ef9a489..258500f 100644 --- a/src/anycode/providers/adapter.py +++ b/src/anycode/providers/adapter.py @@ -90,7 +90,7 @@ async def _create_raw_adapter( elif provider == PROVIDER_OLLAMA: from anycode.providers.ollama import OllamaAdapter - return OllamaAdapter(base_url=base_url, model=model) + return OllamaAdapter(base_url=base_url, model=model, api_key=api_key) elif provider == PROVIDER_BEDROCK: from anycode.providers.bedrock import BedrockAdapter diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index a44b9d3..a26644f 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import uuid from collections.abc import AsyncIterator from typing import Any @@ -75,6 +76,7 @@ def __init__( base_url: str | None = None, model: str | None = None, *, + api_key: str | None = None, keep_alive: str | float | None = None, think: bool | str | None = None, default_options: dict[str, Any] | None = None, @@ -82,10 +84,17 @@ def __init__( _ensure_httpx() self._base_url = (base_url or OLLAMA_DEFAULT_BASE_URL).rstrip("/") self._default_model = model + self._api_key = api_key or os.environ.get("OLLAMA_API_KEY") self._keep_alive = keep_alive self._think = think self._default_options = dict(default_options) if default_options else {} + def _headers(self) -> dict[str, str]: + # ollama.com cloud requires a Bearer key; local servers ignore the header. + if self._api_key: + return {"Authorization": f"Bearer {self._api_key}"} + return {} + @property def name(self) -> str: return "ollama" @@ -142,7 +151,7 @@ async def chat( payload["format"] = self._to_native_format(response_format) model = payload["model"] - async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT) as client: + async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT, headers=self._headers()) as client: resp = await client.post(f"{self._base_url}/api/chat", json=payload) resp.raise_for_status() data = resp.json() @@ -187,7 +196,7 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> output_tokens = 0 try: - async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT) as client: + async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT, headers=self._headers()) as client: async with client.stream("POST", f"{self._base_url}/api/chat", json=payload) as resp: resp.raise_for_status() async for line in resp.aiter_lines(): diff --git a/tests/test_providers.py b/tests/test_providers.py index 90308d6..e2f757c 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -413,6 +413,23 @@ async def test_chat_translates_response_format(self, response_format: dict, expe payload = mock_client.post.call_args.kwargs["json"] assert payload["format"] == expected + async def test_chat_sends_bearer_header_for_cloud_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OLLAMA_API_KEY", raising=False) + adapter = OllamaAdapter(base_url="https://ollama.com", api_key="sk-cloud") + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "ok"}}) + + with patch("httpx.AsyncClient", return_value=mock_client) as client_cls: + options = LLMChatOptions(model="gpt-oss:120b") + await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="Hi")])], options) + + assert client_cls.call_args.kwargs["headers"] == {"Authorization": "Bearer sk-cloud"} + + async def test_api_key_falls_back_to_env_and_defaults_to_none(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OLLAMA_API_KEY", "sk-env") + assert OllamaAdapter()._headers() == {"Authorization": "Bearer sk-env"} + monkeypatch.delenv("OLLAMA_API_KEY") + assert OllamaAdapter()._headers() == {} + async def test_chat_returns_llmresponse(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") response_data = { From 4b6388e934b7cea762fe03431e5b9367cf686ecd Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:55:26 +0500 Subject: [PATCH 7/8] fix(providers): map ollama done_reason and surface stream errors --- src/anycode/providers/ollama.py | 45 ++++++++++++++++++++--- tests/test_providers.py | 65 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/src/anycode/providers/ollama.py b/src/anycode/providers/ollama.py index a26644f..5e0bae7 100644 --- a/src/anycode/providers/ollama.py +++ b/src/anycode/providers/ollama.py @@ -8,7 +8,13 @@ from collections.abc import AsyncIterator from typing import Any -from anycode.constants import OLLAMA_DEFAULT_BASE_URL, OLLAMA_REQUEST_TIMEOUT_S, STOP_REASON_END_TURN, STOP_REASON_TOOL_USE +from anycode.constants import ( + OLLAMA_DEFAULT_BASE_URL, + OLLAMA_REQUEST_TIMEOUT_S, + STOP_REASON_END_TURN, + STOP_REASON_MAX_TOKENS, + STOP_REASON_TOOL_USE, +) from anycode.providers._openai_compat import map_messages, map_tool_def, parse_json_safe from anycode.security.redaction import safe_exception_message from anycode.types import ( @@ -34,6 +40,14 @@ # Ollama's `think` levels; `minimal` has no native equivalent and maps to "low". _EFFORT_TO_THINK = {"minimal": "low", "low": "low", "medium": "medium", "high": "high"} +_DONE_REASON_MAP = {"stop": STOP_REASON_END_TURN, "length": STOP_REASON_MAX_TOKENS} + + +def _map_done_reason(done_reason: str | None, *, has_tool_calls: bool) -> str: + if has_tool_calls: + return STOP_REASON_TOOL_USE + return _DONE_REASON_MAP.get(done_reason or "stop", STOP_REASON_END_TURN) + def _ensure_httpx() -> None: if httpx is None: @@ -95,6 +109,18 @@ def _headers(self) -> dict[str, str]: return {"Authorization": f"Bearer {self._api_key}"} return {} + def _raise_for_status(self, resp: Any, model: str | None) -> None: + try: + resp.raise_for_status() + except httpx.HTTPStatusError as e: + if e.response.status_code == 404: + raise httpx.HTTPStatusError( + f"Model '{model}' was not found on the Ollama server at {self._base_url}. Pull it first: ollama pull {model}", + request=e.request, + response=e.response, + ) from e + raise + @property def name(self) -> str: return "ollama" @@ -153,7 +179,7 @@ async def chat( async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT, headers=self._headers()) as client: resp = await client.post(f"{self._base_url}/api/chat", json=payload) - resp.raise_for_status() + self._raise_for_status(resp, model) data = resp.json() content: list[ContentBlock] = [] @@ -172,7 +198,7 @@ async def chat( tool_id = f"call_{uuid.uuid4().hex[:24]}" content.append(ToolUseBlock(id=tool_id, name=func.get("name", ""), input=args)) - stop_reason = STOP_REASON_TOOL_USE if msg_data.get("tool_calls") else STOP_REASON_END_TURN + stop_reason = _map_done_reason(data.get("done_reason"), has_tool_calls=bool(msg_data.get("tool_calls"))) input_tokens = data.get("prompt_eval_count", 0) output_tokens = data.get("eval_count", 0) @@ -194,11 +220,12 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> tool_blocks: list[ToolUseBlock] = [] input_tokens = 0 output_tokens = 0 + done_reason: str | None = None try: async with httpx.AsyncClient(timeout=OLLAMA_REQUEST_TIMEOUT, headers=self._headers()) as client: async with client.stream("POST", f"{self._base_url}/api/chat", json=payload) as resp: - resp.raise_for_status() + self._raise_for_status(resp, model) async for line in resp.aiter_lines(): if not line.strip(): continue @@ -207,6 +234,14 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> except json.JSONDecodeError: continue + # Ollama reports mid-stream failures as NDJSON error objects + # on an already-established 200 response. + if chunk.get("error"): + yield StreamEvent(type="error", data=str(chunk["error"])) + return + + if chunk.get("done_reason"): + done_reason = chunk["done_reason"] if chunk.get("prompt_eval_count"): input_tokens = chunk["prompt_eval_count"] if chunk.get("eval_count"): @@ -239,7 +274,7 @@ async def stream(self, messages: list[LLMMessage], options: LLMStreamOptions) -> done_content.append(TextBlock(text=full_text)) done_content.extend(tool_blocks) - stop = STOP_REASON_TOOL_USE if tool_blocks else STOP_REASON_END_TURN + stop = _map_done_reason(done_reason, has_tool_calls=bool(tool_blocks)) yield StreamEvent( type="done", diff --git a/tests/test_providers.py b/tests/test_providers.py index e2f757c..afe1ab3 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -430,6 +430,71 @@ async def test_api_key_falls_back_to_env_and_defaults_to_none(self, monkeypatch: monkeypatch.delenv("OLLAMA_API_KEY") assert OllamaAdapter()._headers() == {} + async def test_chat_maps_done_reason_length_to_max_tokens(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + mock_client = _ollama_chat_client({"message": {"role": "assistant", "content": "truncat"}, "done_reason": "length"}) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="llama3.3:70b", max_tokens=8) + result = await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="Hi")])], options) + + assert result.stop_reason == "max_tokens" + + async def test_chat_404_includes_pull_guidance(self) -> None: + import httpx + + adapter = OllamaAdapter(base_url="http://localhost:11434") + request = httpx.Request("POST", "http://localhost:11434/api/chat") + response = httpx.Response(status_code=404, request=request) + mock_resp = MagicMock() + mock_resp.raise_for_status.side_effect = httpx.HTTPStatusError("404", request=request, response=response) + mock_client = AsyncMock() + mock_client.__aenter__ = AsyncMock(return_value=mock_client) + mock_client.__aexit__ = AsyncMock(return_value=False) + mock_client.post = AsyncMock(return_value=mock_resp) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMChatOptions(model="missing-model") + with pytest.raises(httpx.HTTPStatusError, match="ollama pull missing-model"): + await adapter.chat([LLMMessage(role="user", content=[TextBlock(text="Hi")])], options) + + async def test_stream_surfaces_mid_stream_error(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + lines = [ + json.dumps({"message": {"role": "assistant", "content": "partial"}}), + json.dumps({"error": "an error was encountered while running the model"}), + json.dumps({"message": {"role": "assistant", "content": "never seen"}}), + ] + mock_client = _ollama_stream_client(lines) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMStreamOptions(model="llama3.3:70b") + events = [e async for e in adapter.stream([LLMMessage(role="user", content=[TextBlock(text="Q")])], options)] + + assert [e.type for e in events] == ["text", "error"] + assert "error was encountered" in events[-1].data + + async def test_stream_accumulates_tool_calls(self) -> None: + adapter = OllamaAdapter(base_url="http://localhost:11434") + tool_chunk = { + "message": {"role": "assistant", "content": "", "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "NYC"}}}]} + } + final_chunk = {"message": {"role": "assistant", "content": ""}, "done": True, "done_reason": "stop", "prompt_eval_count": 4, "eval_count": 2} + lines = [json.dumps(tool_chunk), json.dumps(final_chunk)] + mock_client = _ollama_stream_client(lines) + + with patch("httpx.AsyncClient", return_value=mock_client): + options = LLMStreamOptions(model="llama3.3:70b") + events = [e async for e in adapter.stream([LLMMessage(role="user", content=[TextBlock(text="Weather?")])], options)] + + tool_events = [e for e in events if e.type == "tool_use"] + assert len(tool_events) == 1 + assert tool_events[0].data.name == "get_weather" + done = events[-1] + assert done.type == "done" + assert done.data.stop_reason == "tool_use" + assert done.data.usage.input_tokens == 4 + async def test_chat_returns_llmresponse(self) -> None: adapter = OllamaAdapter(base_url="http://localhost:11434") response_data = { From 59fbe462aef4056d1f1cff08d210d0ed8e021eb3 Mon Sep 17 00:00:00 2001 From: Ali Muhammad Date: Fri, 24 Jul 2026 06:58:00 +0500 Subject: [PATCH 8/8] docs: document robust ollama integration --- CHANGELOG.md | 8 ++++++++ site_docs/guides/providers.md | 23 ++++++++++++++++++++++- site_docs/guides/reasoning-models.md | 10 +++++++--- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8189a2..3409caf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Robust Ollama integration** - the Ollama adapter now supports thinking (`reasoning_effort`/`thinking_budget_tokens` map to the native `think` parameter, with `ThinkingBlock` content and `thinking` stream events), structured outputs via `chat(..., response_format=...)` translated to Ollama's `format`, sampling options (`max_tokens` → `num_predict`, plus a `default_options` passthrough for `seed`, `top_p`, `num_ctx`, and friends), `keep_alive`, and ollama.com cloud authentication through `OLLAMA_API_KEY` or `api_key=`. + +### Fixed + +- **Ollama image input and error reporting** - image blocks are now sent as the native base64 `images` array (previously an unsupported `image_url` shape the server ignored), mid-stream NDJSON error objects surface as terminal `error` stream events instead of a silently truncated answer, `done_reason: "length"` maps to `stop_reason="max_tokens"`, and a `404` names the missing model with the exact `ollama pull` command. + ## [0.8.2] - 2026-07-19 ### Changed diff --git a/site_docs/guides/providers.md b/site_docs/guides/providers.md index 7db74a4..b21f576 100644 --- a/site_docs/guides/providers.md +++ b/site_docs/guides/providers.md @@ -15,7 +15,7 @@ AnyCode is provider-agnostic: every agent names a `provider` and a `model`, and | `anthropic` | `claude-haiku-4-5`, `claude-sonnet-5` | `ANTHROPIC_API_KEY` | `anycode-py[anthropic]` | | `openai` | `gpt-4o-mini`, `gpt-5` | `OPENAI_API_KEY` | `anycode-py[openai]` | | `google` | `gemini-2.5-flash` | `GOOGLE_API_KEY` or `GEMINI_API_KEY` | `anycode-py[google]` | -| `ollama` | any local model (e.g. `llama3.1`) | none (local server) | `anycode-py[ollama]` | +| `ollama` | any local model (e.g. `llama3.1`), or cloud models on ollama.com | none locally; `OLLAMA_API_KEY` for ollama.com | `anycode-py[ollama]` | | `bedrock` | AWS-hosted model IDs | AWS creds + `AWS_DEFAULT_REGION` | `anycode-py[bedrock]` | | `azure` | your deployment name | `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_API_KEY` | `anycode-py[azure]` | @@ -110,6 +110,27 @@ print(result.content[0].text, result.usage.input_tokens) | `endpoint`, `api_version` | azure | Azure resource endpoint and API version | | `region`, `profile` | bedrock | AWS region and named profile | +### Ollama: local and cloud + +The Ollama adapter speaks the native `/api/chat` endpoint and covers streaming, tool calling, thinking, structured outputs, and multimodal image input. A few knobs are adapter-specific, so construct `OllamaAdapter` directly when you need them: + +```python title="ollama_tuned.py" +from anycode.providers.ollama import OllamaAdapter + +adapter = OllamaAdapter( + model="qwen3", + keep_alive="10m", # keep the model loaded between calls + think="high", # default thinking level (bool or low/medium/high/max) + default_options={"num_ctx": 8192, "seed": 7}, # any native Ollama option +) +``` + +- `LLMChatOptions.max_tokens` maps to Ollama's `num_predict`; `temperature` passes through. Per-call options always override `default_options`. +- `reasoning_effort` maps to the `think` level and `message.thinking` comes back as `ThinkingBlock` content and `thinking` stream events — see [Use reasoning models](reasoning-models.md). +- `chat(..., response_format=...)` translates to Ollama's `format` parameter (`"json"` or a JSON schema). +- Image blocks are sent as native base64 `images` arrays for multimodal models such as `llava`. +- For [Ollama cloud](https://docs.ollama.com/cloud), set `base_url="https://ollama.com"` and export `OLLAMA_API_KEY` (or pass `api_key=`); the adapter sends the `Authorization: Bearer` header. A `404` from any server raises an error that names the missing model and the exact `ollama pull` command, and mid-stream server failures surface as terminal `error` stream events instead of a silently truncated answer. + ## Resilience is on by default Every adapter from `create_adapter` (and every agent run) is wrapped in a `ResilientAdapter`: a concurrency bulkhead, optional request pacing, automatic retries with exponential backoff and full jitter, a circuit breaker, and a per-call deadline. You get the default concurrency bound without configuring anything. diff --git a/site_docs/guides/reasoning-models.md b/site_docs/guides/reasoning-models.md index 13ff092..5e3b452 100644 --- a/site_docs/guides/reasoning-models.md +++ b/site_docs/guides/reasoning-models.md @@ -32,7 +32,7 @@ options = RunnerOptions( | `openai` | `reasoning_effort` only | `reasoning_effort` + `max_completion_tokens` on reasoning models | Non-reasoning models drop the effort and use `max_tokens` + `temperature` | | `azure` | `reasoning_effort` only | Same as `openai` | Same as `openai` | | `bedrock` | — | Ignored (thinking blocks in responses are still parsed) | Silently ignored | -| `ollama` | — | Ignored | Silently ignored | +| `ollama` | both knobs | `reasoning_effort` → `think` level (`minimal`→`low`); a token budget enables `think: true` | Non-thinking models return no `thinking` field | | `google` | — | Ignored | Silently ignored | ### Anthropic: budget resolution @@ -82,8 +82,12 @@ async for event in adapter.stream(messages, options): ... # answer text delta ``` -!!! note "Streaming thinking is Anthropic-only today" - Bedrock parses thinking blocks in non-streaming responses but does not emit `thinking` stream events, and the OpenAI-family adapters emit none at all. Don't build UI that requires live thinking deltas on those providers. +!!! note "Streaming thinking: Anthropic and Ollama" + Anthropic and Ollama emit live `thinking` stream events. Bedrock parses thinking blocks in non-streaming responses but does not emit `thinking` stream events, and the OpenAI-family adapters emit none at all. Don't build UI that requires live thinking deltas on those providers. + +### Ollama: think levels + +The Ollama adapter maps `reasoning_effort` to the native `think` parameter (`minimal` and `low` → `"low"`, `medium` → `"medium"`, `high` → `"high"`); a `thinking_budget_tokens` request enables `think: true` since Ollama has no budget concept. You can also set a default on the adapter itself with `OllamaAdapter(think=...)`, including the `"max"` level that has no portable tier. Thinking text returns as a `ThinkingBlock` (no signature) and as `thinking` stream events on models such as `qwen3`, `gpt-oss`, and `deepseek-r1`. See [`examples/32_reasoning_models.py`](https://github.com/Quantlix/anycode/blob/main/examples/32_reasoning_models.py) for a runnable walkthrough, including model classification and parameter shaping.