diff --git a/.env.example b/.env.example index f1762a901..08e119b31 100644 --- a/.env.example +++ b/.env.example @@ -102,6 +102,19 @@ OPENAI_API_KEY= # OPENAI_EMBEDDING_MODEL=BAAI/bge-m3 # TEI rejects the ``dimensions=`` SDK kwarg — must be false when talking to TEI. # OPENAI_EMBEDDING_SEND_DIMENSIONS=false + +# -- Self-hosted chat LLM (opt-in) ------------------------------------------- +# The chat path (dedup judge, enrichment, contradiction detection, entity +# extraction) talks to each provider's hosted URL by default. Each URL can +# be overridden, so ``ENTITY_EXTRACTION_PROVIDER=openai`` can point at any +# OpenAI-compatible server: LM Studio, vLLM, or a gateway. Set the matching +# API key to whatever the server expects (LM Studio ignores it) and name a +# loaded model in ``ENTITY_EXTRACTION_MODEL``. From inside a container, +# ``host.docker.internal`` reaches a server on the host. +# +# OPENAI_CHAT_BASE_URL=http://host.docker.internal:1234/v1 +# ANTHROPIC_CHAT_BASE_URL=https://api.anthropic.com/v1 +# OPENROUTER_CHAT_BASE_URL=https://openrouter.ai/api/v1 # Optional: only set when running an instruction-aware model (Qwen3-Embedding, # e5-instruct). bge-m3 is symmetric — leave empty. # EMBEDDING_QUERY_INSTRUCTION= diff --git a/common/llm/constants.py b/common/llm/constants.py index 7f92b8952..08fc442d5 100644 --- a/common/llm/constants.py +++ b/common/llm/constants.py @@ -19,20 +19,30 @@ "GEMINI_DEFAULT_MODEL", "gemini-3.1-flash-lite-preview" ) -# OpenAI's chat-completions base URL — used by ``OpenAILLMProvider`` -# (and for openrouter / anthropic compat where the API mirrors OpenAI's -# shape, with the base URL swapped via tenant-config override). -OPENAI_CHAT_BASE_URL = "https://api.openai.com/v1" +# Chat-completions base URLs. ``OpenAILLMProvider`` works against any +# OpenAI-compatible endpoint by varying ``base_url``; the registry picks +# the URL from the provider name (``ProviderName``). Each URL can be +# overridden from the environment, so a provider can talk to a self-hosted +# server (LM Studio, vLLM, a gateway) without a code change. +# +# ``OPENAI_HOSTED_CHAT_BASE_URL`` is the literal hosted endpoint. It stays +# separate from the overridable value so the provider can tell whether it +# is talking to api.openai.com, which accepts request shapes that other +# compatible servers reject. +OPENAI_HOSTED_CHAT_BASE_URL = "https://api.openai.com/v1" +OPENAI_CHAT_BASE_URL = os.environ.get( + "OPENAI_CHAT_BASE_URL", OPENAI_HOSTED_CHAT_BASE_URL +) -# Anthropic + OpenRouter base URLs and default models. The -# ``OpenAILLMProvider`` works against any of these endpoints by varying -# ``base_url``; the registry picks the right tuple based on -# ``ProviderName``. -ANTHROPIC_CHAT_BASE_URL = "https://api.anthropic.com/v1" +ANTHROPIC_CHAT_BASE_URL = os.environ.get( + "ANTHROPIC_CHAT_BASE_URL", "https://api.anthropic.com/v1" +) ANTHROPIC_DEFAULT_MODEL = os.environ.get( "ANTHROPIC_DEFAULT_MODEL", "claude-haiku-4-5-20251001" ) # Anthropic API requires native model IDs -OPENROUTER_CHAT_BASE_URL = "https://openrouter.ai/api/v1" +OPENROUTER_CHAT_BASE_URL = os.environ.get( + "OPENROUTER_CHAT_BASE_URL", "https://openrouter.ai/api/v1" +) OPENROUTER_DEFAULT_MODEL = os.environ.get( "OPENROUTER_DEFAULT_MODEL", "openai/gpt-5.4-nano" ) diff --git a/common/llm/providers/openai.py b/common/llm/providers/openai.py index c3f4f088f..ae9979153 100644 --- a/common/llm/providers/openai.py +++ b/common/llm/providers/openai.py @@ -18,6 +18,7 @@ import json import logging import time +from urllib.parse import urlsplit import httpx import openai @@ -27,6 +28,7 @@ LLM_JSON_MAX_OUTPUT_TOKENS, LLM_PROVIDER_MAX_RETRIES, OPENAI_CHAT_BASE_URL, + OPENAI_HOSTED_CHAT_BASE_URL, OPENAI_HTTPX_CONNECT_TIMEOUT_SECONDS, OPENAI_HTTPX_MAX_CONNECTIONS, OPENAI_HTTPX_MAX_KEEPALIVE_CONNECTIONS, @@ -35,6 +37,7 @@ ) from common.llm.providers._shape_error import ProviderResponseShapeError from common.llm.providers._truncation import raise_if_truncated +from common.provider_names import ProviderName logger = logging.getLogger(__name__) @@ -88,6 +91,157 @@ def _int(value: object) -> int: ) +_HOSTED_OPENAI_HOST = urlsplit(OPENAI_HOSTED_CHAT_BASE_URL).hostname or "" +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) + + +def _is_hosted_openai(base_url: str) -> bool: + """True when ``base_url`` points at api.openai.com, whatever the scheme. + + The decision is by host, not by string: ``http://api.openai.com/v1`` or + a trailing slash still count as hosted. A proxy or regional alias on + another host does not. + """ + return (urlsplit(base_url).hostname or "").lower() == _HOSTED_OPENAI_HOST + + +def _is_plaintext_off_host(base_url: str) -> bool: + """True when ``base_url`` is plain ``http`` to a host that is not loopback.""" + parts = urlsplit(base_url) + return ( + parts.scheme == "http" and (parts.hostname or "").lower() not in _LOOPBACK_HOSTS + ) + + +def _allows_null(schema: dict) -> bool: + """True when ``schema`` already admits ``null``.""" + t = schema.get("type") + if t == "null" or (isinstance(t, list) and "null" in t): + return True + return any( + isinstance(opt, dict) and opt.get("type") == "null" + for opt in schema.get("anyOf", []) + schema.get("oneOf", []) + ) + + +def _strict_schema(schema: dict) -> dict: + """Return a copy of ``schema`` that strict JSON mode accepts. + + Strict mode, as OpenAI defines it and as Anthropic's OpenAI-compatible + endpoint enforces it, wants every object closed + (``additionalProperties: false``) with every property listed under + ``required``. Pydantic-generated schemas leave both open for fields + with defaults. This walks the schema (``properties``, ``items``, + ``anyOf``/``oneOf``/``allOf``, ``$defs``) and closes each object. + + Marking a property required changes what the model must emit, so a + property the source schema left optional is made nullable as well: + the model answers ``null`` where it would have omitted the key, and + ``_drop_optional_nulls`` removes those keys after parsing, so callers + see the same absent-or-present shape they saw before. A property that + already admits ``null`` is left as it is. + + ``default`` and ``title`` are dropped: strict mode rejects or ignores + them and they carry no meaning for the model. The input is not + modified. + """ + if isinstance(schema, list): + return [_strict_schema(s) for s in schema] # type: ignore[return-value] + if not isinstance(schema, dict): + return schema + out: dict = {} + for key, value in schema.items(): + if key in ("default", "title"): + continue + if key in ("properties", "$defs"): + out[key] = {k: _strict_schema(v) for k, v in value.items()} + elif key in ("items", "anyOf", "oneOf", "allOf"): + out[key] = _strict_schema(value) + else: + out[key] = value + if out.get("type") == "object" or "properties" in out: + was_required = set(schema.get("required", [])) + props = out.get("properties", {}) + for name, prop in props.items(): + if ( + name not in was_required + and isinstance(prop, dict) + and not _allows_null(prop) + ): + props[name] = {"anyOf": [prop, {"type": "null"}]} + out["additionalProperties"] = False + out["required"] = list(props.keys()) + return out + + +def _resolve_ref(schema: dict, root: dict) -> dict: + """Follow a local ``$ref`` into ``root["$defs"]``; other schemas pass through.""" + ref = schema.get("$ref", "") + if ref.startswith("#/$defs/"): + return root.get("$defs", {}).get(ref[len("#/$defs/") :], {}) + return schema + + +def _drop_optional_nulls(value, schema: dict, root: dict | None = None): + """Remove ``null`` values for keys the source schema left optional. + + The counterpart of ``_strict_schema``: it walks the parsed reply with + the *original* schema and deletes a key whose value is ``null`` when + that key was not in the object's ``required`` list. A ``null`` for a + required key, or for a key the source schema made nullable itself, is + kept as the model sent it. Arrays and local ``$ref`` definitions are + followed; other shapes are returned unchanged. + """ + root = schema if root is None else root + schema = _resolve_ref(schema, root) + if isinstance(value, dict) and isinstance(schema.get("properties"), dict): + required = set(schema.get("required", [])) + props = schema["properties"] + out = {} + for key, item in value.items(): + prop = props.get(key) + if ( + item is None + and prop is not None + and key not in required + and not _allows_null(prop) + ): + continue + out[key] = ( + _drop_optional_nulls(item, prop, root) + if isinstance(prop, dict) + else item + ) + return out + if isinstance(value, list) and isinstance(schema.get("items"), dict): + return [_drop_optional_nulls(item, schema["items"], root) for item in value] + for option in schema.get("anyOf", []) + schema.get("oneOf", []): + if isinstance(option, dict) and ( + option.get("properties") or option.get("items") or "$ref" in option + ): + return _drop_optional_nulls(value, option, root) + return value + + +def _strip_code_fence(content: str) -> str: + """Remove a Markdown code fence around ``content`` when there is one. + + Without a ``response_format`` to constrain them, models often answer + with the JSON wrapped in a fence. The JSON inside is what the caller + asked for. + """ + text = content.strip() + if not text.startswith("```"): + return content + first_newline = text.find("\n") + if first_newline == -1: + return content + body = text[first_newline + 1 :] + if body.rstrip().endswith("```"): + body = body.rstrip()[:-3] + return body + + class OpenAILLMProvider: """LLM provider using the OpenAI chat completions API. @@ -107,6 +261,16 @@ def __init__( self._model = model self._base_url = base_url self._provider_name = provider_name + if _is_plaintext_off_host(base_url): + # Operator configuration, not caller input, so this is a warning + # and not a refusal: a plain-http base URL off the loopback sends + # the provider key in the clear on every call. + logger.warning( + "%s chat base URL %s is plain http to a non-loopback host; " + "the API key is sent unencrypted", + provider_name, + base_url, + ) # Explicit per-call timeout — without this the SDK rides httpx's # default and a single hung upstream call would eat the whole # enrichment budget silently. @@ -193,9 +357,20 @@ async def complete_json( ) -> dict: """Send a prompt and return a parsed JSON dict. - Without ``response_schema``, uses - ``response_format={"type": "json_object"}`` to enforce shape-less - JSON output (back-compat for enrichment and dedup callers). + The ``response_format`` sent depends on the endpoint, because the + compatible servers do not agree on what they accept: + + - Hosted OpenAI (``OPENAI_HOSTED_CHAT_BASE_URL``) and OpenRouter keep + the shapes they previously received: ``json_object`` without a + schema, and a non-strict ``json_schema`` with one. + - Any other base URL (a self-hosted server or Anthropic's compatible + endpoint) gets no ``response_format`` without a + schema, because LM Studio and Anthropic both reject + ``json_object``; the prompt already asks for JSON and a code + fence in the reply is stripped before parsing. With a schema it + gets ``strict: true`` and a closed schema (see + ``_strict_schema``), which is the one shape Anthropic accepts and + which every other compatible server also takes. ``seed`` (A5a #2): when provided, forwarded to OpenAI's chat completions API for response determinism. ``temperature=0.0`` is @@ -208,11 +383,10 @@ async def complete_json( ``response_schema`` (A5b #3): when provided, switches to ``response_format={"type": "json_schema", ...}`` so the API - enforces the output shape server-side. ``strict=False`` — - Pydantic-generated schemas don't always satisfy OpenAI's strict- - mode requirements (additionalProperties=false everywhere); the - client-side Pydantic parse is the real guardrail. Passing - ``None`` preserves today's shape-less behaviour. + enforces the output shape server-side. Hosted OpenAI and OpenRouter + receive the source schema with ``strict=False``. Other endpoints + receive a closed schema from ``_strict_schema`` with ``strict=True``. + Passing ``None`` preserves today's shape-less behaviour. ``reasoning_effort`` (E3): forwarded to the chat completions API only when set, so callers doing bounded classification work (the @@ -226,27 +400,34 @@ async def complete_json( see the inline note at the call. """ t0 = time.perf_counter() - if response_schema is not None: - response_format: dict = { - "type": "json_schema", - "json_schema": { - "name": "response", - "schema": response_schema, - "strict": False, - }, - } - else: - response_format = {"type": "json_object"} + uses_openai_format = ( + self._provider_name == ProviderName.OPENROUTER + or _is_hosted_openai(self._base_url) + ) create_kwargs: dict = { "model": self._model, "messages": [{"role": "user", "content": prompt}], - "response_format": response_format, "temperature": temperature, # Runaway guard — same failure mode as the Gemini-backed # providers: an uncapped looping generation comes back as # truncated JSON (finish_reason="length"). "max_completion_tokens": LLM_JSON_MAX_OUTPUT_TOKENS, } + if response_schema is not None: + create_kwargs["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": "response", + "schema": ( + response_schema + if uses_openai_format + else _strict_schema(response_schema) + ), + "strict": not uses_openai_format, + }, + } + elif uses_openai_format: + create_kwargs["response_format"] = {"type": "json_object"} if seed is not None: create_kwargs["seed"] = seed if reasoning_effort is not None: @@ -281,9 +462,11 @@ async def complete_json( model=self._model, max_tokens=LLM_JSON_MAX_OUTPUT_TOKENS, ) - parsed = json.loads(content) + parsed = json.loads(_strip_code_fence(content)) if not isinstance(parsed, dict): raise OpenAIResponseShapeError(content, type(parsed).__name__) + if response_schema is not None and not uses_openai_format: + parsed = _drop_optional_nulls(parsed, response_schema) return parsed async def complete_text( diff --git a/tests/test_llm_openai_compatible_json_mode.py b/tests/test_llm_openai_compatible_json_mode.py new file mode 100644 index 000000000..01b8ce910 --- /dev/null +++ b/tests/test_llm_openai_compatible_json_mode.py @@ -0,0 +1,359 @@ +"""``complete_json`` sends a ``response_format`` the endpoint accepts. + +OpenAI-compatible servers disagree on JSON mode. Hosted OpenAI takes +``json_object`` and a non-strict ``json_schema``. LM Studio rejects +``json_object``. Anthropic's compatible endpoint rejects ``json_object``, +rejects ``strict: false``, and rejects any strict schema whose objects +are not closed. These tests pin the shape sent to each kind of endpoint, +and the fence strip that makes a bare-prompt reply parse. + +The provider is built by its own constructor and only the transport is +swapped, the same way ``test_llm_provider_sdk_retries.py`` does it. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import httpx +import pytest + +from common.llm import constants +from common.llm.providers.openai import ( + OpenAILLMProvider, + _drop_optional_nulls, + _is_hosted_openai, + _strict_schema, + _strip_code_fence, +) + +HOSTED = "https://api.openai.com/v1" +OPENROUTER = "https://openrouter.ai/api/v1" +SELF_HOSTED = "http://localhost:1234/v1" + + +def _provider( + base_url: str, + reply: str, + sent: list[dict], + *, + provider_name: str = "openai", +) -> OpenAILLMProvider: + """A real provider whose socket records the request body and answers ``reply``.""" + + def _handler(request: httpx.Request) -> httpx.Response: + sent.append(json.loads(request.content)) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": reply}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + ) + + provider = OpenAILLMProvider( + api_key="test-key", + model="test-model", + base_url=base_url, + provider_name=provider_name, + ) + provider._client._client._transport = httpx.MockTransport(_handler) + return provider + + +SCHEMA = { + "type": "object", + "title": "Graph", + "properties": { + "entities": { + "type": "array", + "default": [], + "items": {"$ref": "#/$defs/Entity"}, + }, + "note": {"anyOf": [{"type": "string"}, {"type": "null"}], "default": None}, + }, + "$defs": { + "Entity": { + "type": "object", + "title": "Entity", + "properties": {"name": {"type": "string"}, "kind": {"type": "string"}}, + "required": ["name"], + } + }, +} + + +class TestNoSchema: + async def test_hosted_openai_keeps_json_object(self): + sent: list[dict] = [] + provider = _provider(HOSTED, '{"ok": true}', sent) + assert await provider.complete_json("give me json") == {"ok": True} + assert sent[0]["response_format"] == {"type": "json_object"} + + async def test_other_endpoints_get_no_response_format(self): + sent: list[dict] = [] + provider = _provider(SELF_HOSTED, '{"ok": true}', sent) + assert await provider.complete_json("give me json") == {"ok": True} + assert "response_format" not in sent[0] + + async def test_openrouter_keeps_json_object(self): + sent: list[dict] = [] + provider = _provider( + OPENROUTER, '{"ok": true}', sent, provider_name="openrouter" + ) + assert await provider.complete_json("give me json") == {"ok": True} + assert sent[0]["response_format"] == {"type": "json_object"} + + async def test_a_fenced_reply_parses(self): + sent: list[dict] = [] + provider = _provider(SELF_HOSTED, '```json\n{"ok": true}\n```', sent) + assert await provider.complete_json("give me json") == {"ok": True} + + +class TestWithSchema: + async def test_hosted_openai_sends_the_schema_as_given(self): + sent: list[dict] = [] + provider = _provider(HOSTED, '{"entities": []}', sent) + await provider.complete_json("extract", response_schema=SCHEMA) + fmt = sent[0]["response_format"] + assert fmt["type"] == "json_schema" + assert fmt["json_schema"]["strict"] is False + assert fmt["json_schema"]["schema"] == SCHEMA + + async def test_other_endpoints_get_a_strict_closed_schema(self): + sent: list[dict] = [] + provider = _provider(SELF_HOSTED, '{"entities": [], "note": null}', sent) + await provider.complete_json("extract", response_schema=SCHEMA) + fmt = sent[0]["response_format"] + assert fmt["json_schema"]["strict"] is True + schema = fmt["json_schema"]["schema"] + assert schema["additionalProperties"] is False + assert schema["required"] == ["entities", "note"] + entity = schema["$defs"]["Entity"] + assert entity["additionalProperties"] is False + assert entity["required"] == ["name", "kind"] + + async def test_openrouter_keeps_the_compatible_schema_shape(self): + sent: list[dict] = [] + provider = _provider( + OPENROUTER, '{"entities": []}', sent, provider_name="openrouter" + ) + await provider.complete_json("extract", response_schema=SCHEMA) + fmt = sent[0]["response_format"] + assert fmt["json_schema"]["strict"] is False + assert fmt["json_schema"]["schema"] == SCHEMA + + +class TestStrictSchema: + def test_closes_every_object_and_drops_defaults_and_titles(self): + out = _strict_schema(SCHEMA) + assert "title" not in out + assert "default" not in out["properties"]["entities"] + assert "default" not in out["properties"]["note"] + # ``entities`` was optional, so it is wrapped nullable; the array is inside. + assert out["properties"]["entities"]["anyOf"][0]["items"] == { + "$ref": "#/$defs/Entity" + } + assert out["properties"]["note"]["anyOf"] == [ + {"type": "string"}, + {"type": "null"}, + ] + + def test_does_not_modify_its_input(self): + before = json.dumps(SCHEMA, sort_keys=True) + _strict_schema(SCHEMA) + assert json.dumps(SCHEMA, sort_keys=True) == before + + +class TestOptionalFieldsUnderStrictMode: + """Strict mode requires every property; the source schema's optional ones + become nullable on the wire and absent again after parsing.""" + + def test_the_test_schema_optional_fields_become_nullable(self): + out = _strict_schema(SCHEMA) + entities = out["properties"]["entities"] + assert entities["anyOf"][1] == {"type": "null"} + assert entities["anyOf"][0]["type"] == "array" + # ``note`` already admits null and is not wrapped twice. + assert out["properties"]["note"]["anyOf"] == [ + {"type": "string"}, + {"type": "null"}, + ] + kind = out["$defs"]["Entity"]["properties"]["kind"] + assert kind == {"anyOf": [{"type": "string"}, {"type": "null"}]} + assert out["$defs"]["Entity"]["properties"]["name"] == {"type": "string"} + + def test_extracted_graph_optional_fields_stay_optional_for_the_caller(self): + from core_api.services.entity_extraction import ExtractedGraph + + source = ExtractedGraph.model_json_schema() + strict = _strict_schema(source) + mention = strict["$defs"]["Mention"] + assert mention["required"] == ["surface", "cluster_id", "entity_canonical"] + # The three list fields default to [] in the model and are not required + # in the source schema; on the wire they are required but nullable. + for name in ("entities", "relations", "mentions"): + assert name in strict["required"] + assert strict["properties"][name]["anyOf"][1] == {"type": "null"} + reply = { + "entities": None, + "relations": [], + "mentions": [ + {"surface": "ACME", "cluster_id": None, "entity_canonical": None} + ], + } + cleaned = _drop_optional_nulls(reply, source) + assert "entities" not in cleaned + assert cleaned["relations"] == [] + # Mention's two optional fields are nullable in the model itself, so a + # null is a value the model may send and is kept. + assert cleaned["mentions"] == [ + {"surface": "ACME", "cluster_id": None, "entity_canonical": None} + ] + assert ExtractedGraph.model_validate(cleaned).entities == [] + + def test_enrichment_result_survives_the_round_trip(self): + from common.enrichment.schema import EnrichmentResult + + source = EnrichmentResult.model_json_schema() + strict = _strict_schema(source) + assert set(strict["required"]) == set(source["properties"]) + for name, prop in strict["properties"].items(): + assert prop.get("anyOf", [{}])[-1] == {"type": "null"}, name + # A null is dropped for every field the model would have omitted, and + # kept only where the source schema itself admits null. + reply = dict.fromkeys(source["properties"]) + nullable = { + name + for name, prop in source["properties"].items() + if any(o.get("type") == "null" for o in prop.get("anyOf", [])) + } + cleaned = _drop_optional_nulls(reply, source) + assert set(cleaned) == nullable + assert all(v is None for v in cleaned.values()) + assert EnrichmentResult.model_validate(cleaned).title == "" + + async def test_nulls_for_optional_fields_are_dropped_after_parsing(self): + sent: list[dict] = [] + provider = _provider( + SELF_HOSTED, + '{"entities": [{"name": "a", "kind": null}], "note": null}', + sent, + ) + result = await provider.complete_json("extract", response_schema=SCHEMA) + # ``kind`` was optional and not nullable: dropped. ``note`` admits null + # in the source schema: kept as sent. + assert result == {"entities": [{"name": "a"}], "note": None} + + async def test_hosted_openai_replies_are_returned_as_sent(self): + sent: list[dict] = [] + provider = _provider(HOSTED, '{"entities": null, "note": null}', sent) + result = await provider.complete_json("extract", response_schema=SCHEMA) + assert result == {"entities": None, "note": None} + + +class TestHostedDetection: + @pytest.mark.parametrize( + ("url", "hosted"), + [ + ("https://api.openai.com/v1", True), + ("https://api.openai.com/v1/", True), + ("http://api.openai.com/v1", True), + ("https://API.OpenAI.com/v1", True), + (OPENROUTER, False), + ("https://my-proxy.example.com/v1", False), + ("http://localhost:1234/v1", False), + ], + ) + def test_decides_by_host(self, url: str, hosted: bool): + assert _is_hosted_openai(url) is hosted + + @pytest.mark.parametrize( + ("url", "warns"), + [ + ("http://lmstudio.internal:1234/v1", True), + ("http://localhost:1234/v1", False), + ("http://127.0.0.1:1234/v1", False), + ("https://my-proxy.example.com/v1", False), + ], + ) + def test_plaintext_off_host_warns_at_construction( + self, caplog, url: str, warns: bool + ): + with caplog.at_level("WARNING", logger="common.llm.providers.openai"): + OpenAILLMProvider(api_key="test-key", model="test-model", base_url=url) + assert ("sent unencrypted" in caplog.text) is warns + + +class TestStripCodeFence: + @pytest.mark.parametrize( + "content", + [ + '{"a": 1}', + '```json\n{"a": 1}\n```', + '```\n{"a": 1}\n```', + ' ```json\n{"a": 1}\n``` ', + ], + ) + def test_the_json_inside_is_returned(self, content: str): + assert json.loads(_strip_code_fence(content)) == {"a": 1} + + def test_a_bare_fence_marker_is_left_alone(self): + assert _strip_code_fence("```") == "```" + + +class TestBaseUrlOverrides: + """The constants read the environment at import time. + + A fresh interpreter is the honest way to test an import-time read: + ``importlib.reload`` depends on ``sys.modules`` state that other tests + in the suite rearrange. + """ + + @pytest.mark.parametrize( + "variable", + ["OPENAI_CHAT_BASE_URL", "ANTHROPIC_CHAT_BASE_URL", "OPENROUTER_CHAT_BASE_URL"], + ) + def test_the_environment_wins(self, variable: str): + code = ( + "from common.llm import constants as c; " + f"print(c.{variable}); print(c.OPENAI_HOSTED_CHAT_BASE_URL)" + ) + env = {k: v for k, v in os.environ.items() if not k.endswith("_CHAT_BASE_URL")} + default = _constants_in_a_fresh_interpreter(code, env) + env[variable] = SELF_HOSTED + overridden = _constants_in_a_fresh_interpreter(code, env) + assert default[0] == getattr(constants, variable) + assert overridden[0] == SELF_HOSTED + assert overridden[1] == HOSTED + + +def _constants_in_a_fresh_interpreter(code: str, env: dict[str, str]) -> list[str]: + repo_root = Path(__file__).resolve().parents[1] + result = subprocess.run( + [sys.executable, "-c", code], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.split()