From 0c6ad7802ddd7b1f95918271450c01f27e233cfa Mon Sep 17 00:00:00 2001 From: Felipe Infante de Castro <18445184+fidecastro@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:03:44 -0500 Subject: [PATCH] endpoint.headers_env, credential-named headers refused; playbook after #7 Follow-up to #6: - `endpoint.headers_env` maps a header name to an environment variable, read when the backend is built and never written to the recipe; an unset variable fails before any request. A name set in both `headers` and `headers_env` is refused. - A static `endpoint.headers` name that looks like a credential (key, token, secret, auth, cookie, ...) is refused when the recipe loads, since the recipe is committed and its hash quoted. Anti-accident only: a secret under an innocent name still passes. `x-client-id` keeps working. - The playbook now covers servers without reply-token logprobs (the prompt_logprobs rung from #7, raw-mode recipes) and the two new refusals. Its examples and the Makefile's help use the reproducible Gemma recipe, and the sample answer is labelled illustrative. The Qwen3.8 router recipe stays as the example of `endpoint.headers`. - SKILL.md and ADR-0004 describe `headers_env` and the refusal. Co-Authored-By: Claude Opus 5.5 --- Makefile | 2 +- SKILL.md | 14 +++++++---- docs/adr/0004-recipes.md | 2 +- docs/playbook.md | 46 +++++++++++++++++++++++------------ jevify/compose.py | 10 +++++++- jevify/recipes/schema.py | 45 ++++++++++++++++++++++++++++++++-- tests/test_endpoint_client.py | 41 +++++++++++++++++++++++++++++++ tests/test_recipes.py | 37 ++++++++++++++++++++++++++++ 8 files changed, 172 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 1fee8fa..c50fb47 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ help: @echo " HOST other than 127.0.0.1 needs JEVIFY_API_KEY set: clients must then send it as a bearer key." @echo "" @echo "Example:" - @echo " make ask RECIPE=recipes/qwen38-27b.vllm.yaml \\" + @echo " make ask RECIPE=recipes/gemma-4-e4b-it.llamacpp.yaml \\" @echo " STATE=\"Customer: I was charged twice. Fix it today or I cancel.\" \\" @echo " CHOICE=\"dept:Which team should handle this?=billing,support,sales\" \\" @echo " SCORE=\"urgency:How urgent is this?=low,medium,high\" \\" diff --git a/SKILL.md b/SKILL.md index 39be408..59330ec 100644 --- a/SKILL.md +++ b/SKILL.md @@ -54,7 +54,7 @@ the rest. **`endpoint`** (any OpenAI-compatible server): `endpoint: {base_url, api_key_env, dialect: auto|vllm|llamacpp|generic, timeout_s, extra_body, -headers}`, `template` and `answers`. +headers, headers_env}`, `template` and `answers`. - `template.mode: messages` sends chat messages; put thinking off in `template.kwargs` (`{thinking: false}` for DeepSeek, `{enable_thinking: false}` for Qwen3-class models; names are model-specific, the probe checks @@ -69,10 +69,14 @@ headers}`, `template` and `answers`. fields also reach the tokenize and template routes the probe uses. - `endpoint.headers` adds static HTTP headers to every request, for a router or gateway that requires one (`{x-client-id: agenticApp}`). The values are - written in the recipe and hashed with it, so never put a secret there: a key - goes in the env var that `api_key_env` names, sent as `Authorization: - Bearer`. `Content-Type` and `Authorization` are refused when the backend is - built. Every endpoint-backed kind sends them (endpoint, rerank, embedding). + written in the recipe and hashed with it, so never put a secret there: a + bearer key goes in the env var that `api_key_env` names, and any other + secret header in `endpoint.headers_env` (header name to env var name, read + when the backend is built; an unset variable fails then). A static header + whose name looks like a credential (`key`, `token`, `secret`, `auth`, + `cookie`, ...) is refused when the recipe loads; `Content-Type` and + `Authorization` are refused when the backend is built. Every + endpoint-backed kind sends them (endpoint, rerank, embedding). - `template.mode: raw` sends a completion prompt you write with the model's own control tokens; use it when the chat template would not put the model in its trained regime (the Qwen3-VL reranker recipe is the example), and diff --git a/docs/adr/0004-recipes.md b/docs/adr/0004-recipes.md index e09c5a7..68bc133 100644 --- a/docs/adr/0004-recipes.md +++ b/docs/adr/0004-recipes.md @@ -32,7 +32,7 @@ hash is the recipe's identity in every answer and every scorecard. | Section | Holds | |---|---| | `model` | Identifier as the backend knows it, revision when known, the adapter kind, and for endpoints the dialect the probe detected. | -| `endpoint` | For endpoint-backed kinds: base URL, the environment variable holding an API key, the dialect, the timeout, and `extra_body`, server-specific fields merged into the top level of every request (a field jevify itself sets is refused when the backend is built, from one registry in the endpoint client; omitted from the hash when empty, so older recipes keep theirs), and `headers`, static HTTP headers sent on every request (`Content-Type` and `Authorization` are refused when the backend is built; never a secret, since the recipe is hashed and shared; omitted from the hash when empty). | +| `endpoint` | For endpoint-backed kinds: base URL, the environment variable holding an API key, the dialect, the timeout, and `extra_body`, server-specific fields merged into the top level of every request (a field jevify itself sets is refused when the backend is built, from one registry in the endpoint client; omitted from the hash when empty, so older recipes keep theirs), `headers`, static HTTP headers sent on every request (`Content-Type` and `Authorization` are refused when the backend is built; never a secret, since the recipe is hashed and shared, so a name that looks like a credential is refused when the recipe loads; omitted from the hash when empty), and `headers_env`, header names mapped to environment variables whose values are read when the backend is built and never written to the recipe (an unset variable fails then; omitted from the hash when empty). | | `template` | For endpoints: how state, instructions and options are rendered into messages or a raw prompt; where the question goes; the answer-slot suffix; template arguments such as thinking off; whether assistant prefill is used. For encoders: the sequence layout and marker scheme. | | `answers` | The answer tokens for `noul`, the identifier scheme for `choice`, the level tokens for `score`, each with the token ids the probe verified. | | `readout` | The rung of the ladder to use or `auto`; the permutation policy and call count for `choice`. | diff --git a/docs/playbook.md b/docs/playbook.md index c0c8381..92e9f2c 100644 --- a/docs/playbook.md +++ b/docs/playbook.md @@ -46,7 +46,7 @@ systems. `make help` lists every variable and every recipe. Use `make ask`. Give a recipe, a state and at least one question: ```sh -make ask RECIPE=recipes/qwen38-27b.vllm.yaml \ +make ask RECIPE=recipes/gemma-4-e4b-it.llamacpp.yaml \ STATE="Customer: I was charged twice. Fix it today or I cancel." \ CHOICE="dept:Which team should handle this?=billing,support,sales" \ SCORE="urgency:How urgent is this?=low,medium,high" \ @@ -55,7 +55,7 @@ make ask RECIPE=recipes/qwen38-27b.vllm.yaml \ | Variable | Format | Example | |---|---|---| -| `RECIPE` | Path to a recipe | `recipes/qwen38-27b.vllm.yaml` | +| `RECIPE` | Path to a recipe | `recipes/gemma-4-e4b-it.llamacpp.yaml` | | `STATE` | The text, or `@path` to read it from a file | `STATE=@ticket.txt` | | `CHOICE` | `id:question=option1,option2,...` | `dept:Which team?=billing,support` | | `SCORE` | `id:question=level0,level1,...`, lowest level first | `urgency:How urgent?=low,medium,high` | @@ -72,19 +72,19 @@ Rules: flag: ```sh - uv run jevify ask recipes/qwen38-27b.vllm.yaml --state @ticket.txt \ + uv run jevify ask recipes/gemma-4-e4b-it.llamacpp.yaml --state @ticket.txt \ --noul "cancel:The customer threatens to cancel." \ --noul "refund:The customer asks for a refund." ``` ## 4. Read the answer -`make ask` prints JSON. This is the answer to the example in step 3, -shortened: +`make ask` prints JSON. This is the shape of the answer to the example in +step 3, shortened; the numbers are illustrative, not a recorded run: ```json { - "model": "qwen38-27b", + "model": "gemma-4-e4b-it", "answers": { "dept": { "type": "choice", @@ -152,7 +152,7 @@ shortened: Use `make serve`. It serves TypeSafe's Jev API for one recipe: ```sh -make serve RECIPE=recipes/qwen38-27b.vllm.yaml +make serve RECIPE=recipes/gemma-4-e4b-it.llamacpp.yaml ``` It listens on `127.0.0.1:8600`. Use `PORT=8601` to pick another port. @@ -170,7 +170,7 @@ Ask with `curl`. The body is TypeSafe's `system_one` request: curl -s -X POST http://127.0.0.1:8600/v1/systemone \ -H "Content-Type: application/json" \ -d '{ - "model": "qwen38-27b", + "model": "gemma-4-e4b-it", "state": "Customer: I was charged twice. Fix it today or I cancel.", "questions": { "dept": {"type": "choice", "instructions": "Which team should handle this?", @@ -214,7 +214,7 @@ set a key first. Clients must then send it as `Authorization: Bearer `. ```sh export JEVIFY_API_KEY= -make serve RECIPE=recipes/qwen38-27b.vllm.yaml HOST=0.0.0.0 +make serve RECIPE=recipes/gemma-4-e4b-it.llamacpp.yaml HOST=0.0.0.0 ``` `make serve` refuses a `HOST` other than `127.0.0.1` when `JEVIFY_API_KEY` @@ -222,16 +222,30 @@ is not set. ## 6. Add a new model -jevify works only with a server that returns next-token log-probabilities -(`logprobs` with `top_logprobs`). vLLM and llama.cpp do. Many hosted APIs and -proxies do not; for those the probe stops with `no readout is possible`. +jevify needs a server that returns next-token log-probabilities, in one of +two ways: + +- on the reply token (`logprobs` with `top_logprobs`), as vLLM and llama.cpp + do; any recipe works; +- for the prompt (vLLM's `prompt_logprobs` on `/v1/completions`), as + TensorFold's native server does for GLM-5.3-Flash, which refuses the + first way; the recipe must use `template.mode: raw`, and the probe proves + the `prompt_logprobs` rung. + +Many hosted APIs and proxies do neither; for those the probe stops with +`no readout is possible`. 1. Copy the nearest recipe under `recipes/`. 2. Set `model.name` to the id the server lists at `/v1/models`. 3. Set `endpoint.base_url` (it ends in `/v1`). 4. If the server needs a header, set `endpoint.headers`, for example - `{x-client-id: agenticApp}`. Never put a secret there: put the key in an - environment variable and name it in `endpoint.api_key_env`. + `{x-client-id: agenticApp}`. Never put a secret there: a header whose + value is a secret goes in `endpoint.headers_env` as header name to + environment variable (`{X-Api-Key: GATEWAY_KEY}`), and a bearer key in the + variable that `endpoint.api_key_env` names. A header name that looks like + a credential (`key`, `token`, `secret`, `auth`, `cookie`, ...) is refused in + `endpoint.headers`. + `recipes/qwen38-27b.vllm.yaml` sends a header to a private router. 5. If the model reasons before it answers, turn reasoning off in `template.kwargs`. Try `{enable_thinking: false}` (Qwen3, GLM) or `{thinking: false}` (DeepSeek). @@ -274,7 +288,9 @@ calibrated. | You see | Cause | Do this | |---|---|---| | `missing required header 'x-client-id'` | The server needs a header the recipe does not send. | Add `endpoint.headers: {x-client-id: }`. | -| `no readout is possible on this backend: it returned no logprobs` | The server or proxy hides log-probabilities. | Use a server that returns them (vLLM, llama.cpp). No recipe setting fixes this. | +| `no readout is possible on this backend: it returned no logprobs` | The server or proxy returns no log-probabilities for the reply token, and the recipe could not read them through `prompt_logprobs`. | If the probe notes `reply-token logprobs refused`, write a raw-mode recipe so it can prove `prompt_logprobs` (step 6). Otherwise use a server that returns them (vLLM, llama.cpp). | +| `endpoint.headers names X-Api-Key, which looks like a credential` | A secret would be written into the recipe. | Move it to `endpoint.headers_env: {X-Api-Key: }` and export the variable. | +| `endpoint.headers_env sends X-Api-Key from $GATEWAY_KEY, which is not set` | The variable the recipe names is not exported in this shell. | `export GATEWAY_KEY=...`, then run again. | | `degraded: true`, options listed in `missing` | The options were not among the model's most likely tokens. Often the model starts to reason instead of answering. | Turn reasoning off (step 6, item 5), then probe again. | | `off_menu_mass` close to 1 | The model wants to write something that is not an option. | Same as above. Check the question text too. | | A context-limit error, or HTTP 413 from `serve` | The state is longer than the model's context. | Shorten the state. | diff --git a/jevify/compose.py b/jevify/compose.py index 7efb9a5..76074c7 100644 --- a/jevify/compose.py +++ b/jevify/compose.py @@ -30,12 +30,20 @@ def build_app(recipe: Recipe, *, http: Any | None = None, api_key_env: str | Non def _endpoint_client(endpoint: EndpointSpec, http: Any | None) -> OpenAICompatibleClient: """The one place an endpoint section becomes an HTTP client, for every endpoint-backed kind.""" api_key = os.environ.get(endpoint.api_key_env) if endpoint.api_key_env else None + headers = dict(endpoint.headers) + for name, variable in endpoint.headers_env.items(): + value = os.environ.get(variable) + if not value: + raise BackendError( + f"endpoint.headers_env sends {name} from ${variable}, which is not set" + ) + headers[name] = value return OpenAICompatibleClient( endpoint.base_url, api_key=api_key, timeout_s=endpoint.timeout_s, extra_body=endpoint.extra_body, - headers=endpoint.headers, + headers=headers, http=http, ) diff --git a/jevify/recipes/schema.py b/jevify/recipes/schema.py index 9494294..6027e70 100644 --- a/jevify/recipes/schema.py +++ b/jevify/recipes/schema.py @@ -48,9 +48,50 @@ class EndpointSpec(_Strict): extra_body: dict[str, Any] = Field(default_factory=dict, exclude_if=lambda v: not v) # Static headers sent on every request, e.g. a router's required client/product id. # May not set Content-Type or Authorization (api_key_env is the only way to set - # Authorization); the client refuses at construction. Left out of the dump when empty - # so recipes that predate it keep their hash. + # Authorization); the client refuses at construction. A name that looks like a + # credential is refused here: the recipe is committed and its hash quoted. Left out of + # the dump when empty so recipes that predate it keep their hash. headers: dict[str, str] = Field(default_factory=dict, exclude_if=lambda v: not v) + # Headers whose values come from the environment: header name -> variable name, read + # when the backend is built (an unset variable fails then), never written to the file. + headers_env: dict[str, str] = Field(default_factory=dict, exclude_if=lambda v: not v) + + @model_validator(mode="after") + def _check_headers(self) -> EndpointSpec: + credential = sorted(n for n in self.headers if _looks_like_credential(n)) + if credential: + raise ValueError( + f"endpoint.headers names {', '.join(credential)}, which looks like a " + "credential; the recipe is committed and hashed, so name the value's " + "environment variable in endpoint.headers_env instead (Authorization comes " + "only from endpoint.api_key_env)" + ) + both = sorted({n.lower() for n in self.headers} & {n.lower() for n in self.headers_env}) + if both: + raise ValueError( + f"{', '.join(both)} is set in both endpoint.headers and endpoint.headers_env" + ) + return self + + +# Header names a static `endpoint.headers` entry may not use because they usually carry a +# secret. Anti-accident, not anti-adversary: a secret under an innocent name still passes. +_CREDENTIAL_HEADER_WORDS = ( + "key", + "token", + "secret", + "password", + "passwd", + "auth", + "cookie", + "credential", + "session", + "signature", +) + + +def _looks_like_credential(name: str) -> bool: + return any(word in name.lower() for word in _CREDENTIAL_HEADER_WORDS) class QuestionTemplates(_Strict): diff --git a/tests/test_endpoint_client.py b/tests/test_endpoint_client.py index bf3fde0..b43766c 100644 --- a/tests/test_endpoint_client.py +++ b/tests/test_endpoint_client.py @@ -69,3 +69,44 @@ async def handler(request: httpx.Request) -> httpx.Response: def test_endpoint_headers_may_not_override_what_jevify_sets(name: str) -> None: with pytest.raises(BackendError, match="headers"): OpenAICompatibleClient("http://fake/v1", headers={name: "x"}) + + +def _probe_headers(recipe) -> list[httpx.Headers]: + seen: list[httpx.Headers] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers) + return httpx.Response(400, json={"error": {"message": "refused by the test"}}) + + http = httpx.AsyncClient(transport=httpx.MockTransport(handler), base_url="http://fake") + with pytest.raises(BackendError): + run(build_backend(recipe, http=http).probe()) + return seen + + +def test_headers_env_sends_the_value_of_the_named_variable(monkeypatch) -> None: + monkeypatch.setenv("JEVIFY_TEST_GATEWAY_KEY", "s3cr3t") + recipe = load_recipe(RECIPES / "fake-vllm.yaml") + recipe.endpoint.headers = {"x-client-id": "agenticApp"} + recipe.endpoint.headers_env = {"X-Api-Key": "JEVIFY_TEST_GATEWAY_KEY"} + seen = _probe_headers(recipe) + assert seen + assert all(h.get("x-api-key") == "s3cr3t" for h in seen) + assert all(h.get("x-client-id") == "agenticApp" for h in seen) + + +def test_headers_env_naming_an_unset_variable_fails_before_any_request(monkeypatch) -> None: + monkeypatch.delenv("JEVIFY_TEST_UNSET_KEY", raising=False) + recipe = load_recipe(RECIPES / "fake-vllm.yaml") + recipe.endpoint.headers_env = {"X-Api-Key": "JEVIFY_TEST_UNSET_KEY"} + with pytest.raises(BackendError, match="JEVIFY_TEST_UNSET_KEY"): + build_backend(recipe) + + +@pytest.mark.parametrize("name", ["Authorization", "content-type"]) +def test_headers_env_may_not_override_what_jevify_sets(monkeypatch, name: str) -> None: + monkeypatch.setenv("JEVIFY_TEST_GATEWAY_KEY", "s3cr3t") + recipe = load_recipe(RECIPES / "fake-vllm.yaml") + recipe.endpoint.headers_env = {name: "JEVIFY_TEST_GATEWAY_KEY"} + with pytest.raises(BackendError, match="headers"): + build_backend(recipe) diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 0bc55d4..9a0d9be 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -209,6 +209,43 @@ def test_endpoint_headers_are_part_of_the_recipe_and_its_hash(tmp_path: Path) -> assert recipe_hash(load_recipe(plain)) != recipe_hash(load_recipe(with_headers)) +@pytest.mark.parametrize( + "name", ["X-Api-Key", "api-key", "Proxy-Authorization", "Cookie", "X-Auth-Token", "x-secret"] +) +def test_a_credential_header_is_refused_in_the_recipe(tmp_path: Path, name: str) -> None: + # The recipe is committed and its hash quoted, so a static header that looks like a + # credential is refused at load, naming headers_env, where the value stays out of the file. + doc = yaml.safe_load((FIXTURES / "fake-vllm.yaml").read_text()) + doc["endpoint"]["headers"] = {name: "s3cr3t"} + path = tmp_path / "secret.yaml" + path.write_text(yaml.safe_dump(doc)) + with pytest.raises(RecipeError, match="headers_env"): + load_recipe(path) + + +def test_headers_env_names_variables_and_is_part_of_the_hash(tmp_path: Path) -> None: + doc = yaml.safe_load((FIXTURES / "fake-vllm.yaml").read_text()) + plain = tmp_path / "plain.yaml" + plain.write_text(yaml.safe_dump(doc)) + doc["endpoint"]["headers_env"] = {"X-Api-Key": "GATEWAY_KEY"} + with_env = tmp_path / "with_env.yaml" + with_env.write_text(yaml.safe_dump(doc)) + + assert load_recipe(plain).endpoint.headers_env == {} + assert load_recipe(with_env).endpoint.headers_env == {"X-Api-Key": "GATEWAY_KEY"} + assert recipe_hash(load_recipe(plain)) != recipe_hash(load_recipe(with_env)) + + +def test_a_header_named_in_both_headers_and_headers_env_is_refused(tmp_path: Path) -> None: + doc = yaml.safe_load((FIXTURES / "fake-vllm.yaml").read_text()) + doc["endpoint"]["headers"] = {"X-Client-Id": "agenticApp"} + doc["endpoint"]["headers_env"] = {"x-client-id": "CLIENT_ID"} + path = tmp_path / "both.yaml" + path.write_text(yaml.safe_dump(doc)) + with pytest.raises(RecipeError, match="x-client-id"): + load_recipe(path) + + def test_a_recipe_without_extra_body_keeps_the_hash_it_had_before_the_field_existed() -> None: # Scorecards cite recipe hashes; a new optional field must not re-hash old recipes. # Literals recorded on the parent commit, before `endpoint.extra_body` was added.