Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/accounts.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,10 @@ In external mode:
- Each model request names the principal that caused the work, in the
`X-Nerve-Inference-Principal` header: the session's cause, or the system
actor for Nerve's own model calls. See
[Inference principal header](config.md#inference-principal-header).
[Inference principal header](config.md#inference-principal-header). The
model requests of a cron job's session also name the job, in the
`X-Nerve-Inference-Cron-Job` header. See
[Inference cron job header](config.md#inference-cron-job-header).

> **Warning:** Without `auth.external.gateway_jwks_file`, Nerve trusts the
> header. Then use external mode only when the gateway is the only caller that
Expand Down
27 changes: 27 additions & 0 deletions docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -2391,6 +2391,33 @@ and a CLI process keeps it for its whole life. Nerve does not prove the
value. The same requests as above have no header, and Codex requests to
`codex.base_url` have no trigger header.

#### Inference cron job header

In external mode, Nerve also adds one `X-Nerve-Inference-Cron-Job` header to
each model request of a session that a cron job started: the sessions of
isolated and persistent cron jobs, and the session of a workflow run that a
cron job started. The value names the job, so that a proxy in front of the
model API can attribute model usage to it. Other sessions ("Run later" too)
and Nerve's own model calls send no cron job header.

Cron job IDs are free text, and the value must be a short, safe header
value. So Nerve changes the job ID:

1. Each run of characters other than ASCII letters, digits, `.`, `_` and `-`
becomes one `-`.
2. Nerve removes `-` at the start and at the end.
3. An empty result becomes `job-` and the first 8 hex digits of the SHA-256
of the job ID (UTF-8).
4. A result longer than 64 bytes becomes its first 55 bytes, `-`, and the
same 8 hex digits.

For example, `daily-digest` stays `daily-digest`, and `Morning briefing`
becomes `Morning-briefing`. The value is 1 to 64 bytes. The header has the
same rules as the trigger header: local mode sends none, Nerve removes an
inherited line for it from `ANTHROPIC_CUSTOM_HEADERS`, and a CLI process keeps
it for its whole life. Nerve does not prove the value. Codex requests have no
cron job header.

#### Inference origin header

In external mode, Nerve also adds one `X-Nerve-Inference-Origin` header to
Expand Down
4 changes: 4 additions & 0 deletions nerve/agent/backends/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,10 @@ class SessionSpec:
# The session's trigger for the same proxy (nerve.session_origin), in
# external mode. None: no trigger. A backend that cannot send it ignores it.
inference_trigger: str | None = None
# The header value of the session's cron job for the same proxy
# (nerve.inference_principal.inference_cron_job), in external mode. None:
# no cron job. A backend that cannot send it ignores it.
inference_cron_job: str | None = None
# The tag of the session's origin for the same proxy, for example the
# cron job (nerve.session_origin.origin_tag), in external mode. None: no
# origin. A backend that cannot send it ignores it.
Expand Down
14 changes: 10 additions & 4 deletions nerve/agent/backends/claude.py
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ def _cli_stderr(line: str) -> None:
cache_ttl=spec.cache_ttl,
inference_principal=spec.inference_principal,
inference_trigger=spec.inference_trigger,
inference_cron_job=spec.inference_cron_job,
inference_origin=spec.inference_origin,
),
cwd=spec.cwd,
Expand Down Expand Up @@ -843,13 +844,16 @@ def _sweep_system_prompt_dir(dir_path: Path) -> None:

def _build_env(
self, cache_ttl: str = "5m", inference_principal: str | None = None,
inference_trigger: str | None = None, inference_origin: str | None = None,
inference_trigger: str | None = None,
inference_cron_job: str | None = None,
inference_origin: str | None = None,
) -> dict[str, str]:
"""Build environment variables for the SDK subprocess.

``inference_principal``, ``inference_trigger`` and
``inference_origin``: the principal, the trigger and the origin that
every model request of the process names (external mode; see
``inference_cron_job`` and ``inference_origin``: the principal, the
trigger, the cron job and the origin that every model request of the
process names (external mode; see
nerve.inference_principal).
"""
config = self.config
Expand Down Expand Up @@ -926,7 +930,9 @@ def _build_env(
# The SDK merges this dict over Nerve's own environment, so an
# inherited ANTHROPIC_CUSTOM_HEADERS is kept and extended here.
env.update(custom_headers_env(
inference_principal, trigger=inference_trigger, origin=inference_origin,
inference_principal, trigger=inference_trigger,
cron_job=inference_cron_job,
origin=inference_origin,
))
return env

Expand Down
14 changes: 10 additions & 4 deletions nerve/agent/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
resolve_cause,
)
from nerve.inference_principal import (
inference_cron_job,
inference_origin,
inference_principal,
inference_trigger,
Expand All @@ -111,7 +112,7 @@
)
from nerve.observability.langfuse import attributes as lf_attrs
from nerve.platform.activity import LEASE_MEMORIZE, keep_awake
from nerve.session_origin import cron_origin, session_origin
from nerve.session_origin import DETAIL_CRON_JOB, cron_origin, session_origin
from nerve.skills.manager import SkillManager
from nerve.utils.llm import response_text

Expand Down Expand Up @@ -1778,6 +1779,9 @@ async def _record_wakeup_cb(sid: str, tool_input: dict) -> Any:
# or a newer one (see _mcp_catalog_rebuild_due).
self._client_mcp_revisions[session_id] = self._mcp_catalog_revision

# The session's origin, fixed for the session: its trigger and,
# for a session of a cron job, the job.
origin = session_origin(session or {})
spec = SessionSpec(
session_id=session_id,
source=source,
Expand Down Expand Up @@ -1805,12 +1809,14 @@ async def _record_wakeup_cb(sid: str, tool_input: dict) -> Any:
(session or {}).get("caused_by_actor_id"),
),
# The session's trigger, also fixed for the session.
inference_trigger=inference_trigger(
session_origin(session or {}).trigger,
inference_trigger=inference_trigger(origin.trigger),
# The session's cron job, if a cron job started it.
inference_cron_job=inference_cron_job(
origin.detail.get(DETAIL_CRON_JOB),
),
# Where the session comes from (its cron job, channel, ...),
# also fixed for the session.
inference_origin=inference_origin(session_origin(session or {})),
inference_origin=inference_origin(origin),
extra=(
# Per-session codex sandbox override (review-loop verifier
# legs run workspace-write instead of the global default).
Expand Down
91 changes: 71 additions & 20 deletions nerve/inference_principal.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""The principal, trigger and origin that Nerve names on model requests in external mode.
"""The principal, trigger, cron job and origin Nerve names on model requests
in external mode.

In external mode, a proxy in front of the model API can attribute model usage
to a person. Nerve gives it the principal that caused the work: one
Expand All @@ -14,32 +15,39 @@
:mod:`nerve.session_origin`, or ``system`` for Nerve's own model calls. Like
the cause, the trigger of a session is set when the session is created.

A third header, ``X-Nerve-Inference-Origin``, names where a session comes
A third header, ``X-Nerve-Inference-Cron-Job``, names the cron job of a
session that a cron job started (``origin_detail.cron_job``), so that the
proxy can attribute model usage to the job. Job IDs are free text, so Nerve
sends :func:`cron_job_tag_value` of the ID: 1 to 64 bytes of ASCII letters,
digits, ``.``, ``_`` and ``-``. Nerve's own model calls send no cron job.

Another header, ``X-Nerve-Inference-Origin``, names where a session comes
from: the cron job, the channel conversation, the hook, the plan, the
workflow run or the external client, as a tag such as ``cron=<job id>``
(:func:`nerve.session_origin.origin_tag`). Like the trigger, it comes from
the session's origin, which is set when the session is created. A session
without such a detail, and Nerve's own model calls, send no origin header.

Local mode sends no header, and a NULL cause sends no principal header: Nerve
also removes a principal line, a trigger line and an origin line from the
``ANTHROPIC_CUSTOM_HEADERS`` that its own environment gives the CLI.
also removes inherited principal, trigger, cron job and origin lines from
``ANTHROPIC_CUSTOM_HEADERS`` in the environment it gives the CLI.

The Claude CLI reads ``ANTHROPIC_CUSTOM_HEADERS`` from its environment when it
builds the API client for a request: one ``Name: Value`` header per line,
lines split on ``\\n`` or ``\\r\\n``, name and value split on the first ``:``
and trimmed. The environment of a CLI process is fixed when it starts. That is
enough, because the session's cause, trigger and origin never change.
enough, because the session's cause, trigger, cron job and origin never change.

Codex sends the principal header only to ``codex.base_url``, and sends no
trigger or origin header: the provider that Nerve gives Codex for it reads
the value from an environment variable of the Codex process
(``env_http_headers``). See ``nerve.agent.backends.codex.backend``. One
Codex process serves one session, so the same rule applies.
trigger, cron job or origin header: the provider that Nerve gives Codex for it
reads the value from an environment variable of the Codex process
(``env_http_headers``). See ``nerve.agent.backends.codex.backend``. One Codex
process serves one session, so the same rule applies.
"""

from __future__ import annotations

import hashlib
import logging
import os
import re
Expand All @@ -58,16 +66,25 @@

INFERENCE_PRINCIPAL_HEADER = "X-Nerve-Inference-Principal"
INFERENCE_TRIGGER_HEADER = "X-Nerve-Inference-Trigger"
INFERENCE_CRON_JOB_HEADER = "X-Nerve-Inference-Cron-Job"
INFERENCE_ORIGIN_HEADER = "X-Nerve-Inference-Origin"
CUSTOM_HEADERS_ENV = "ANTHROPIC_CUSTOM_HEADERS"

# The headers that Nerve sets, in lower case. Nerve removes an inherited line
# for each of them.
_NERVE_HEADERS = frozenset({
INFERENCE_PRINCIPAL_HEADER.lower(), INFERENCE_TRIGGER_HEADER.lower(),
INFERENCE_CRON_JOB_HEADER.lower(),
INFERENCE_ORIGIN_HEADER.lower(),
})

# The value rule of the cron job header, which the proxy checks again.
MAX_CRON_JOB_BYTES = 64
_CRON_JOB_VALUE = re.compile(rf"[A-Za-z0-9._-]{{1,{MAX_CRON_JOB_BYTES}}}")
_CRON_JOB_OTHER = re.compile(r"[^A-Za-z0-9._-]+")
# Length of the hash part of a changed cron job value, in hex digits.
_CRON_JOB_HASH_HEX = 8

# The CLI splits the variable on these line ends (and on nothing else).
_LINE_END = re.compile(r"\r?\n")

Expand Down Expand Up @@ -125,6 +142,36 @@ def inference_trigger(trigger: str | None) -> str | None:
return trigger


def cron_job_tag_value(job_id: str) -> str:
"""The value of the cron job header for cron job ``job_id``.

Each run of characters other than ASCII letters, digits, ``.``, ``_``
and ``-`` becomes one ``-``, and ``-`` at the start and the end is
removed. An empty result becomes ``job-`` and the first 8 hex digits of
the SHA-256 of ``job_id`` (UTF-8). A result longer than 64 bytes becomes
its first 55 bytes, ``-`` and the same 8 hex digits. So the value is 1
to 64 bytes, and two different IDs give the same value only rarely.
"""
value = _CRON_JOB_OTHER.sub("-", job_id).strip("-")
if value and len(value) <= MAX_CRON_JOB_BYTES:
return value
digest = hashlib.sha256(job_id.encode("utf-8")).hexdigest()[:_CRON_JOB_HASH_HEX]
if not value:
return f"job-{digest}"
return f"{value[:MAX_CRON_JOB_BYTES - _CRON_JOB_HASH_HEX - 1]}-{digest}"


def inference_cron_job(job_id: str | None) -> str | None:
"""The cron job to name on the model requests of a session of ``job_id``.

``None`` in local mode and when the session has no cron job; else
:func:`cron_job_tag_value` of the ID.
"""
if not job_id or not _is_external_mode():
return None
return cron_job_tag_value(job_id)


def inference_origin(origin: Origin) -> str | None:
"""The origin to name on the model requests of a session with ``origin``.

Expand Down Expand Up @@ -178,13 +225,13 @@ def direct_client_headers(

The Anthropic SDK also adds the lines of ``ANTHROPIC_CUSTOM_HEADERS`` in
the environment of Nerve to its default headers, with the name as written.
Each such name of the principal, the trigger or the origin header gets
Each such name of the principal, trigger, cron job or origin header gets
``anthropic.omit``, which removes it. Then, in external mode, the header
for ``principal_id`` is added, or for the system actor of the global
database when it is ``None``, and the trigger ``system``
(:func:`system_inference_headers`). So a request names one principal and
one trigger in external mode, and neither in local mode. It never names
an origin.
a cron job or an origin.
"""
from anthropic import omit

Expand All @@ -204,29 +251,31 @@ def direct_client_headers(

def custom_headers_env(
principal: str | None, environ: Mapping[str, str] | None = None,
*, trigger: str | None = None, origin: str | None = None,
*, trigger: str | None = None, cron_job: str | None = None,
origin: str | None = None,
) -> dict[str, str]:
"""Environment entries that make a Claude CLI process name ``principal``,
``trigger`` and ``origin``.
``trigger``, ``cron_job`` and ``origin``.

``principal`` comes from :func:`inference_principal`, ``trigger`` from
:func:`inference_trigger` and ``origin`` from :func:`inference_origin`.
:func:`inference_trigger`, ``cron_job`` from :func:`inference_cron_job`
and ``origin`` from :func:`inference_origin`.
The SDK merges the entries over the environment of Nerve, which the CLI
inherits. The value is the inherited ``ANTHROPIC_CUSTOM_HEADERS`` without
blank lines and without a line for the principal, the trigger or the
origin header, plus, in external mode, one line for each of
``principal``, ``trigger`` and ``origin`` that is set. So the CLI sends
at most one header of each in external mode, and none in local mode. No
entry when that value is the inherited value, and in local mode when the
inherited value has no line to remove.
cron job or origin header, plus, in external mode, one line for each of
``principal``, ``trigger``, ``cron_job`` and ``origin`` that is set. So
the CLI sends at most one header of each in external mode, and none in
local mode. No entry when that value is the inherited value, and in local
mode when the inherited value has no line to remove.
"""
env = os.environ if environ is None else environ
inherited = env.get(CUSTOM_HEADERS_ENV, "")
lines = _LINE_END.split(inherited) if inherited else []
if not _is_external_mode():
if not any(_header_name(line) in _NERVE_HEADERS for line in lines):
return {}
principal = trigger = origin = None
principal = trigger = cron_job = origin = None
kept = [
line for line in lines
if line.strip() and _header_name(line) not in _NERVE_HEADERS
Expand All @@ -236,6 +285,8 @@ def custom_headers_env(
kept.append(f"{INFERENCE_PRINCIPAL_HEADER}: {principal}")
if trigger in TRIGGERS:
kept.append(f"{INFERENCE_TRIGGER_HEADER}: {trigger}")
if cron_job and _CRON_JOB_VALUE.fullmatch(cron_job):
kept.append(f"{INFERENCE_CRON_JOB_HEADER}: {cron_job}")
if is_origin_tag(origin):
kept.append(f"{INFERENCE_ORIGIN_HEADER}: {origin}")
value = "\n".join(kept)
Expand Down
25 changes: 23 additions & 2 deletions nerve/workflows/service.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,12 @@
from nerve.db.usage import estimate_turn_cost
from nerve.db.workflow_runs import ACTIVE_STATUSES
from nerve.identity import UNATTRIBUTED, Attribution, attribution_for
from nerve.session_origin import DETAIL_WORKFLOW_RUN, TRIGGER_WORKFLOW, Origin
from nerve.session_origin import (
DETAIL_CRON_JOB,
DETAIL_WORKFLOW_RUN,
TRIGGER_WORKFLOW,
Origin,
)
from nerve.utils.time import utc_now_iso

if TYPE_CHECKING:
Expand All @@ -85,6 +90,8 @@
_ERROR_MAX = 2000

_SESSION_PREFIX = "workflow:"
# ``created_by`` prefix of a run that a cron job started (nerve.cron.service).
_CRON_CREATED_BY = "cron:"


class WorkflowRunError(ValueError):
Expand Down Expand Up @@ -446,6 +453,20 @@ async def _origin_session_id(self, run: dict) -> str | None:
return None
return sid if await self.db.get_session(sid) else None

@staticmethod
def _leg_origin(run: dict) -> Origin:
"""The origin of the leg session of ``run``.

The run's ID, and for a run that a cron job started
(``created_by`` ``cron:<job>``) the job, so that the model requests
of the leg name the job (nerve.inference_principal).
"""
detail = {DETAIL_WORKFLOW_RUN: run["id"]}
cb = str(run.get("created_by") or "")
if cb.startswith(_CRON_CREATED_BY):
detail[DETAIL_CRON_JOB] = cb[len(_CRON_CREATED_BY):]
return Origin(TRIGGER_WORKFLOW, detail)

def _default_model(self, backend: str) -> str:
if backend == "codex":
return self.config.codex.model
Expand Down Expand Up @@ -540,7 +561,7 @@ async def _execute(self, run_id: str) -> None:
cwd=spec.get("cwd") or None,
actor=leg_actor,
cause=run.get("caused_by_actor_id"),
origin=Origin(TRIGGER_WORKFLOW, {DETAIL_WORKFLOW_RUN: run_id}),
origin=self._leg_origin(run),
)
if origin:
await self.db.update_session_fields(
Expand Down
Loading