diff --git a/agents/common/llm.py b/agents/common/llm.py index be3e2bff..f236bc37 100644 --- a/agents/common/llm.py +++ b/agents/common/llm.py @@ -137,6 +137,20 @@ async def _wrapped_ainvoke(*args, **kwargs): # type: ignore[override] tokens_out = int(usage.get("completion_tokens", 0) or 0) cost = float(getattr(res, "cost", 0.0) or 0.0) metrics.model_call(model=model_name, tokens_in=tokens_in, tokens_out=tokens_out, cost_usd=cost) + # Token counts on the active span, next to the model id and redacted + # prompt/response already stamped here: trace backends (Arize Phoenix via + # the agent-observability-stack collector's OpenInference transform) map + # agentic-netops.tokens.* to llm.token_count.* so each model call shows + # its usage in the trace tree. NOTE (2026-10-11): the SDK's context + # scoping currently keeps agentic-netops.* off exported spans (the + # OpenInference instrumentation already records usage natively as + # gen_ai.usage.* / llm.usage.total_tokens), so this stamping is + # redundant-but-harmless until the SDK context is fixed. + span = trace.get_current_span() + if span is not None and hasattr(span, "set_attribute"): + span.set_attribute("agentic-netops.tokens.prompt", tokens_in) + span.set_attribute("agentic-netops.tokens.completion", tokens_out) + span.set_attribute("agentic-netops.tokens.total", tokens_in + tokens_out) # Update per-thread token budget if _TOK_BUDGET > 0: _add_tokens(tokens_in + tokens_out) @@ -178,6 +192,12 @@ def _wrapped_invoke(*args, **kwargs): # type: ignore[override] tokens_out = int(usage.get("completion_tokens", 0) or 0) cost = float(getattr(res, "cost", 0.0) or 0.0) metrics.model_call(model=model_name, tokens_in=tokens_in, tokens_out=tokens_out, cost_usd=cost) + # Token counts on the active span (see the ainvoke wrapper above for why). + span = trace.get_current_span() + if span is not None and hasattr(span, "set_attribute"): + span.set_attribute("agentic-netops.tokens.prompt", tokens_in) + span.set_attribute("agentic-netops.tokens.completion", tokens_out) + span.set_attribute("agentic-netops.tokens.total", tokens_in + tokens_out) if _TOK_BUDGET > 0: _add_tokens(tokens_in + tokens_out) # Record redacted response on the span (T335) diff --git a/deploy/agents/README.md b/deploy/agents/README.md index 821e668a..bfb43fef 100644 --- a/deploy/agents/README.md +++ b/deploy/agents/README.md @@ -14,6 +14,39 @@ with its translator sidecar, ui, otel collector, clickhouse, RBAC, NetworkPolicies) land in later phases. The manifests follow the supervisor's contract for the Kubernetes objects it submits. +## Telemetry (traces) + +Each agent emits OTLP exactly once to the tier-owned `agent-otel-collector` +(research.md Decision 8), which is the single fan-out point with three sinks: + +- `clickhouse` — the tier's `agent_analytics` store (traces + metrics). +- `otlp/feature001` — feature 001's collector (metrics only; its filter keeps + `agentic_netops_agent_*` metrics into Prometheus). +- `otlphttp/phoenix` — **traces only**, to the host-side + `mairp/agent-observability-stack` collector (Arize Phoenix trace backend). + Endpoint comes from `AOS_OTLP_HTTP_ENDPOINT` (lab default + `http://172.17.0.1:4318`, the docker bridge gateway from inside kind). + Traces appear in Phoenix (UI host `:3006`) as one project per + `service.name` — `intent-supervisor`, `intent-mapper`, `intent-allocator`, + `intent-deployer` — though A2A round-trips propagate W3C context, so a + request's full trace (client + remote agent spans) files under the + supervisor's project. The spans are OpenInference-native (ioa-observe is a + rebranded Traceloop SDK): the supervisor's `ChatLiteLLM.chat` spans carry + full prompt/completion content plus token counts, and the AOS collector + maps them to LLM/AGENT kinds and fixes the model name + (`transform/phoenix_agentic_netops`). Note: the `agentic-netops.*` span + attributes stamped in `agents/common/llm.py` (incl. the T333-T335 redacted + prompt/response) do not reach exported spans — the SDK's context scoping + keeps them off the spans — so content is captured as-is. The collector's + egress to the AOS host is admitted by the `allow-egress-aos-otel` + NetworkPolicy (`namespace-rbac.yaml`, T051) — collector pod only; without + it the dial is dropped by the deny-all baseline and the sink fails + silently. Note: the collector mounts its config as a file, so applying a + changed `telemetry.yaml` also needs `kubectl rollout restart + deploy/agent-otel-collector -n agentic-netops-agents`. Kill switch: + remove `otlphttp/phoenix` from the traces pipeline exporters and re-apply + (plus the restart). + ## Credential policy No credential literal may appear anywhere under this directory or under diff --git a/deploy/agents/namespace-rbac.yaml b/deploy/agents/namespace-rbac.yaml index 7c37ff49..898bdc42 100644 --- a/deploy/agents/namespace-rbac.yaml +++ b/deploy/agents/namespace-rbac.yaml @@ -358,3 +358,32 @@ spec: port: 443 - protocol: TCP port: 6443 +--- +# T051: OTLP/HTTP egress for the tier collector ONLY, to the host-side +# agent-observability-stack collector (the otlphttp/phoenix third fan-out sink +# in telemetry.yaml). From inside kind the docker host is bridge gateway +# 172.17.0.1 and the AOS stack publishes OTLP/HTTP on host :4318. The broad +# allow-egress-scoped policy admits 172.x only on 443 (T049), so without this +# policy the collector's dial i/o-times-out and the Phoenix sink silently +# never receives traces (observed 2026-10-11: code=000 from a tier-labelled +# probe pod). Scoped to the collector pod alone -- the four agents reach AOS +# only via the tier collector and keep their existing egress. If +# AOS_OTLP_HTTP_ENDPOINT is overridden to a different host, update this +# ipBlock to match. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-aos-otel + namespace: agentic-netops-agents +spec: + podSelector: + matchLabels: + app: agent-otel-collector + policyTypes: ["Egress"] + egress: + - to: + - ipBlock: + cidr: 172.17.0.1/32 + ports: + - protocol: TCP + port: 4318 diff --git a/deploy/agents/telemetry.yaml b/deploy/agents/telemetry.yaml index c609f3a3..8b4718dd 100644 --- a/deploy/agents/telemetry.yaml +++ b/deploy/agents/telemetry.yaml @@ -1,12 +1,25 @@ -# T035: tier telemetry pipeline — one emission, two sinks (research.md Decision 8). +# T035: tier telemetry pipeline — one emission, three sinks (research.md Decision 8). # # Agents emit OTLP exactly ONCE, to this tier-owned collector # (http://agent-otel-collector.agentic-netops-agents.svc:4318, OTLP/HTTP). This collector # is the single fan-out point: a clickhouse exporter into the tier's agent-analytics -# store, and an otlp exporter forwarding to feature 001's unmodified collector at -# otel-collector.agentic-netops-system:4317. No feature-001 file changes. +# store, an otlp exporter forwarding to feature 001's unmodified collector at +# otel-collector.agentic-netops-system:4317, and an otlphttp/phoenix exporter forwarding +# TRACES to the host-side agent-observability-stack collector (Arize Phoenix trace +# backend, UI on host :3006) — the only sink that sees traces from this tier outside +# the cluster. No feature-001 file changes. # # --------------------------------------------------------------------------- +# NOTE (2026-10-11, Phoenix trace work): the shape below was qualified from +# isolated SDK entrypoints. Live spans from the running tier are +# OpenInference-NATIVE (ioa-observe is a rebranded Traceloop SDK): +# ChatLiteLLM.chat carries gen_ai.prompt/completion + gen_ai.usage.* / +# llm.usage.* tokens; the agentic-netops.* attributes stamped in +# agents/common/llm.py do not reach exported spans (SDK context scoping), +# so source-side redaction is inert. Verified live shape + mapping: +# prompts/PHOENIX_TRACES_PROMPT.md (Grounding) and the AOS +# transform/phoenix_agentic_netops. +# # QUALIFIED OTLP SHAPE (T034) — captured live from the pinned # ioa-observe-sdk==1.0.24 (agents/.venv) by POSTing the subject's own # instrumentation entrypoints (Observe.init, @agent, @graph, session_start) @@ -118,6 +131,23 @@ data: retry_on_failure: enabled: true max_elapsed_time: 5m + otlphttp/phoenix: + # Third sink (traces only): the host-side agent-observability-stack collector + # (docker compose; Arize Phoenix 20.x + Postgres). From inside kind the docker + # host is the bridge gateway, so the default targets its host-published OTLP/HTTP + # port; the exporter appends /v1/traces. Override AOS_OTLP_HTTP_ENDPOINT when the + # stack runs elsewhere. Same queue/retry posture as T351/T352: a down stack + # buffers here and never back-pressures the receiver, so the agents are + # unaffected (they only see a failure if THIS collector pod is down, as before). + endpoint: ${env:AOS_OTLP_HTTP_ENDPOINT} + compression: gzip + sending_queue: + enabled: true + num_consumers: 4 + queue_size: 5000 + retry_on_failure: + enabled: true + max_elapsed_time: 5m # health_check must exist: the Deployment's liveness AND readiness probes both # target port 13133, which is this extension's port. Without it nothing listens @@ -136,10 +166,12 @@ data: service: extensions: [health_check] pipelines: + # Traces fan out to all three sinks; metrics stay on the two established ones + # (Phoenix is a trace backend — its metrics pipeline is not fed from this tier). traces: receivers: [otlp] processors: [batch] - exporters: [clickhouse, otlp/feature001] + exporters: [clickhouse, otlp/feature001, otlphttp/phoenix] metrics: receivers: [otlp] processors: [batch] @@ -187,6 +219,15 @@ spec: - name: CLICKHOUSE_PASSWORD valueFrom: secretKeyRef: {name: clickhouse-auth, key: password} + # The otlphttp/phoenix exporter references ${env:AOS_OTLP_HTTP_ENDPOINT} in the + # ConfigMap; if the variable were ever missing the collector would refuse to + # start (same failure class as the clickhouse-auth incident above, which took + # the whole tier down). The env entry is therefore part of the manifest, with + # the lab default inline: from inside kind the docker host is bridge gateway + # 172.17.0.1, and the agent-observability-stack stack publishes OTLP/HTTP on + # host :4318. Override the value when that stack runs elsewhere. + - name: AOS_OTLP_HTTP_ENDPOINT + value: "http://172.17.0.1:4318" # Nothing else creates this database. The OTLP exporter below targets # `database: agent_analytics` and fails its pipeline startup without it -- # "cannot start pipelines: create database: code: 81, message: Database