feat(agents): fan out intent-tier traces to Phoenix via AOS

agents/common/llm.py

@@ -137,6 +137,20 @@ async def _wrapped_ainvoke(*args, **kwargs):  # type: ignore[override]
             tokens_out = int(usage.get("completion_tokens", 0) or 0)
             cost = float(getattr(res, "cost", 0.0) or 0.0)
             metrics.model_call(model=model_name, tokens_in=tokens_in, tokens_out=tokens_out, cost_usd=cost)
+            # Token counts on the active span, next to the model id and redacted
+            # prompt/response already stamped here: trace backends (Arize Phoenix via
+            # the agent-observability-stack collector's OpenInference transform) map
+            # agentic-netops.tokens.* to llm.token_count.* so each model call shows
+            # its usage in the trace tree. NOTE (2026-10-11): the SDK's context
+            # scoping currently keeps agentic-netops.* off exported spans (the
+            # OpenInference instrumentation already records usage natively as
+            # gen_ai.usage.* / llm.usage.total_tokens), so this stamping is
+            # redundant-but-harmless until the SDK context is fixed.
+            span = trace.get_current_span()
+            if span is not None and hasattr(span, "set_attribute"):
+                span.set_attribute("agentic-netops.tokens.prompt", tokens_in)
+                span.set_attribute("agentic-netops.tokens.completion", tokens_out)
+                span.set_attribute("agentic-netops.tokens.total", tokens_in + tokens_out)
             # Update per-thread token budget
             if _TOK_BUDGET > 0:
                 _add_tokens(tokens_in + tokens_out)
@@ -178,6 +192,12 @@ def _wrapped_invoke(*args, **kwargs):  # type: ignore[override]
             tokens_out = int(usage.get("completion_tokens", 0) or 0)
             cost = float(getattr(res, "cost", 0.0) or 0.0)
             metrics.model_call(model=model_name, tokens_in=tokens_in, tokens_out=tokens_out, cost_usd=cost)
+            # Token counts on the active span (see the ainvoke wrapper above for why).
+            span = trace.get_current_span()
+            if span is not None and hasattr(span, "set_attribute"):
+                span.set_attribute("agentic-netops.tokens.prompt", tokens_in)
+                span.set_attribute("agentic-netops.tokens.completion", tokens_out)
+                span.set_attribute("agentic-netops.tokens.total", tokens_in + tokens_out)
             if _TOK_BUDGET > 0:
                 _add_tokens(tokens_in + tokens_out)
             # Record redacted response on the span (T335)

deploy/agents/README.md

@@ -14,6 +14,39 @@ with its translator sidecar, ui, otel collector, clickhouse, RBAC,
 NetworkPolicies) land in later phases. The manifests follow the supervisor's
 contract for the Kubernetes objects it submits.
 
+## Telemetry (traces)
+
+Each agent emits OTLP exactly once to the tier-owned `agent-otel-collector`
+(research.md Decision 8), which is the single fan-out point with three sinks:
+
+- `clickhouse` — the tier's `agent_analytics` store (traces + metrics).
+- `otlp/feature001` — feature 001's collector (metrics only; its filter keeps
+  `agentic_netops_agent_*` metrics into Prometheus).
+- `otlphttp/phoenix` — **traces only**, to the host-side
+  `mairp/agent-observability-stack` collector (Arize Phoenix trace backend).
+  Endpoint comes from `AOS_OTLP_HTTP_ENDPOINT` (lab default
+  `http://172.17.0.1:4318`, the docker bridge gateway from inside kind).
+  Traces appear in Phoenix (UI host `:3006`) as one project per
+  `service.name` — `intent-supervisor`, `intent-mapper`, `intent-allocator`,
+  `intent-deployer` — though A2A round-trips propagate W3C context, so a
+  request's full trace (client + remote agent spans) files under the
+  supervisor's project. The spans are OpenInference-native (ioa-observe is a
+  rebranded Traceloop SDK): the supervisor's `ChatLiteLLM.chat` spans carry
+  full prompt/completion content plus token counts, and the AOS collector
+  maps them to LLM/AGENT kinds and fixes the model name
+  (`transform/phoenix_agentic_netops`). Note: the `agentic-netops.*` span
+  attributes stamped in `agents/common/llm.py` (incl. the T333-T335 redacted
+  prompt/response) do not reach exported spans — the SDK's context scoping
+  keeps them off the spans — so content is captured as-is. The collector's
+  egress to the AOS host is admitted by the `allow-egress-aos-otel`
+  NetworkPolicy (`namespace-rbac.yaml`, T051) — collector pod only; without
+  it the dial is dropped by the deny-all baseline and the sink fails
+  silently. Note: the collector mounts its config as a file, so applying a
+  changed `telemetry.yaml` also needs `kubectl rollout restart
+  deploy/agent-otel-collector -n agentic-netops-agents`. Kill switch:
+  remove `otlphttp/phoenix` from the traces pipeline exporters and re-apply
+  (plus the restart).
+
 ## Credential policy
 
 No credential literal may appear anywhere under this directory or under

deploy/agents/namespace-rbac.yaml

@@ -358,3 +358,32 @@ spec:
       port: 443
     - protocol: TCP
       port: 6443
+---
+# T051: OTLP/HTTP egress for the tier collector ONLY, to the host-side
+# agent-observability-stack collector (the otlphttp/phoenix third fan-out sink
+# in telemetry.yaml). From inside kind the docker host is bridge gateway
+# 172.17.0.1 and the AOS stack publishes OTLP/HTTP on host :4318. The broad
+# allow-egress-scoped policy admits 172.x only on 443 (T049), so without this
+# policy the collector's dial i/o-times-out and the Phoenix sink silently
+# never receives traces (observed 2026-10-11: code=000 from a tier-labelled
+# probe pod). Scoped to the collector pod alone -- the four agents reach AOS
+# only via the tier collector and keep their existing egress. If
+# AOS_OTLP_HTTP_ENDPOINT is overridden to a different host, update this
+# ipBlock to match.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-egress-aos-otel
+  namespace: agentic-netops-agents
+spec:
+  podSelector:
+    matchLabels:
+      app: agent-otel-collector
+  policyTypes: ["Egress"]
+  egress:
+  - to:
+    - ipBlock:
+        cidr: 172.17.0.1/32
+    ports:
+    - protocol: TCP
+      port: 4318

deploy/agents/telemetry.yaml

@@ -1,12 +1,25 @@
-# T035: tier telemetry pipeline — one emission, two sinks (research.md Decision 8).
+# T035: tier telemetry pipeline — one emission, three sinks (research.md Decision 8).
 #
 # Agents emit OTLP exactly ONCE, to this tier-owned collector
 # (http://agent-otel-collector.agentic-netops-agents.svc:4318, OTLP/HTTP). This collector
 # is the single fan-out point: a clickhouse exporter into the tier's agent-analytics
-# store, and an otlp exporter forwarding to feature 001's unmodified collector at
-# otel-collector.agentic-netops-system:4317. No feature-001 file changes.
+# store, an otlp exporter forwarding to feature 001's unmodified collector at
+# otel-collector.agentic-netops-system:4317, and an otlphttp/phoenix exporter forwarding
+# TRACES to the host-side agent-observability-stack collector (Arize Phoenix trace
+# backend, UI on host :3006) — the only sink that sees traces from this tier outside
+# the cluster. No feature-001 file changes.
 #
 # ---------------------------------------------------------------------------
+# NOTE (2026-10-11, Phoenix trace work): the shape below was qualified from
+# isolated SDK entrypoints. Live spans from the running tier are
+# OpenInference-NATIVE (ioa-observe is a rebranded Traceloop SDK):
+# ChatLiteLLM.chat carries gen_ai.prompt/completion + gen_ai.usage.* /
+# llm.usage.* tokens; the agentic-netops.* attributes stamped in
+# agents/common/llm.py do not reach exported spans (SDK context scoping),
+# so source-side redaction is inert. Verified live shape + mapping:
+# prompts/PHOENIX_TRACES_PROMPT.md (Grounding) and the AOS
+# transform/phoenix_agentic_netops.
+#
 # QUALIFIED OTLP SHAPE (T034) — captured live from the pinned
 # ioa-observe-sdk==1.0.24 (agents/.venv) by POSTing the subject's own
 # instrumentation entrypoints (Observe.init, @agent, @graph, session_start)
@@ -118,6 +131,23 @@ data:
         retry_on_failure:
           enabled: true
           max_elapsed_time: 5m
+      otlphttp/phoenix:
+        # Third sink (traces only): the host-side agent-observability-stack collector
+        # (docker compose; Arize Phoenix 20.x + Postgres). From inside kind the docker
+        # host is the bridge gateway, so the default targets its host-published OTLP/HTTP
+        # port; the exporter appends /v1/traces. Override AOS_OTLP_HTTP_ENDPOINT when the
+        # stack runs elsewhere. Same queue/retry posture as T351/T352: a down stack
+        # buffers here and never back-pressures the receiver, so the agents are
+        # unaffected (they only see a failure if THIS collector pod is down, as before).
+        endpoint: ${env:AOS_OTLP_HTTP_ENDPOINT}
+        compression: gzip
+        sending_queue:
+          enabled: true
+          num_consumers: 4
+          queue_size: 5000
+        retry_on_failure:
+          enabled: true
+          max_elapsed_time: 5m
 
     # health_check must exist: the Deployment's liveness AND readiness probes both
     # target port 13133, which is this extension's port. Without it nothing listens
@@ -136,10 +166,12 @@ data:
     service:
       extensions: [health_check]
       pipelines:
+        # Traces fan out to all three sinks; metrics stay on the two established ones
+        # (Phoenix is a trace backend — its metrics pipeline is not fed from this tier).
         traces:
           receivers: [otlp]
           processors: [batch]
-          exporters: [clickhouse, otlp/feature001]
+          exporters: [clickhouse, otlp/feature001, otlphttp/phoenix]
         metrics:
           receivers: [otlp]
           processors: [batch]
@@ -187,6 +219,15 @@ spec:
         - name: CLICKHOUSE_PASSWORD
           valueFrom:
             secretKeyRef: {name: clickhouse-auth, key: password}
+        # The otlphttp/phoenix exporter references ${env:AOS_OTLP_HTTP_ENDPOINT} in the
+        # ConfigMap; if the variable were ever missing the collector would refuse to
+        # start (same failure class as the clickhouse-auth incident above, which took
+        # the whole tier down). The env entry is therefore part of the manifest, with
+        # the lab default inline: from inside kind the docker host is bridge gateway
+        # 172.17.0.1, and the agent-observability-stack stack publishes OTLP/HTTP on
+        # host :4318. Override the value when that stack runs elsewhere.
+        - name: AOS_OTLP_HTTP_ENDPOINT
+          value: "http://172.17.0.1:4318"
         # Nothing else creates this database. The OTLP exporter below targets
         # `database: agent_analytics` and fails its pipeline startup without it --
         # "cannot start pipelines: create database: code: 81, message: Database