From 9cf389d3f49c15b84a67b4ec9a40bcadcfa08917 Mon Sep 17 00:00:00 2001 From: blockmandev <194144408+blockmandev@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:40:26 -0600 Subject: [PATCH] telemetry: opt-in OpenTelemetry export as OTLP/HTTP JSON (refs #26) Item 1 of #26. Off unless INCODE_TELEMETRY=1 (or admin.json telemetry) and an OTLP endpoint are set. Exported by hand over the reqwest incode already has, so no OpenTelemetry crates and no gRPC stack: http/json only (another protocol is reported and nothing is sent), and only to https or this machine, since OTEL_EXPORTER_OTLP_HEADERS carries tokens. Metrics as delta sums: sessions, tokens by type and model, cost, tool calls by tool and outcome (allowed when the tool ran, denied when a gate refused it, counted around Toolbox::run_call), turns and their seconds. Events as log records: permission prompts, compactions, user prompts (length only; text only with OTEL_LOG_USER_PROMPTS=1). Exported after each turn on a thread of its own, and at the end of -t. --- README.md | 27 ++ src/acp_agent/mod.rs | 1 + src/admin.rs | 11 + src/agent/api.rs | 6 +- src/cli/agent_turn.rs | 76 +++- src/cli/mod.rs | 4 + src/lib.rs | 1 + src/task_run.rs | 13 + src/telemetry.rs | 824 ++++++++++++++++++++++++++++++++++++++++++ src/toolbox/mod.rs | 51 ++- src/toolbox/tests.rs | 41 +++ 11 files changed, 1047 insertions(+), 8 deletions(-) create mode 100644 src/telemetry.rs diff --git a/README.md b/README.md index 75688fa..7a099c4 100644 --- a/README.md +++ b/README.md @@ -1420,6 +1420,33 @@ there gives way), and the memory engine's files stay where they are. `INCODE_TRACE` (area `migrate`) says what moved. The old `INCORD_*` variables are not read any more: rename them. +### Telemetry (OpenTelemetry) + +Off unless you turn it on. Set `INCODE_TELEMETRY=1` (or `"telemetry": true` in +admin.json) **and** name a collector: + +```sh +export INCODE_TELEMETRY=1 +export OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.com # /v1/metrics and /v1/logs are added +export OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer%20" +``` + +incode sends OTLP over HTTP as JSON (`OTEL_EXPORTER_OTLP_PROTOCOL` unset or +`http/json`; any other protocol is reported and nothing is sent), only to +`https` or to this machine, after every turn and at the end of `-t`. The +per-signal `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` / `..._LOGS_ENDPOINT` override +the base. `incode --admin` shows whether it is on and where it goes. + +- **Metrics** (delta sums): `incode.session.count` (`mode`), + `incode.token.usage` (`type`, `model`), `incode.cost.usage` (USD, priced + models), `incode.tool.calls` (`tool`, `outcome`: `allowed` or `denied`), + `incode.turn.count` and `incode.turn.duration` (seconds). +- **Events** (log records): `incode.permission_prompt` (`tool`, `answer`), + `incode.compaction` (`trigger`, tokens before and after), + `incode.user_prompt` (`prompt_length`). +- **Never sent:** the text of a prompt, a file, or a tool's input or output. A + prompt's text is added to its event only with `OTEL_LOG_USER_PROMPTS=1`. + ### Proxies and certificates Behind a corporate proxy, every outbound request — the model API, web fetch and diff --git a/src/acp_agent/mod.rs b/src/acp_agent/mod.rs index 1d53e2d..172f8bd 100644 --- a/src/acp_agent/mod.rs +++ b/src/acp_agent/mod.rs @@ -233,6 +233,7 @@ impl SessionFactory for CliSessions { config.resume_session = loading.then(|| session_id.to_string()); let prepared = crate::cli::prepare_session(&config, cancel, Instant::now(), session_id) .map_err(|e| e.to_string())?; + crate::telemetry::session_started("acp"); let mut session = prepared.session; // Kept whether or not an engine reads it: it is what `session/load` // replays. diff --git a/src/admin.rs b/src/admin.rs index fb9d7cb..2a3c7ee 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -350,6 +350,11 @@ pub struct AdminConfig { skip_serializing_if = "crate::permissions::rules::RuleLists::is_empty" )] pub permissions: crate::permissions::rules::RuleLists, + /// Export usage to an OpenTelemetry collector ([`crate::telemetry`]): + /// `true` turns it on (as `INCODE_TELEMETRY=1` does), given an + /// `OTEL_EXPORTER_OTLP_ENDPOINT`. Off when unset. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub telemetry: Option, } /// One service that serves API agents and lists them on request. @@ -1133,6 +1138,10 @@ impl AdminConfig { codex_tools.value.as_str(), codex_tools.source.as_str() )); + out.push_str(&format!( + " telemetry {}\n", + crate::telemetry::describe(self.telemetry) + )); let env_style = std::env::var(crate::styles::ENV).ok(); out.push_str(&match crate::styles::chosen( @@ -1631,6 +1640,8 @@ mod tests { ask: vec!["Bash(git push:*)".to_string()], deny: vec!["Read(.env)".to_string()], }, + // Telemetry the owner turned on must stay on. + telemetry: Some(true), }; admin.save(dir.path()).expect("saves"); diff --git a/src/agent/api.rs b/src/agent/api.rs index 94bbd03..ecfbb9b 100644 --- a/src/agent/api.rs +++ b/src/agent/api.rs @@ -971,6 +971,7 @@ impl ApiBackend { match self.compact_with(transport, "", super::compact::KEEP_TURNS) { Ok(Some(done)) => { crate::trace!("compact", "{}", done.describe()); + crate::telemetry::compaction("auto", done.tokens_before, done.tokens_after); // Shown apart from the reply, and never heard by the // transcript or the stop hook: it is the harness speaking. on_event(&AgentEvent::Thinking(format!( @@ -1371,7 +1372,10 @@ impl AgentBackend for ApiBackend { .map_err(|e| e.to_string()) .and_then(|transport| self.compact_with(&transport, instructions, 0)); Some(ran.map(|done| match done { - Some(done) => done.describe(), + Some(done) => { + crate::telemetry::compaction("manual", done.tokens_before, done.tokens_after); + done.describe() + } None => "Nothing to compact yet: the conversation is empty.".to_string(), })) } diff --git a/src/cli/agent_turn.rs b/src/cli/agent_turn.rs index 128106f..fe19249 100644 --- a/src/cli/agent_turn.rs +++ b/src/cli/agent_turn.rs @@ -1379,15 +1379,19 @@ impl AgentSession { self.usage.output_tokens += turn_usage.output_tokens; self.usage.cache_creation_input_tokens += turn_usage.cache_creation_input_tokens; self.usage.cache_read_input_tokens += turn_usage.cache_read_input_tokens; + let model = self + .model() + .map_or_else(|| self.backend_id().to_string(), str::to_string); if turn_usage.total_tokens() > 0 { - let model = self - .model() - .map_or_else(|| self.backend_id().to_string(), str::to_string); match self.usage_by_model.iter_mut().find(|(m, _)| *m == model) { Some((_, used)) => *used = used.plus(turn_usage), - None => self.usage_by_model.push((model, turn_usage)), + None => self.usage_by_model.push((model.clone(), turn_usage)), } } + // The operator's collector, when telemetry is on (`crate::telemetry`): + // the turn's usage and time, exported without anyone waiting. + crate::telemetry::turn_finished(&model, &turn_usage, started.elapsed().as_secs_f64()); + crate::telemetry::export_in_background(); // THE TURN'S CHECKPOINT IS SEALED: only what changed is kept, what // the turn created is added, and — when the session keeps its @@ -1982,6 +1986,70 @@ mod tests { } } + /// A backend whose turn uses 100 tokens in and 20 out. + struct Spending; + + impl AgentBackend for Spending { + fn id(&self) -> &str { + "claude-code" + } + fn run_turn( + &mut self, + _prompt: &str, + on_event: &mut dyn FnMut(&AgentEvent), + ) -> Result { + on_event(&AgentEvent::Usage { + input_tokens: 100, + output_tokens: 20, + cache_creation_tokens: 0, + cache_read_tokens: 0, + }); + on_event(&AgentEvent::Text("done".to_string())); + Ok(TurnOutcome { + session_id: String::new(), + stop_reason: "end_turn".to_string(), + is_error: false, + detail: None, + served_by: Vec::new(), + }) + } + fn session_id(&self) -> Option<&str> { + None + } + } + + /// **A FINISHED TURN'S USAGE GOES TO THE OPERATOR'S COLLECTOR** + /// (`crate::telemetry`), under the model that served it, as one turn. + #[test] + fn a_finished_turn_records_its_tokens_and_one_turn() { + let ws = tempfile::tempdir().expect("ws"); + let mut s = AgentSession::new( + ws.path().to_path_buf(), + "claude-code", + "claude".to_string(), + &crate::guards::Guards::default(), + Vec::new(), + ) + .with_backend(Box::new(Spending)); + let (outcome, metrics, _) = crate::telemetry::tests::captured(|| { + s.turn(ws.path(), "go", &[], &mut |_| {}, &mut |_| {}) + }); + assert!(outcome.is_ok(), "{outcome:?}"); + let metrics = metrics.expect("the turn was recorded"); + let sum = + |name, pairs: &[(&str, &str)]| crate::telemetry::tests::sum_of(&metrics, name, pairs); + assert_eq!( + sum( + "incode.token.usage", + &[("type", "input"), ("model", "claude-code")] + ), + 100.0, + "{metrics}" + ); + assert_eq!(sum("incode.token.usage", &[("type", "output")]), 20.0); + assert_eq!(sum("incode.turn.count", &[]), 1.0); + } + /// **incode's NOTICE IS SHOWN ON A LINE OF ITS OWN**, even when it lands /// mid-reply, and is not part of the reply. #[test] diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 595b556..07bf415 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -78,6 +78,7 @@ impl CliApp { // makes one for each `session/new`. let session_id = uuid::Uuid::new_v4().to_string(); let mut prepared = prepare_session(&self.config, cancel.clone(), booted_at, &session_id)?; + crate::telemetry::session_started("repl"); connect_ext_tools(&prepared); // Someone is at the keyboard only when both ends are a terminal; piped, // a mid-turn question would wait on input nobody can give. @@ -1854,6 +1855,9 @@ pub(crate) fn plan_turn( Ok(added) => added, Err(why) => return TurnPlan::Refused(why), }; + // Its length to the operator's collector; its text only when they asked + // for prompts (`crate::telemetry`). + crate::telemetry::user_prompt(line); match session.memory_prompt(line) { Some(block) => context.push(block), None => { diff --git a/src/lib.rs b/src/lib.rs index ec2096a..5c3c3a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -62,6 +62,7 @@ pub mod subagents; pub mod task_run; pub mod team; pub mod team_mcp; +pub mod telemetry; pub mod text; pub mod tool_bridge; pub mod toolbox; diff --git a/src/task_run.rs b/src/task_run.rs index 38fb290..628ca88 100644 --- a/src/task_run.rs +++ b/src/task_run.rs @@ -463,6 +463,10 @@ fn run_task_hooked( } None => Vec::new(), }; + // The operator's collector, when telemetry is on (`crate::telemetry`). + crate::telemetry::session_started("task"); + crate::telemetry::user_prompt(instruction); + let begun = std::time::Instant::now(); // The owner's installed external tools, started as MCP servers now, in // the background, so they connect while the memory session opens. let tools = connect_ext_tools(workspace); @@ -548,6 +552,15 @@ fn run_task_hooked( let ran = drive(&task, &mut memory, report); memory.end("other"); operator_end(); + // The run is one turn. Exported now, before this process ends: a thread + // of its own would end with it. + let model = report + .model + .clone() + .or_else(|| report.agent.clone()) + .unwrap_or_default(); + crate::telemetry::turn_finished(&model, &report.spent(), begun.elapsed().as_secs_f64()); + crate::telemetry::export_now(); ran } diff --git a/src/telemetry.rs b/src/telemetry.rs new file mode 100644 index 0000000..2619646 --- /dev/null +++ b/src/telemetry.rs @@ -0,0 +1,824 @@ +//! Opt-in OpenTelemetry export: usage as OTLP/HTTP JSON. +//! +//! # When it is on +//! +//! Only when the operator turns it on — `INCODE_TELEMETRY=1`, or +//! `"telemetry": true` in admin.json — AND names a collector with +//! `OTEL_EXPORTER_OTLP_ENDPOINT` (or the per-signal +//! `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` / `..._LOGS_ENDPOINT`). Otherwise +//! nothing is recorded and nothing is sent. +//! +//! - **Protocol:** `http/json` only (`OTEL_EXPORTER_OTLP_PROTOCOL` unset or +//! that), posted by hand with the client incode already has: no +//! OpenTelemetry crates, no gRPC stack. Another protocol turns it off and +//! says so, rather than sending what the collector cannot read. +//! - **Where:** `https`, or `http` to this machine — the rule every key in +//! incode is held to (`agent::catalogue::key_may_travel_to`), since +//! `OTEL_EXPORTER_OTLP_HEADERS` usually carries a token. +//! - **What is never sent:** the contents of a prompt, a file, a tool's +//! input or output. A prompt's text is added to its event only with +//! `OTEL_LOG_USER_PROMPTS=1`; its length always. +//! +//! # What is sent +//! +//! Metrics, as delta sums since the last export: +//! +//! | name | unit | attributes | +//! |---|---|---| +//! | `incode.session.count` | 1 | `mode` (`repl`, `acp`, `task`) | +//! | `incode.token.usage` | tokens | `type` (`input`, `output`, `cache_read`, `cache_creation`), `model` | +//! | `incode.cost.usage` | USD | `model` (priced models only) | +//! | `incode.tool.calls` | 1 | `tool`, `outcome` (`allowed`, `denied`) | +//! | `incode.turn.count` · `incode.turn.duration` | 1 · s | `model` | +//! +//! Events, as log records: `incode.permission_prompt` (`tool`, `answer`), +//! `incode.compaction` (`trigger`, `tokens_before`, `tokens_after`), +//! `incode.user_prompt` (`prompt_length`, and `prompt` only as above). +//! +//! # When it is sent +//! +//! After every turn, on a thread of its own so nobody waits for it; at the +//! end of a `-t` run, before it exits. Each export is bounded at +//! [`EXPORT_BOUND`]; one that fails is traced (`telemetry`) and dropped. +//! +//! # Wired to +//! +//! - [`session_started`] — the REPL's and `--acp`'s callers of +//! `cli::prepare_session`, and `task_run::run_task_hooked` (`-t`). +//! - [`turn_finished`] and [`export_in_background`] — +//! `cli::agent_turn::AgentSession::turn_observed`, where a turn's usage is +//! final. +//! - [`tool_call`] — `toolbox::Toolbox::run_call`, around every call. +//! - [`permission_prompt`] — `toolbox::Toolbox::confirm_with_person` and +//! `ask_protected`, the two places a person is asked. +//! - [`compaction`] — `agent::api::ApiBackend` (`compact`, +//! `compact_if_near_window`). +//! - [`user_prompt`] — `cli::plan_turn`, and `task_run` for `-t`. +//! - [`export_now`] — `task_run`, at the end of a run. +//! - [`describe`] — `admin::AdminConfig::report`. + +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde_json::{json, Value}; + +/// The most one export may take. +pub const EXPORT_BOUND: Duration = Duration::from_secs(5); + +/// Where and how to export. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + metrics_url: String, + logs_url: String, + headers: Vec<(String, String)>, + log_prompts: bool, +} + +/// The export as the environment (`get`) and admin.json's `telemetry` +/// (`admin`) set it: `Ok(None)` when it is off, `Err` with why when it was +/// asked for and cannot be done. +/// +/// # Errors +/// +/// Turned on with no collector named, a protocol other than `http/json`, +/// a collector that is neither `https` nor this machine, or headers that do +/// not parse. +pub fn config_from( + get: &dyn Fn(&str) -> Option, + admin: Option, +) -> Result, String> { + let set = |name: &str| { + get(name) + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + }; + let on = + matches!(set("INCODE_TELEMETRY").as_deref(), Some("1" | "true")) || admin == Some(true); + if !on { + return Ok(None); + } + if let Some(protocol) = set("OTEL_EXPORTER_OTLP_PROTOCOL") { + if protocol != "http/json" { + return Err(format!( + "OTEL_EXPORTER_OTLP_PROTOCOL={protocol}: incode exports only http/json, so \ + telemetry is off" + )); + } + } + let base = set("OTEL_EXPORTER_OTLP_ENDPOINT").map(|b| b.trim_end_matches('/').to_string()); + let signal = + |own: &str, path: &str| set(own).or_else(|| base.as_ref().map(|b| format!("{b}{path}"))); + let (Some(metrics_url), Some(logs_url)) = ( + signal("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "/v1/metrics"), + signal("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "/v1/logs"), + ) else { + return Err( + "telemetry is on but no OTEL_EXPORTER_OTLP_ENDPOINT is set, so nothing is exported" + .to_string(), + ); + }; + for url in [&metrics_url, &logs_url] { + crate::agent::catalogue::key_may_travel_to(url) + .map_err(|why| format!("telemetry is off: {url}: {why}"))?; + } + let headers = match set("OTEL_EXPORTER_OTLP_HEADERS") { + Some(raw) => parse_headers(&raw)?, + None => Vec::new(), + }; + Ok(Some(Config { + metrics_url, + logs_url, + headers, + log_prompts: matches!(set("OTEL_LOG_USER_PROMPTS").as_deref(), Some("1" | "true")), + })) +} + +/// `key=value,key2=value2`, values percent-decoded, as the OpenTelemetry +/// specification writes `OTEL_EXPORTER_OTLP_HEADERS`. +fn parse_headers(raw: &str) -> Result, String> { + raw.split(',') + .map(str::trim) + .filter(|pair| !pair.is_empty()) + .map(|pair| { + let (name, value) = pair.split_once('=').ok_or_else(|| { + "OTEL_EXPORTER_OTLP_HEADERS has an entry with no `=`, so telemetry is off" + .to_string() + })?; + Ok((name.trim().to_string(), percent_decode(value.trim()))) + }) + .collect() +} + +fn percent_decode(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + let hex = std::str::from_utf8(&bytes[i + 1..i + 3]).ok(); + if let Some(byte) = hex.and_then(|h| u8::from_str_radix(h, 16).ok()) { + out.push(byte); + i += 3; + continue; + } + } + out.push(bytes[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +/// A metric's attributes, sorted, as its identity. +type Attributes = Vec<(String, String)>; + +/// What is waiting to be exported. +#[derive(Debug, Default)] +struct Pending { + sums: BTreeMap<(&'static str, Attributes), f64>, + events: Vec<(&'static str, u128, Attributes)>, + since: u128, +} + +/// Records metrics and events in memory and exports them. +#[derive(Debug)] +pub struct Recorder { + config: Config, + pending: Mutex, +} + +fn now_nanos() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) +} + +fn attributes(pairs: &[(&str, &str)]) -> Attributes { + let mut out: Attributes = pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + out.sort(); + out +} + +/// The unit OTLP is told for a metric. +fn unit_of(name: &str) -> &'static str { + match name { + "incode.token.usage" => "tokens", + "incode.cost.usage" => "USD", + "incode.turn.duration" => "s", + _ => "1", + } +} + +fn otlp_attributes(attributes: &[(String, String)]) -> Value { + Value::Array( + attributes + .iter() + .map(|(k, v)| json!({"key": k, "value": {"stringValue": v}})) + .collect(), + ) +} + +fn resource() -> Value { + json!({"attributes": otlp_attributes(&[ + ("service.name".to_string(), "incode".to_string()), + ("service.version".to_string(), env!("CARGO_PKG_VERSION").to_string()), + ])}) +} + +fn scope() -> Value { + json!({"name": "incode", "version": env!("CARGO_PKG_VERSION")}) +} + +impl Recorder { + #[must_use] + pub fn new(config: Config) -> Self { + Self { + config, + pending: Mutex::new(Pending { + since: now_nanos(), + ..Pending::default() + }), + } + } + + fn locked(&self) -> std::sync::MutexGuard<'_, Pending> { + self.pending + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + /// Add `value` to the sum `name` with `pairs`. + pub fn add(&self, name: &'static str, pairs: &[(&str, &str)], value: f64) { + *self + .locked() + .sums + .entry((name, attributes(pairs))) + .or_insert(0.0) += value; + } + + /// Record the event `name` with `pairs`, now. + pub fn event(&self, name: &'static str, pairs: &[(&str, &str)]) { + self.locked() + .events + .push((name, now_nanos(), attributes(pairs))); + } + + /// Whether a prompt's text may go in its event. + #[must_use] + fn logs_prompts(&self) -> bool { + self.config.log_prompts + } + + /// What is pending, as the OTLP/JSON bodies for `/v1/metrics` and + /// `/v1/logs` (`None` for a signal with nothing to say), emptied. + fn drain(&self) -> (Option, Option) { + let now = now_nanos(); + let mut pending = self.locked(); + let since = std::mem::replace(&mut pending.since, now); + let sums = std::mem::take(&mut pending.sums); + let events = std::mem::take(&mut pending.events); + drop(pending); + + let metrics = (!sums.is_empty()).then(|| { + let mut by_name: BTreeMap<&'static str, Vec> = BTreeMap::new(); + for ((name, attributes), value) in sums { + by_name.entry(name).or_default().push(json!({ + "attributes": otlp_attributes(&attributes), + "startTimeUnixNano": since.to_string(), + "timeUnixNano": now.to_string(), + "asDouble": value, + })); + } + let metrics: Vec = by_name + .into_iter() + .map(|(name, points)| { + json!({ + "name": name, + "unit": unit_of(name), + // 1: AGGREGATION_TEMPORALITY_DELTA. + "sum": {"aggregationTemporality": 1, "isMonotonic": true, "dataPoints": points}, + }) + }) + .collect(); + json!({"resourceMetrics": [{ + "resource": resource(), + "scopeMetrics": [{"scope": scope(), "metrics": metrics}], + }]}) + }); + let logs = (!events.is_empty()).then(|| { + let records: Vec = events + .into_iter() + .map(|(name, at, attributes)| { + let mut attributes = attributes; + attributes.push(("event.name".to_string(), name.to_string())); + json!({ + "timeUnixNano": at.to_string(), + "body": {"stringValue": name}, + "attributes": otlp_attributes(&attributes), + }) + }) + .collect(); + json!({"resourceLogs": [{ + "resource": resource(), + "scopeLogs": [{"scope": scope(), "logRecords": records}], + }]}) + }); + (metrics, logs) + } + + /// Send what is pending, and empty it. + /// + /// # Errors + /// + /// A signal the collector did not take: why. What it did not take is + /// dropped, not kept for the next export. + pub fn export(&self) -> Result<(), String> { + let (metrics, logs) = self.drain(); + let mut failed = Vec::new(); + for (url, body) in [ + (&self.config.metrics_url, metrics), + (&self.config.logs_url, logs), + ] { + let Some(body) = body else { continue }; + let (url, headers) = (url.clone(), self.config.headers.clone()); + let sent = crate::runtime::http::request_empty( + move |client| { + let mut request = client + .post(&url) + .timeout(EXPORT_BOUND) + .header("content-type", "application/json") + .body(body.to_string()); + for (name, value) in &headers { + request = request.header(name.as_str(), value.as_str()); + } + request + }, + "telemetry collector", + ); + if let Err(why) = sent { + failed.push(why); + } + } + if failed.is_empty() { + Ok(()) + } else { + Err(failed.join("; ")) + } + } +} + +/// The process's recorder, when telemetry is on. +fn global() -> Option<&'static Recorder> { + static GLOBAL: OnceLock> = OnceLock::new(); + GLOBAL + .get_or_init(|| { + let admin = crate::admin::AdminConfig::load_default().telemetry; + match config_from(&|name| std::env::var(name).ok(), admin) { + Ok(config) => config.map(Recorder::new), + Err(why) => { + eprintln!("incode: {why}"); + None + } + } + }) + .as_ref() +} + +#[cfg(test)] +thread_local! { + /// A test's own recorder for this thread, used instead of the process's. + static CAPTURE: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; +} + +/// Run `record` on the recorder in use: a test's on its thread, else the +/// process's when telemetry is on. +fn with(record: impl FnOnce(&Recorder)) { + #[cfg(test)] + { + if let Some(recorder) = CAPTURE.with(|c| c.borrow().clone()) { + record(&recorder); + return; + } + } + if let Some(recorder) = global() { + record(recorder); + } +} + +/// A session started; `mode` is `repl`, `acp` or `task`. +pub fn session_started(mode: &str) { + with(|r| r.add("incode.session.count", &[("mode", mode)], 1.0)); +} + +/// A turn on `model` finished, using `usage`, after `seconds`. +pub fn turn_finished(model: &str, usage: &crate::runtime::usage::TokenUsage, seconds: f64) { + with(|r| { + for (kind, tokens) in [ + ("input", usage.input_tokens), + ("output", usage.output_tokens), + ("cache_read", usage.cache_read_input_tokens), + ("cache_creation", usage.cache_creation_input_tokens), + ] { + if tokens > 0 { + r.add( + "incode.token.usage", + &[("type", kind), ("model", model)], + f64::from(tokens), + ); + } + } + if let Some(cost) = usage.estimate_cost_usd(Some(model)) { + r.add("incode.cost.usage", &[("model", model)], cost); + } + r.add("incode.turn.count", &[("model", model)], 1.0); + r.add("incode.turn.duration", &[("model", model)], seconds); + }); +} + +/// A tool call was decided: `outcome` is `allowed` (it ran, whatever it +/// then returned) or `denied` (it never ran). +pub fn tool_call(tool: &str, outcome: &str) { + with(|r| { + r.add( + "incode.tool.calls", + &[("tool", tool), ("outcome", outcome)], + 1.0, + ); + }); +} + +/// A person was asked about `tool` and answered `answer` (`allow`, `deny`, +/// or `unanswered` when there was nobody to ask). +pub fn permission_prompt(tool: &str, answer: &str) { + with(|r| { + r.event( + "incode.permission_prompt", + &[("tool", tool), ("answer", answer)], + ) + }); +} + +/// The conversation was compacted (`trigger`: `manual` or `auto`). +pub fn compaction(trigger: &str, tokens_before: u64, tokens_after: u64) { + let (before, after) = (tokens_before.to_string(), tokens_after.to_string()); + with(|r| { + r.event( + "incode.compaction", + &[ + ("trigger", trigger), + ("tokens_before", &before), + ("tokens_after", &after), + ], + ); + }); +} + +/// A person's line went to the agent. Its length always; its text only +/// with `OTEL_LOG_USER_PROMPTS=1`. +pub fn user_prompt(text: &str) { + let length = text.chars().count().to_string(); + with(|r| { + if r.logs_prompts() { + r.event( + "incode.user_prompt", + &[("prompt_length", &length), ("prompt", text)], + ); + } else { + r.event("incode.user_prompt", &[("prompt_length", &length)]); + } + }); +} + +/// Export what is pending on a thread of its own: nobody waits for it. +pub fn export_in_background() { + if let Some(recorder) = global() { + std::thread::spawn(move || { + if let Err(why) = recorder.export() { + crate::trace!("telemetry", "export failed: {why}"); + } + }); + } +} + +/// Export what is pending now, before the process ends (`-t`). +pub fn export_now() { + if let Some(recorder) = global() { + if let Err(why) = recorder.export() { + crate::trace!("telemetry", "export failed: {why}"); + } + } +} + +/// admin.json's `telemetry` and the environment, as `--admin` shows them. +#[must_use] +pub fn describe(admin: Option) -> String { + match config_from(&|name| std::env::var(name).ok(), admin) { + Ok(Some(config)) => format!( + "on: {} · {}{}", + config.metrics_url, + config.logs_url, + if config.log_prompts { + " · prompt text included" + } else { + "" + } + ), + Ok(None) => "off".to_string(), + Err(why) => format!("off: {why}"), + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::io::{Read as _, Write as _}; + + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let pairs: Vec<(String, String)> = pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + move |name| { + pairs + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.clone()) + } + } + + /// Run `f` with a recorder of this thread's own, and return what was + /// recorded as `(metrics, logs)` bodies. For tests of the wiring. + pub(crate) fn captured(f: impl FnOnce() -> T) -> (T, Option, Option) { + let recorder = std::sync::Arc::new(Recorder::new(Config { + metrics_url: "http://127.0.0.1:1/v1/metrics".to_string(), + logs_url: "http://127.0.0.1:1/v1/logs".to_string(), + headers: Vec::new(), + log_prompts: false, + })); + CAPTURE.with(|c| *c.borrow_mut() = Some(std::sync::Arc::clone(&recorder))); + let out = f(); + CAPTURE.with(|c| *c.borrow_mut() = None); + let (metrics, logs) = recorder.drain(); + (out, metrics, logs) + } + + /// The value of the sum `name` whose attributes include every one of + /// `pairs`, in a metrics body. + pub(crate) fn sum_of(metrics: &Value, name: &str, pairs: &[(&str, &str)]) -> f64 { + let mut total = 0.0; + for metric in metrics["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] + .as_array() + .into_iter() + .flatten() + .filter(|m| m["name"] == name) + { + for point in metric["sum"]["dataPoints"].as_array().into_iter().flatten() { + let has = |k: &str, v: &str| { + point["attributes"] + .as_array() + .into_iter() + .flatten() + .any(|a| a["key"] == k && a["value"]["stringValue"] == v) + }; + if pairs.iter().all(|(k, v)| has(k, v)) { + total += point["asDouble"].as_f64().unwrap_or(0.0); + } + } + } + total + } + + /// **OFF UNLESS TURNED ON, AND THEN ONLY TO A COLLECTOR IT MAY REACH, + /// OVER A PROTOCOL IT SPEAKS.** + #[test] + fn telemetry_is_off_unless_turned_on_and_refuses_what_it_cannot_do() { + let endpoint = ("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel.example.com/"); + assert_eq!( + config_from(&env(&[endpoint]), None), + Ok(None), + "not turned on" + ); + assert_eq!(config_from(&env(&[endpoint]), Some(false)), Ok(None)); + + let on = config_from(&env(&[("INCODE_TELEMETRY", "1"), endpoint]), None) + .expect("valid") + .expect("on"); + assert_eq!(on.metrics_url, "https://otel.example.com/v1/metrics"); + assert_eq!(on.logs_url, "https://otel.example.com/v1/logs"); + assert!(!on.log_prompts); + assert!( + config_from(&env(&[endpoint]), Some(true)) + .expect("valid") + .is_some(), + "admin.json turns it on too" + ); + + let refused = + |pairs: &[(&str, &str)]| config_from(&env(pairs), Some(true)).expect_err("refused"); + assert!(refused(&[]).contains("no OTEL_EXPORTER_OTLP_ENDPOINT")); + assert!( + refused(&[endpoint, ("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc")]) + .contains("only http/json") + ); + assert!( + refused(&[("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel.example.com")]) + .contains("telemetry is off"), + "plain http off this machine" + ); + assert!( + refused(&[endpoint, ("OTEL_EXPORTER_OTLP_HEADERS", "nonsense")]).contains("no `=`") + ); + + let local = config_from( + &env(&[ + ( + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + "http://127.0.0.1:4318/m", + ), + ( + "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", + "http://localhost:4318/l", + ), + ( + "OTEL_EXPORTER_OTLP_HEADERS", + "authorization=Bearer%20abc, x-team = core", + ), + ("OTEL_LOG_USER_PROMPTS", "1"), + ]), + Some(true), + ) + .expect("valid") + .expect("on"); + assert_eq!(local.metrics_url, "http://127.0.0.1:4318/m"); + assert_eq!( + local.headers, + [ + ("authorization".to_string(), "Bearer abc".to_string()), + ("x-team".to_string(), "core".to_string()) + ] + ); + assert!(local.log_prompts); + } + + /// A collector on 127.0.0.1 that keeps each request's path, headers and + /// body: its base URL and what it received. + fn collector(times: usize) -> (String, std::sync::mpsc::Receiver<(String, String, Value)>) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + for stream in listener.incoming().take(times) { + let Ok(mut stream) = stream else { continue }; + let mut seen = Vec::new(); + let mut buf = [0u8; 16384]; + let (head, body) = loop { + let Ok(n) = stream.read(&mut buf) else { return }; + if n == 0 { + return; + } + seen.extend_from_slice(&buf[..n]); + let text = String::from_utf8_lossy(&seen).to_string(); + if let Some(end) = text.find("\r\n\r\n") { + let length = text[..end] + .lines() + .find_map(|l| { + l.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(|v| v.trim().parse::().unwrap_or(0)) + }) + .unwrap_or(0); + if seen.len() >= end + 4 + length { + break ( + text[..end].to_string(), + text[end + 4..end + 4 + length].to_string(), + ); + } + } + }; + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}", + ); + let path = head.split_whitespace().nth(1).unwrap_or("").to_string(); + let body = serde_json::from_str(&body).unwrap_or(Value::Null); + let _ = tx.send((path, head, body)); + } + }); + (format!("http://127.0.0.1:{port}"), rx) + } + + /// **WHAT IS RECORDED REACHES THE COLLECTOR AS OTLP/JSON**: metrics as + /// delta sums to `/v1/metrics`, events as log records to `/v1/logs`, the + /// operator's headers on both — and no prompt text unless asked for. + #[test] + fn recorded_usage_reaches_a_collector_as_otlp_json() { + let (base, received) = collector(2); + let config = config_from( + &env(&[ + ("OTEL_EXPORTER_OTLP_ENDPOINT", &base), + ("OTEL_EXPORTER_OTLP_HEADERS", "x-api-key=k1"), + ]), + Some(true), + ) + .expect("valid") + .expect("on"); + let recorder = std::sync::Arc::new(Recorder::new(config)); + CAPTURE.with(|c| *c.borrow_mut() = Some(std::sync::Arc::clone(&recorder))); + session_started("repl"); + turn_finished( + "claude-sonnet-4", + &crate::runtime::usage::TokenUsage { + input_tokens: 1000, + output_tokens: 200, + ..Default::default() + }, + 2.5, + ); + tool_call("Write", "denied"); + tool_call("Read", "allowed"); + tool_call("Read", "allowed"); + permission_prompt("Bash", "deny"); + compaction("auto", 9000, 3000); + user_prompt("delete the secret plans"); + CAPTURE.with(|c| *c.borrow_mut() = None); + recorder.export().expect("the collector took both"); + + let mut got: BTreeMap = BTreeMap::new(); + for _ in 0..2 { + let (path, head, body) = received + .recv_timeout(Duration::from_secs(10)) + .expect("an export arrived"); + got.insert(path, (head, body)); + } + let (head, metrics) = &got["/v1/metrics"]; + assert!( + head.to_ascii_lowercase().contains("x-api-key: k1"), + "{head}" + ); + assert_eq!( + sum_of(metrics, "incode.session.count", &[("mode", "repl")]), + 1.0 + ); + assert_eq!( + sum_of( + metrics, + "incode.token.usage", + &[("type", "input"), ("model", "claude-sonnet-4")] + ), + 1000.0 + ); + assert_eq!( + sum_of(metrics, "incode.token.usage", &[("type", "output")]), + 200.0 + ); + assert!( + sum_of(metrics, "incode.cost.usage", &[]) > 0.0, + "a priced model has a cost" + ); + assert_eq!( + sum_of( + metrics, + "incode.tool.calls", + &[("tool", "Read"), ("outcome", "allowed")] + ), + 2.0 + ); + assert_eq!( + sum_of(metrics, "incode.tool.calls", &[("outcome", "denied")]), + 1.0 + ); + assert_eq!(sum_of(metrics, "incode.turn.duration", &[]), 2.5); + let metric = metrics["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0].clone(); + assert_eq!(metric["sum"]["aggregationTemporality"], 1, "delta"); + + let (_, logs) = &got["/v1/logs"]; + let text = logs.to_string(); + for name in [ + "incode.permission_prompt", + "incode.compaction", + "incode.user_prompt", + ] { + assert!(text.contains(name), "{name} missing: {text}"); + } + assert!(text.contains("\"prompt_length\""), "{text}"); + assert!( + !text.contains("secret plans"), + "a prompt's text went out unasked: {text}" + ); + + // Exported means emptied: nothing goes twice. + let (metrics, logs) = recorder.drain(); + assert!(metrics.is_none() && logs.is_none()); + } + + #[test] + fn a_percent_encoded_header_value_is_decoded() { + assert_eq!(percent_decode("Bearer%20a%2Cb"), "Bearer a,b"); + assert_eq!(percent_decode("100%"), "100%"); + assert_eq!(percent_decode("%zz"), "%zz"); + assert_eq!(percent_decode("é%2"), "é%2"); + } +} diff --git a/src/toolbox/mod.rs b/src/toolbox/mod.rs index e35da57..d4291bb 100644 --- a/src/toolbox/mod.rs +++ b/src/toolbox/mod.rs @@ -468,6 +468,13 @@ enum Gate { /// characters. The whole reason is still the call's result. const MAX_DECISION_REASON_CHARS: usize = 500; +thread_local! { + /// Set when the call [`Toolbox::run_call`] is deciding reaches the tool + /// itself: every gate passed. Read by `run_call` to count it `allowed` or + /// `denied` (`crate::telemetry`). + static CALL_RAN: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + /// Why the operator's guards refused a call: the reason the model is told, /// and the protected pattern and the path when one matched. struct Refusal { @@ -1339,6 +1346,14 @@ impl Toolbox { "tool", "{name}: protected `{rule}` put to a person: {answer:?}" ); + crate::telemetry::permission_prompt( + name, + match &answer { + Some(crate::hooks::engine::Verdict::Allow) => "allow", + Some(crate::hooks::engine::Verdict::Deny(_)) => "deny", + None => "unanswered", + }, + ); answer } @@ -1650,7 +1665,27 @@ impl Toolbox { /// [`Self::run`], for the call the model numbered `call_id` — the id a /// person is shown if the call has to be confirmed — with the answer's /// pictures kept, for a caller that can send them on. + /// + /// Its outcome goes to the operator's collector when telemetry is on + /// (`crate::telemetry::tool_call`): `allowed` when every gate passed and + /// the tool ran (however it then ended), `denied` when one refused it. pub fn run_call(&self, call_id: &str, name: &str, input: &Value) -> Result { + // Saved and put back, so a sub-agent's calls made inside this one + // (on this thread) cannot change how this one is counted. + let outer = CALL_RAN.with(|ran| ran.replace(false)); + let result = self.decide_and_run(call_id, name, input); + let ran = CALL_RAN.with(|ran| ran.replace(outer)); + crate::telemetry::tool_call(name, if ran { "allowed" } else { "denied" }); + result + } + + /// Every gate, then the tool: [`Self::run_call`]'s body. + fn decide_and_run( + &self, + call_id: &str, + name: &str, + input: &Value, + ) -> Result { // A name nobody classified is not a tool: refused before a guard, a // person or the engine is asked about it. A tool of the client's MCP // servers RUNS something nobody here can inspect, so it is `run`: @@ -1860,6 +1895,8 @@ impl Toolbox { }, ); } + // Every gate passed: the call RUNS, whatever it then returns. + CALL_RAN.with(|ran| ran.set(true)); let outcome = self.dispatch(call_id, name, input, reach); // Confirmed or cancelled as it came out; a read recorded once it was // shown, so the next write's check knows what this session saw. @@ -2149,11 +2186,19 @@ impl Toolbox { .as_ref() .is_some_and(crate::runtime::cancel::CancelToken::is_cancelled) }; - match self + let answer = self .prompter .as_ref() - .and_then(|p| p.confirm(&ask, &stopped)) - { + .and_then(|p| p.confirm(&ask, &stopped)); + crate::telemetry::permission_prompt( + name, + match &answer { + Some(crate::hooks::engine::Verdict::Allow) => "allow", + Some(crate::hooks::engine::Verdict::Deny(_)) => "deny", + None => "unanswered", + }, + ); + match answer { Some(crate::hooks::engine::Verdict::Allow) => Ok(()), Some(crate::hooks::engine::Verdict::Deny(why)) => Err(why), None => Err(format!("NOT RUN — {reason}, and nobody could be asked.")), diff --git a/src/toolbox/tests.rs b/src/toolbox/tests.rs index f39e370..20b0f55 100644 --- a/src/toolbox/tests.rs +++ b/src/toolbox/tests.rs @@ -24,6 +24,47 @@ fn every_advertised_tool_can_actually_be_run() { } } +/// **EVERY CALL IS COUNTED BY WHAT WAS DECIDED, NOT BY HOW IT ENDED** +/// (`crate::telemetry`): refused before it ran is `denied`; ran is +/// `allowed`, a read of a missing file included — the tool ran and failed. +#[test] +fn a_tool_call_is_counted_allowed_when_it_ran_and_denied_when_refused() { + let dir = tempfile::tempdir().expect("tmp"); + std::fs::write(dir.path().join("here.txt"), "hi").expect("write"); + let mut tools = toolbox(dir.path()); + tools.restrict_tools(vec!["Read".to_string()]); + let ((), metrics, _) = crate::telemetry::tests::captured(|| { + tools + .run("Write", &json!({"path": "x.txt", "content": "no"})) + .expect_err("Write is not in this toolbox's list"); + tools + .run("Read", &json!({"path": "here.txt"})) + .expect("read"); + tools + .run("Read", &json!({"path": "missing.txt"})) + .expect_err("no such file"); + }); + let metrics = metrics.expect("calls were counted"); + let count = |tool, outcome| { + crate::telemetry::tests::sum_of( + &metrics, + "incode.tool.calls", + &[("tool", tool), ("outcome", outcome)], + ) + }; + assert_eq!(count("Write", "denied"), 1.0, "{metrics}"); + assert_eq!(count("Read", "allowed"), 2.0, "{metrics}"); + assert_eq!( + count("Read", "denied"), + 0.0, + "a failed read is not a refusal" + ); + assert!( + !std::path::Path::new(&dir.path().join("x.txt")).exists(), + "the refused write ran" + ); +} + #[test] fn an_unknown_tool_is_an_error_the_model_can_read() { let dir = tempfile::tempdir().expect("tmp");