Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1420,6 +1420,33 @@ there gives way), and the memory
engine's files stay where they are. `INCODE_TRACE` (area `migrate`) says what
moved. The old `INCORD_*` variables are not read any more: rename them.

### Telemetry (OpenTelemetry)

Off unless you turn it on. Set `INCODE_TELEMETRY=1` (or `"telemetry": true` in
admin.json) **and** name a collector:

```sh
export INCODE_TELEMETRY=1
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otel.example.com # /v1/metrics and /v1/logs are added
export OTEL_EXPORTER_OTLP_HEADERS="authorization=Bearer%20<token>"
```

incode sends OTLP over HTTP as JSON (`OTEL_EXPORTER_OTLP_PROTOCOL` unset or
`http/json`; any other protocol is reported and nothing is sent), only to
`https` or to this machine, after every turn and at the end of `-t`. The
per-signal `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` / `..._LOGS_ENDPOINT` override
the base. `incode --admin` shows whether it is on and where it goes.

- **Metrics** (delta sums): `incode.session.count` (`mode`),
`incode.token.usage` (`type`, `model`), `incode.cost.usage` (USD, priced
models), `incode.tool.calls` (`tool`, `outcome`: `allowed` or `denied`),
`incode.turn.count` and `incode.turn.duration` (seconds).
- **Events** (log records): `incode.permission_prompt` (`tool`, `answer`),
`incode.compaction` (`trigger`, tokens before and after),
`incode.user_prompt` (`prompt_length`).
- **Never sent:** the text of a prompt, a file, or a tool's input or output. A
prompt's text is added to its event only with `OTEL_LOG_USER_PROMPTS=1`.

### Proxies and certificates

Behind a corporate proxy, every outbound request — the model API, web fetch and
Expand Down
1 change: 1 addition & 0 deletions src/acp_agent/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ impl SessionFactory for CliSessions {
config.resume_session = loading.then(|| session_id.to_string());
let prepared = crate::cli::prepare_session(&config, cancel, Instant::now(), session_id)
.map_err(|e| e.to_string())?;
crate::telemetry::session_started("acp");
let mut session = prepared.session;
// Kept whether or not an engine reads it: it is what `session/load`
// replays.
Expand Down
11 changes: 11 additions & 0 deletions src/admin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,11 @@ pub struct AdminConfig {
skip_serializing_if = "crate::permissions::rules::RuleLists::is_empty"
)]
pub permissions: crate::permissions::rules::RuleLists,
/// Export usage to an OpenTelemetry collector ([`crate::telemetry`]):
/// `true` turns it on (as `INCODE_TELEMETRY=1` does), given an
/// `OTEL_EXPORTER_OTLP_ENDPOINT`. Off when unset.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub telemetry: Option<bool>,
}

/// One service that serves API agents and lists them on request.
Expand Down Expand Up @@ -1133,6 +1138,10 @@ impl AdminConfig {
codex_tools.value.as_str(),
codex_tools.source.as_str()
));
out.push_str(&format!(
" telemetry {}\n",
crate::telemetry::describe(self.telemetry)
));

let env_style = std::env::var(crate::styles::ENV).ok();
out.push_str(&match crate::styles::chosen(
Expand Down Expand Up @@ -1631,6 +1640,8 @@ mod tests {
ask: vec!["Bash(git push:*)".to_string()],
deny: vec!["Read(.env)".to_string()],
},
// Telemetry the owner turned on must stay on.
telemetry: Some(true),
};
admin.save(dir.path()).expect("saves");

Expand Down
6 changes: 5 additions & 1 deletion src/agent/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -971,6 +971,7 @@ impl ApiBackend {
match self.compact_with(transport, "", super::compact::KEEP_TURNS) {
Ok(Some(done)) => {
crate::trace!("compact", "{}", done.describe());
crate::telemetry::compaction("auto", done.tokens_before, done.tokens_after);
// Shown apart from the reply, and never heard by the
// transcript or the stop hook: it is the harness speaking.
on_event(&AgentEvent::Thinking(format!(
Expand Down Expand Up @@ -1371,7 +1372,10 @@ impl AgentBackend for ApiBackend {
.map_err(|e| e.to_string())
.and_then(|transport| self.compact_with(&transport, instructions, 0));
Some(ran.map(|done| match done {
Some(done) => done.describe(),
Some(done) => {
crate::telemetry::compaction("manual", done.tokens_before, done.tokens_after);
done.describe()
}
None => "Nothing to compact yet: the conversation is empty.".to_string(),
}))
}
Expand Down
76 changes: 72 additions & 4 deletions src/cli/agent_turn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1379,15 +1379,19 @@ impl AgentSession {
self.usage.output_tokens += turn_usage.output_tokens;
self.usage.cache_creation_input_tokens += turn_usage.cache_creation_input_tokens;
self.usage.cache_read_input_tokens += turn_usage.cache_read_input_tokens;
let model = self
.model()
.map_or_else(|| self.backend_id().to_string(), str::to_string);
if turn_usage.total_tokens() > 0 {
let model = self
.model()
.map_or_else(|| self.backend_id().to_string(), str::to_string);
match self.usage_by_model.iter_mut().find(|(m, _)| *m == model) {
Some((_, used)) => *used = used.plus(turn_usage),
None => self.usage_by_model.push((model, turn_usage)),
None => self.usage_by_model.push((model.clone(), turn_usage)),
}
}
// The operator's collector, when telemetry is on (`crate::telemetry`):
// the turn's usage and time, exported without anyone waiting.
crate::telemetry::turn_finished(&model, &turn_usage, started.elapsed().as_secs_f64());
crate::telemetry::export_in_background();

// THE TURN'S CHECKPOINT IS SEALED: only what changed is kept, what
// the turn created is added, and — when the session keeps its
Expand Down Expand Up @@ -1982,6 +1986,70 @@ mod tests {
}
}

/// A backend whose turn uses 100 tokens in and 20 out.
struct Spending;

impl AgentBackend for Spending {
fn id(&self) -> &str {
"claude-code"
}
fn run_turn(
&mut self,
_prompt: &str,
on_event: &mut dyn FnMut(&AgentEvent),
) -> Result<TurnOutcome, crate::agent::AgentError> {
on_event(&AgentEvent::Usage {
input_tokens: 100,
output_tokens: 20,
cache_creation_tokens: 0,
cache_read_tokens: 0,
});
on_event(&AgentEvent::Text("done".to_string()));
Ok(TurnOutcome {
session_id: String::new(),
stop_reason: "end_turn".to_string(),
is_error: false,
detail: None,
served_by: Vec::new(),
})
}
fn session_id(&self) -> Option<&str> {
None
}
}

/// **A FINISHED TURN'S USAGE GOES TO THE OPERATOR'S COLLECTOR**
/// (`crate::telemetry`), under the model that served it, as one turn.
#[test]
fn a_finished_turn_records_its_tokens_and_one_turn() {
let ws = tempfile::tempdir().expect("ws");
let mut s = AgentSession::new(
ws.path().to_path_buf(),
"claude-code",
"claude".to_string(),
&crate::guards::Guards::default(),
Vec::new(),
)
.with_backend(Box::new(Spending));
let (outcome, metrics, _) = crate::telemetry::tests::captured(|| {
s.turn(ws.path(), "go", &[], &mut |_| {}, &mut |_| {})
});
assert!(outcome.is_ok(), "{outcome:?}");
let metrics = metrics.expect("the turn was recorded");
let sum =
|name, pairs: &[(&str, &str)]| crate::telemetry::tests::sum_of(&metrics, name, pairs);
assert_eq!(
sum(
"incode.token.usage",
&[("type", "input"), ("model", "claude-code")]
),
100.0,
"{metrics}"
);
assert_eq!(sum("incode.token.usage", &[("type", "output")]), 20.0);
assert_eq!(sum("incode.turn.count", &[]), 1.0);
}

/// **incode's NOTICE IS SHOWN ON A LINE OF ITS OWN**, even when it lands
/// mid-reply, and is not part of the reply.
#[test]
Expand Down
4 changes: 4 additions & 0 deletions src/cli/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ impl CliApp {
// makes one for each `session/new`.
let session_id = uuid::Uuid::new_v4().to_string();
let mut prepared = prepare_session(&self.config, cancel.clone(), booted_at, &session_id)?;
crate::telemetry::session_started("repl");
connect_ext_tools(&prepared);
// Someone is at the keyboard only when both ends are a terminal; piped,
// a mid-turn question would wait on input nobody can give.
Expand Down Expand Up @@ -1854,6 +1855,9 @@ pub(crate) fn plan_turn(
Ok(added) => added,
Err(why) => return TurnPlan::Refused(why),
};
// Its length to the operator's collector; its text only when they asked
// for prompts (`crate::telemetry`).
crate::telemetry::user_prompt(line);
match session.memory_prompt(line) {
Some(block) => context.push(block),
None => {
Expand Down
1 change: 1 addition & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ pub mod subagents;
pub mod task_run;
pub mod team;
pub mod team_mcp;
pub mod telemetry;
pub mod text;
pub mod tool_bridge;
pub mod toolbox;
Expand Down
13 changes: 13 additions & 0 deletions src/task_run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,10 @@ fn run_task_hooked(
}
None => Vec::new(),
};
// The operator's collector, when telemetry is on (`crate::telemetry`).
crate::telemetry::session_started("task");
crate::telemetry::user_prompt(instruction);
let begun = std::time::Instant::now();
// The owner's installed external tools, started as MCP servers now, in
// the background, so they connect while the memory session opens.
let tools = connect_ext_tools(workspace);
Expand Down Expand Up @@ -548,6 +552,15 @@ fn run_task_hooked(
let ran = drive(&task, &mut memory, report);
memory.end("other");
operator_end();
// The run is one turn. Exported now, before this process ends: a thread
// of its own would end with it.
let model = report
.model
.clone()
.or_else(|| report.agent.clone())
.unwrap_or_default();
crate::telemetry::turn_finished(&model, &report.spent(), begun.elapsed().as_secs_f64());
crate::telemetry::export_now();
ran
}

Expand Down
Loading
Loading