diff --git a/.changeset/local-span-content.md b/.changeset/local-span-content.md new file mode 100644 index 000000000..848016477 --- /dev/null +++ b/.changeset/local-span-content.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Local trace spans now capture model and tool payloads: the system prompt, prompt messages, and response text/reasoning/tool calls on model spans, and call arguments/results on tool spans, each capped at 32 KB with provider transport metadata stripped. Set `EVE_TRACES_CONTENT=off` to keep payloads out of the spool. diff --git a/docs/guides/instrumentation.md b/docs/guides/instrumentation.md index 78305a9df..d62f6b2bc 100644 --- a/docs/guides/instrumentation.md +++ b/docs/guides/instrumentation.md @@ -13,7 +13,7 @@ When no authored `instrumentation.ts` exists, `eve dev` records agent, AI SDK, a The directory is an immutable OTLP/JSON spool and remains available after `eve dev` exits, subject to the [retention policy](#local-trace-retention) below. Inspection tools may build a query index from these segments, but the index is derived and can be rebuilt without changing the captured trace data. -Use `eve traces ls` to list captured traces and `eve traces ` to inspect a session's span tree. +Use `eve traces ls` to list captured traces and `eve traces ` to inspect a session's span tree. Model and tool-call spans also carry the inputs and outputs (system prompt, prompt messages, and response text for models; call arguments and results for tools, each capped at 32 KB) — set `EVE_TRACES_CONTENT=off` to keep payloads out of the spool. When a model call is served by Vercel AI Gateway, its `agent.step` span also carries the cost the gateway reported: `gen_ai.usage.cost` (raw inference, USD), `gen_ai.usage.gateway_cost` (with the gateway surcharge), `gen_ai.usage.input_cost` / `gen_ai.usage.output_cost` (the split), and `gen_ai.generation.id` for reconciliation with the gateway dashboard. These attributes only exist for gateway-served calls — other providers emit nothing. Cost per turn is the sum across the turn's step spans. @@ -34,12 +34,13 @@ Whatever survives is then evicted oldest-first while the store exceeds `EVE_TRAC Open sessions are tracked per dev worker, so a session waiting across a restart, or a trace a draining worker is still writing, is not covered by the first rule. The five-minute write-recency rule is what protects those, and it applies even when you set `EVE_TRACES_MAX_AGE_MS=0`. -| Variable | Default | Effect | -| ---------------------------- | -------------------- | --------------------------------------------- | -| `EVE_TRACES` | on | `off` stops writing traces and stops sweeping | -| `EVE_TRACES_MAX_AGE_MS` | `604800000` (7d) | Age after which a trace may be evicted | -| `EVE_TRACES_MAX_TOTAL_BYTES` | `536870912` (512 MB) | Size budget for the whole store | -| `EVE_TRACES_RETAIN_COUNT` | `20` | Newest traces kept regardless of age or size | +| Variable | Default | Effect | +| ---------------------------- | -------------------- | ------------------------------------------------------------------------------------------- | +| `EVE_TRACES` | on | `off` stops writing traces and stops sweeping | +| `EVE_TRACES_CONTENT` | on | `off` stops capturing model prompt/response and tool input/output attributes on local spans | +| `EVE_TRACES_MAX_AGE_MS` | `604800000` (7d) | Age after which a trace may be evicted | +| `EVE_TRACES_MAX_TOTAL_BYTES` | `536870912` (512 MB) | Size budget for the whole store | +| `EVE_TRACES_RETAIN_COUNT` | `20` | Newest traces kept regardless of age or size | Set these in `.env.local`, which `eve dev` loads automatically. Each bound accepts `off` to disable it individually — note that `EVE_TRACES_RETAIN_COUNT=off` removes the keep-newest guarantee rather than retaining everything. diff --git a/packages/eve/src/harness/agent-otel-content.ts b/packages/eve/src/harness/agent-otel-content.ts new file mode 100644 index 000000000..17cc53458 --- /dev/null +++ b/packages/eve/src/harness/agent-otel-content.ts @@ -0,0 +1,121 @@ +/** + * Serializes model and tool payloads into span content attributes for the + * local trace viewer: prompt messages, the system prompt, responses, and + * tool arguments/results. Everything is capped so a giant payload cannot + * bloat a span segment past the reader's per-file limit, and message + * payloads stay valid JSON through every truncation path. + */ + +/** Content attributes are capped so a giant payload cannot bloat a span segment. */ +export const CONTENT_ATTRIBUTE_LIMIT = 32 * 1024; + +/** Provider transport noise stripped from messages (not tool data). */ +const CONTENT_NOISE_KEYS = new Set(["providerOptions", "providerMetadata"]); + +/** Marker prepended when oldest messages are dropped to fit the content cap. */ +const TRUNCATED_MESSAGES_KEY = "eve.truncated"; + +/** + * Serializes one payload value. `strip` removes {@link CONTENT_NOISE_KEYS}; + * pass `false` for tool data, where those keys may be legitimate domain + * fields the user authored or the tool returned. + */ +export function contentAttribute(value: unknown, strip = true): string | undefined { + if (value === undefined) return undefined; + const prepared = strip ? stripContentNoise(value, 0) : value; + let json: string | undefined; + try { + json = JSON.stringify(prepared); + } catch { + return undefined; + } + if (json === undefined) return undefined; + return textContentAttribute(json); +} + +/** + * Serializes the prompt messages, keeping the result parseable: oldest + * messages drop first behind an omission marker, and a single message over + * the cap is truncated at the text level rather than the JSON level. + */ +export function messagesContentAttribute(messages: unknown): string | undefined { + if (!Array.isArray(messages)) return contentAttribute(messages); + const stripped = stripContentNoise(messages, 0) as unknown[]; + const full = stringifyContent(stripped); + if (full !== undefined && full.length <= CONTENT_ATTRIBUTE_LIMIT) return full; + for (let omitted = 1; omitted < stripped.length; omitted += 1) { + const json = stringifyContent([ + { [TRUNCATED_MESSAGES_KEY]: { omittedMessages: omitted } }, + ...stripped.slice(omitted), + ]); + if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json; + } + return truncateSingleMessage(stripped); +} + +/** Normalizes the AI SDK's `instructions` prompt to plain text for `ai.prompt.system`. */ +export function systemPromptAttribute(instructions: unknown): string | undefined { + if (typeof instructions === "string") return textContentAttribute(instructions); + if (!isRecord(instructions) && !Array.isArray(instructions)) return undefined; + const messages = Array.isArray(instructions) ? instructions : [instructions]; + const texts: string[] = []; + for (const message of messages) { + if (!isRecord(message)) continue; + if (typeof message.content === "string") texts.push(message.content); + else if (Array.isArray(message.content)) + for (const part of message.content) + if (isRecord(part) && part.type === "text" && typeof part.text === "string") + texts.push(part.text); + } + const joined = texts.join("\n\n").trim(); + return joined.length === 0 ? undefined : textContentAttribute(joined); +} + +/** Caps plain text, marking the cut. */ +export function textContentAttribute(text: string): string | undefined { + if (text.length === 0) return undefined; + return text.length <= CONTENT_ATTRIBUTE_LIMIT + ? text + : `${text.slice(0, CONTENT_ATTRIBUTE_LIMIT)}… [truncated]`; +} + +function stripContentNoise(value: unknown, depth: number): unknown { + if (depth > 32 || value === null || typeof value !== "object") return value; + if (Array.isArray(value)) return value.map((e) => stripContentNoise(e, depth + 1)); + const out: Record = {}; + for (const [k, v] of Object.entries(value)) + if (!CONTENT_NOISE_KEYS.has(k)) out[k] = stripContentNoise(v, depth + 1); + return out; +} + +function truncateSingleMessage(messages: unknown[]): string | undefined { + if (messages.length === 0) return undefined; + const last = messages[messages.length - 1]; + if (!isRecord(last)) return undefined; + const marker = { [TRUNCATED_MESSAGES_KEY]: { omittedMessages: messages.length - 1 } }; + const role = typeof last.role === "string" ? last.role : "user"; + let text = ""; + if (typeof last.content === "string") text = last.content; + else if (Array.isArray(last.content)) + for (const part of last.content) + if (isRecord(part) && part.type === "text" && typeof part.text === "string") + text += (text ? "\n" : "") + part.text; + for (let len = Math.min(text.length, CONTENT_ATTRIBUTE_LIMIT); len > 0; len -= 256) { + const cut = len >= text.length ? text : `${text.slice(0, len)}… [truncated]`; + const json = stringifyContent([marker, { role, content: cut }]); + if (json !== undefined && json.length <= CONTENT_ATTRIBUTE_LIMIT) return json; + } + return stringifyContent([marker, { role, content: "" }]); +} + +function stringifyContent(value: unknown): string | undefined { + try { + return JSON.stringify(value); + } catch { + return undefined; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/eve/src/harness/agent-otel-provider.test.ts b/packages/eve/src/harness/agent-otel-provider.test.ts index d3e3689a2..be93e5b95 100644 --- a/packages/eve/src/harness/agent-otel-provider.test.ts +++ b/packages/eve/src/harness/agent-otel-provider.test.ts @@ -79,13 +79,28 @@ async function emitAttempt(input: { ]); await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]); await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [ - { callId: "call-1", messages: [], modelId: "claude-test", provider: "anthropic" }, + { + callId: "call-1", + instructions: "You are a weather assistant (system prompt).", + messages: [ + { + content: "real user text", + providerOptions: { anthropic: { signature: "sig-blob" } }, + role: "user", + }, + ], + modelId: "claude-test", + provider: "anthropic", + }, ]); await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined }); await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [ { callId: "call-1", - content: [], + content: [ + { type: "reasoning", text: "thinking about weather" }, + { type: "text", text: "Checking the weather." }, + ], finishReason: "tool-calls", performance: { responseTimeMs: 10 }, responseId: "response-1", @@ -231,7 +246,134 @@ describe("createAgentOtelInstrumentation", () => { "agent.framework.name": "eve", "agent.root.session.id": "session-1", }); + }); + + it("captures model and tool inputs/outputs on the operation spans", async () => { + const runtime = createRuntime(); + await emitAttempt({ + hooks: runtime.hooks, + runInContext: runtime.runInContext, + sessionId: "session-1", + turnId: "turn-1", + turnSequence: 0, + }); + await runtime.provider.forceFlush(); + + const spans = runtime.exporter.getFinishedSpans(); + const model = byName(spans, "ai.streamText.doStream")[0]!; + const tool = byName(spans, "ai.toolCall")[0]!; + // Provider transport noise (signatures et al.) is stripped at capture time. + expect(model.attributes["ai.prompt.messages"]).toBe( + '[{"content":"real user text","role":"user"}]', + ); + expect(model.attributes["ai.prompt.system"]).toBe( + "You are a weather assistant (system prompt).", + ); + expect(model.attributes["ai.response.finish_reason"]).toBe("tool-calls"); + expect(model.attributes["ai.response.reasoning"]).toBe("thinking about weather"); + expect(model.attributes["ai.response.text"]).toBe("Checking the weather."); + expect(tool.attributes["gen_ai.tool.call.arguments"]).toBe('{"secret":"value"}'); + expect(tool.attributes["gen_ai.tool.call.result"]).toBe('{"temperature":72}'); + // Structural spans stay structural: content lives only on the operation spans. + const structural = byName(spans, "agent.action")[0]!; + expect(JSON.stringify(structural.attributes)).not.toContain("secret"); + expect(JSON.stringify(structural.attributes)).not.toContain("temperature"); + }); + + it("truncates long conversations from the front, keeping valid JSON and recent messages", async () => { + const runtime = createRuntime(); + const manyMessages = Array.from({ length: 200 }, (_, index) => ({ + content: `message ${index} ${"x".repeat(200)}`, + role: index % 2 === 0 ? "user" : "assistant", + })); + const scope: InstrumentationAttemptScope = { + attemptId: "session-1:turn-1:0:0", + attemptIndex: 0, + functionId: "weather", + sessionId: "session-1", + stepIndex: 0, + turnId: "turn-1", + }; + await runtime.hooks.publish({ + agentName: "weather", + channelKind: "http", + rootSessionId: "session-1", + sessionId: "session-1", + type: "session.started", + }); + await runtime.hooks.publish({ + rootSessionId: "session-1", + sequence: 0, + sessionId: "session-1", + turnId: "turn-1", + type: "turn.started", + }); + const bridge = createAiSdkHookBridge(scope, runtime.hooks, runtime.runInContext); + Reflect.apply(bridge.onStart!, bridge, [ + { + callId: "call-1", + messages: manyMessages, + modelId: "claude-test", + operationId: "ai.streamText", + provider: "anthropic", + }, + ]); + await Reflect.apply(bridge.onStepStart!, bridge, [{ callId: "call-1", stepNumber: 0 }]); + await Reflect.apply(bridge.onLanguageModelCallStart!, bridge, [ + { callId: "call-1", messages: manyMessages, modelId: "claude-test", provider: "anthropic" }, + ]); + await bridge.executeLanguageModelCall!({ callId: "call-1", execute: async () => undefined }); + await Reflect.apply(bridge.onLanguageModelCallEnd!, bridge, [ + { + callId: "call-1", + content: [], + finishReason: "stop", + performance: { responseTimeMs: 10 }, + responseId: "response-1", + usage: { inputTokens: 10, outputTokens: 5 }, + }, + ]); + await runtime.provider.forceFlush(); + + const model = byName(runtime.exporter.getFinishedSpans(), "ai.streamText.doStream")[0]!; + const raw = model.attributes["ai.prompt.messages"]; + expect(typeof raw).toBe("string"); + expect((raw as string).length).toBeLessThanOrEqual(32 * 1024); + const parsed = JSON.parse(raw as string) as Array>; + const marker = parsed[0]!["eve.truncated"] as { omittedMessages: number }; + expect(marker.omittedMessages).toBeGreaterThan(0); + expect(marker.omittedMessages).toBeLessThan(200); + expect(JSON.stringify(parsed)).toContain("message 199"); + expect(JSON.stringify(parsed)).not.toContain("message 0 "); + }); + + it("captures nothing when content capture is off", async () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + const agentOtel = createAgentOtelInstrumentation({ + captureContent: false, + frameworkVersion: "test", + stateStore: new InMemoryAgentTraceStateStore(), + tracer: provider.getTracer("eve.agent"), + }); + const hooks = createInstrumentationHooks([agentOtel.hook]); + await emitAttempt({ + hooks, + runInContext: agentOtel.runInContext, + sessionId: "session-1", + turnId: "turn-1", + turnSequence: 0, + }); + await provider.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("secret"); + expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("temperature"); expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("private"); + expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("real user text"); + expect(JSON.stringify(spans.map((span) => span.attributes))).not.toContain("system prompt"); }); it("writes gateway cost attributes on the step span when the gateway reports them", async () => { diff --git a/packages/eve/src/harness/agent-otel-provider.ts b/packages/eve/src/harness/agent-otel-provider.ts index eb2058887..e758dac86 100644 --- a/packages/eve/src/harness/agent-otel-provider.ts +++ b/packages/eve/src/harness/agent-otel-provider.ts @@ -9,6 +9,12 @@ import { trace, } from "#compiled/@opentelemetry/api/index.js"; +import { + contentAttribute, + messagesContentAttribute, + systemPromptAttribute, + textContentAttribute, +} from "#harness/agent-otel-content.js"; import type { InstrumentationAttemptMetadataEvent, InstrumentationAttemptScope, @@ -110,6 +116,12 @@ export class InMemoryAgentTraceStateStore implements AgentTraceStateStore { } export interface AgentOtelInstrumentationInput { + /** + * Capture model prompts/responses and tool call inputs/outputs as span + * attributes. Content stays on the local machine — this provider only + * wires the dev-time local spool — but can be turned off per project. + */ + readonly captureContent?: boolean; readonly frameworkVersion: string; readonly stateStore: AgentTraceStateStore; readonly tracer: Tracer; @@ -125,6 +137,7 @@ export interface AgentOtelInstrumentation { export function createAgentOtelInstrumentation( input: AgentOtelInstrumentationInput, ): AgentOtelInstrumentation { + const captureContent = input.captureContent ?? true; const executionContexts = new WeakMap< InstrumentationAttemptScope, { readonly models: Map; readonly tools: Map } @@ -302,6 +315,12 @@ export function createAgentOtelInstrumentation( }, attempt.operation.context, ); + if (captureContent) { + const messages = messagesContentAttribute(event.source.messages); + if (messages !== undefined) span.setAttribute("ai.prompt.messages", messages); + const system = systemPromptAttribute(event.source.instructions); + if (system !== undefined) span.setAttribute("ai.prompt.system", system); + } const state = { context: trace.setSpan(attempt.operation.context, span), span }; getExecutionContexts(event.scope).models.set(event.id, state.context); return state; @@ -316,6 +335,31 @@ export function createAgentOtelInstrumentation( setUsage(state.span, event.source.usage); const attempt = steps.get(event.scope); if (attempt !== undefined) setUsage(attempt.step.span, event.source.usage); + if (captureContent) { + state.span.setAttribute("ai.response.finish_reason", event.source.finishReason); + const reasoning = textContentAttribute( + event.source.content + .filter((part) => part.type === "reasoning") + .map((part) => part.text) + .filter((part) => part.trim().length > 0) + .join("\n"), + ); + if (reasoning !== undefined) state.span.setAttribute("ai.response.reasoning", reasoning); + const text = textContentAttribute( + event.source.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join(""), + ); + if (text !== undefined) state.span.setAttribute("ai.response.text", text); + const toolCalls = event.source.content + .filter((part) => part.type === "tool-call") + .map((part) => ({ input: part.input, toolName: part.toolName })); + if (toolCalls.length > 0) { + const json = contentAttribute(toolCalls, false); + if (json !== undefined) state.span.setAttribute("ai.response.tool_calls", json); + } + } } state.span.end(); }; @@ -355,6 +399,10 @@ export function createAgentOtelInstrumentation( }, actionContext, ); + if (captureContent) { + const args = contentAttribute(event.source.toolCall.input, false); + if (args !== undefined) toolSpan.setAttribute("gen_ai.tool.call.arguments", args); + } const state: ToolSpanState = { context: trace.setSpan(actionContext, toolSpan), span: actionSpan, @@ -373,6 +421,9 @@ export function createAgentOtelInstrumentation( } else if (event.source.toolOutput.type !== "tool-result") { recordError(state.toolSpan, event.source.toolOutput.error); recordError(state.span, event.source.toolOutput.error); + } else if (captureContent) { + const result = contentAttribute(event.source.toolOutput.output, false); + if (result !== undefined) state.toolSpan.setAttribute("gen_ai.tool.call.result", result); } state.toolSpan.end(); state.span.end(); @@ -625,7 +676,5 @@ function recordError(span: Span, error: unknown): void { if (error instanceof Error) { span.recordException(error); span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); - } else { - span.setStatus({ code: SpanStatusCode.ERROR }); - } + } else span.setStatus({ code: SpanStatusCode.ERROR }); } diff --git a/packages/eve/src/harness/local-instrumentation-runtime.ts b/packages/eve/src/harness/local-instrumentation-runtime.ts index 63d889e26..ce59ad1db 100644 --- a/packages/eve/src/harness/local-instrumentation-runtime.ts +++ b/packages/eve/src/harness/local-instrumentation-runtime.ts @@ -49,6 +49,7 @@ export function installLocalInstrumentationRuntime(input: { throw new Error("eve could not register OpenTelemetry because another runtime already exists."); } const agentOtel = createAgentOtelInstrumentation({ + captureContent: process.env.EVE_TRACES_CONTENT !== "off", frameworkVersion: input.frameworkVersion, stateStore: new ContextAgentTraceStateStore(), tracer: trace.getTracer("eve.agent", input.frameworkVersion),