diff --git a/apps/cli/src/commands/results/manifest.ts b/apps/cli/src/commands/results/manifest.ts index 253e49127..fdef7301f 100644 --- a/apps/cli/src/commands/results/manifest.ts +++ b/apps/cli/src/commands/results/manifest.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { type EvaluationResult, + type ExternalTraceMetadataWire, type ResultArtifactPointersWire, type TraceSummary, type TranscriptJsonLine, @@ -46,6 +47,7 @@ export interface ResultManifestRecord { readonly transcript_path?: string; readonly raw_provider_log_path?: string; readonly artifact_pointers?: ResultArtifactPointersWire; + readonly external_trace?: ExternalTraceMetadataWire; readonly transcript?: ArtifactPointer; readonly artifacts?: ArtifactPointerMap; readonly response_path?: string; diff --git a/apps/cli/src/commands/results/phoenix-readthrough.ts b/apps/cli/src/commands/results/phoenix-readthrough.ts new file mode 100644 index 000000000..6538d30dc --- /dev/null +++ b/apps/cli/src/commands/results/phoenix-readthrough.ts @@ -0,0 +1,1555 @@ +import type { ExternalTraceMetadata, ExternalTraceMetadataWire } from '@agentv/core'; + +const RESPONSE_SCHEMA_VERSION = 'agentv.dashboard.phoenix_session.v1'; +const DEFAULT_PAGE_LIMIT = 1000; +const REDACTED_VALUE = '[redacted]'; + +export type PhoenixReadStatus = + | 'ok' + | 'missing_external_trace' + | 'not_configured' + | 'unresolved' + | 'unreachable' + | 'schema_mismatch'; + +export interface PhoenixTokenUsage { + readonly input?: number; + readonly output?: number; + readonly reasoning?: number; + readonly cached?: number; + readonly total?: number; +} + +export interface PhoenixAnnotation { + readonly id?: string; + readonly name?: string; + readonly annotator_kind?: string; + readonly label?: string; + readonly score?: number; + readonly explanation?: string; + readonly target?: 'session' | 'trace' | 'span'; + readonly target_id?: string; + readonly result?: unknown; +} + +export interface PhoenixSpanDetail { + readonly span_id: string; + readonly trace_id?: string; + readonly parent_span_id?: string; + readonly name?: string; + readonly span_kind?: string; + readonly status?: string; + readonly start_time?: string; + readonly end_time?: string; + readonly duration_ms?: number; + readonly input?: unknown; + readonly output?: unknown; + readonly token_usage?: PhoenixTokenUsage; + readonly cost_usd?: number; + readonly attributes?: Record; + readonly annotations?: readonly PhoenixAnnotation[]; +} + +export interface PhoenixTraceTreeNode extends PhoenixSpanDetail { + readonly depth: number; + readonly child_count: number; +} + +export interface PhoenixSessionTurn { + readonly index: number; + readonly trace_id?: string; + readonly start_time?: string; + readonly end_time?: string; + readonly duration_ms?: number; + readonly status?: string; + readonly root_span_id?: string; + readonly input?: unknown; + readonly output?: unknown; + readonly token_usage?: PhoenixTokenUsage; + readonly cost_usd?: number; + readonly annotations?: readonly PhoenixAnnotation[]; +} + +export interface PhoenixSessionSummary { + readonly id?: string; + readonly session_id?: string; + readonly project_id?: string; + readonly project?: string; + readonly start_time?: string; + readonly end_time?: string; + readonly duration_ms?: number; + readonly trace_count: number; + readonly token_usage?: PhoenixTokenUsage; + readonly cost_usd?: number; + readonly annotations?: readonly PhoenixAnnotation[]; +} + +export interface PhoenixLinkedSessionResponse { + readonly schema_version: typeof RESPONSE_SCHEMA_VERSION; + readonly status: PhoenixReadStatus; + readonly message?: string; + readonly external_trace?: ExternalTraceMetadataWire; + readonly open_in_phoenix_url?: string; + readonly session?: PhoenixSessionSummary; + readonly turns?: readonly PhoenixSessionTurn[]; + readonly spans?: readonly PhoenixSpanDetail[]; + readonly trace_tree?: readonly PhoenixTraceTreeNode[]; + readonly annotations?: readonly PhoenixAnnotation[]; +} + +interface PhoenixConfig { + readonly baseUrl: string; + readonly graphqlUrl: string; + readonly project?: string; + readonly headers: Record; +} + +interface ReadPhoenixLinkedSessionOptions { + readonly env?: Readonly>; + readonly fetch?: typeof fetch; +} + +interface PhoenixRestSession { + readonly id?: string; + readonly sessionId?: string; + readonly projectId?: string; + readonly startTime?: string; + readonly endTime?: string; + readonly traces: readonly PhoenixRestTrace[]; + readonly raw?: Record; +} + +interface PhoenixRestTrace { + readonly traceId?: string; + readonly startTime?: string; + readonly endTime?: string; + readonly raw?: Record; +} + +class PhoenixReadError extends Error { + constructor( + readonly code: Exclude, + message: string, + ) { + super(message); + this.name = 'PhoenixReadError'; + } +} + +const PHOENIX_LINKED_SESSION_GRAPHQL_FIELDS = ` + fragment AgentVPhoenixSpanFields on Span { + id + spanId + parentId + name + spanKind + statusCode: propagatedStatusCode + startTime + endTime + latencyMs + input { + value + truncatedValue + mimeType + } + output { + value + truncatedValue + mimeType + } + attributes + tokenCountTotal + tokenCountPrompt + tokenCountCompletion + cumulativeTokenCountTotal + cumulativeTokenCountPrompt + cumulativeTokenCountCompletion + costSummary { + prompt { + cost + tokens + } + completion { + cost + tokens + } + total { + cost + tokens + } + } + spanAnnotations { + id + name + annotatorKind + label + score + explanation + metadata + identifier + spanId + createdAt + updatedAt + } + trace { + id + traceId + costSummary { + total { + cost + tokens + } + } + } + project { + id + name + } + } + + fragment AgentVPhoenixTraceFields on Trace { + id + traceId + startTime + endTime + latencyMs + projectId + project { + id + name + } + projectSessionId + costSummary { + total { + cost + tokens + } + } + traceAnnotations { + id + name + annotatorKind + label + score + explanation + metadata + identifier + createdAt + updatedAt + } + rootSpan { + ...AgentVPhoenixSpanFields + } + spans(first: $spanFirst) { + edges { + node { + ...AgentVPhoenixSpanFields + } + } + } + } + + fragment AgentVPhoenixSessionFields on ProjectSession { + id + sessionId + startTime + endTime + project { + id + name + } + numTraces + tokenUsage { + prompt + completion + total + } + costSummary { + prompt { + cost + tokens + } + completion { + cost + tokens + } + total { + cost + tokens + } + } + sessionAnnotations { + id + name + annotatorKind + label + score + explanation + metadata + identifier + projectSessionId + createdAt + updatedAt + } + traces(first: $traceFirst) { + edges { + node { + ...AgentVPhoenixTraceFields + } + } + } + } +`; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asArray(value: unknown): unknown[] { + return Array.isArray(value) ? value : []; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function isCredentialLikeKey(key: string): boolean { + const normalized = key.toLowerCase(); + if (normalized === 'traceparent' || normalized === 'tracestate') { + return false; + } + return /(^|[._-])(api[._-]?key|authorization|bearer|password|secret|private[._-]?key|access[._-]?token|auth[._-]?token|client[._-]?secret|cookie|id[._-]?token|refresh[._-]?token|session[._-]?token|token)($|[._-])/.test( + normalized, + ); +} + +function scrubCredentialFields(value: unknown, depth = 0): unknown { + if (depth > 20) { + return value; + } + if (Array.isArray(value)) { + return value.map((entry) => scrubCredentialFields(entry, depth + 1)); + } + if (!isRecord(value)) { + return value; + } + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [ + key, + isCredentialLikeKey(key) ? REDACTED_VALUE : scrubCredentialFields(entry, depth + 1), + ]), + ); +} + +function scrubCredentialRecord( + value: Record | undefined, +): Record | undefined { + const scrubbed = scrubCredentialFields(value); + return isRecord(scrubbed) ? scrubbed : undefined; +} + +function finiteNumber(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function firstNumber(...values: unknown[]): number | undefined { + for (const value of values) { + const numberValue = finiteNumber(value); + if (numberValue !== undefined) { + return numberValue; + } + } + return undefined; +} + +function compactRecord>(value: T): T | undefined { + const entries = Object.entries(value).filter(([, entry]) => entry !== undefined); + return entries.length > 0 ? (Object.fromEntries(entries) as T) : undefined; +} + +function durationMs( + startTime: string | undefined, + endTime: string | undefined, +): number | undefined { + if (!startTime || !endTime) { + return undefined; + } + const start = Date.parse(startTime); + const end = Date.parse(endTime); + if (Number.isNaN(start) || Number.isNaN(end)) { + return undefined; + } + return Math.max(0, end - start); +} + +function normalizeBaseUrl(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + try { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { + return undefined; + } + url.search = ''; + url.hash = ''; + url.pathname = url.pathname + .replace(/\/graphql\/?$/i, '') + .replace(/\/v1\/traces\/?$/i, '') + .replace(/\/v1\/?$/i, '') + .replace(/\/+$/g, ''); + if (!url.pathname) { + url.pathname = '/'; + } + return url.toString().replace(/\/+$/, ''); + } catch { + return undefined; + } +} + +function normalizeGraphqlUrl(value: string | undefined): string | undefined { + const baseUrl = normalizeBaseUrl(value); + return baseUrl ? `${baseUrl}/graphql` : undefined; +} + +function envString( + env: Readonly>, + keys: readonly string[], +): string | undefined { + for (const key of keys) { + const value = stringValue(env[key]); + if (value) { + return value; + } + } + return undefined; +} + +function parseHeaderEntries(value: string | undefined): Record { + const raw = stringValue(value); + if (!raw) { + return {}; + } + + try { + const parsed = JSON.parse(raw) as unknown; + if (isRecord(parsed)) { + return Object.fromEntries( + Object.entries(parsed).flatMap(([name, headerValue]) => { + const normalizedName = stringValue(name); + const normalizedValue = stringValue(headerValue); + return normalizedName && normalizedValue ? [[normalizedName, normalizedValue]] : []; + }), + ); + } + } catch { + // Fall through to the PHOENIX_CLIENT_HEADERS comma-separated form. + } + + const headers: Record = {}; + for (const segment of raw.split(',')) { + const separator = segment.indexOf('='); + if (separator <= 0) { + continue; + } + const name = decodeURIComponent(segment.slice(0, separator).trim()); + const headerValue = decodeURIComponent(segment.slice(separator + 1).trim()); + if (name && headerValue) { + headers[name] = headerValue; + } + } + return headers; +} + +function hasHeader(headers: Record, name: string): boolean { + const normalized = name.toLowerCase(); + return Object.keys(headers).some((header) => header.toLowerCase() === normalized); +} + +function resolvePhoenixConfig( + externalTrace: ExternalTraceMetadata, + env: Readonly>, +): PhoenixConfig | undefined { + const baseUrl = normalizeBaseUrl( + envString(env, [ + 'AGENTV_PHOENIX_ENDPOINT', + 'AGENTV_PHOENIX_BASE_URL', + 'AGENTV_PHOENIX_HOST', + 'PHOENIX_HOST', + 'PHOENIX_BASE_URL', + 'PHOENIX_ENDPOINT', + 'PHOENIX_COLLECTOR_ENDPOINT', + 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', + 'OTEL_EXPORTER_OTLP_ENDPOINT', + ]), + ); + const graphqlUrl = + normalizeGraphqlUrl(envString(env, ['AGENTV_PHOENIX_GRAPHQL_URL', 'PHOENIX_GRAPHQL_URL'])) ?? + normalizeGraphqlUrl(baseUrl); + + if (!baseUrl || !graphqlUrl) { + return undefined; + } + + const headers = { + ...parseHeaderEntries(envString(env, ['AGENTV_PHOENIX_HEADERS', 'PHOENIX_CLIENT_HEADERS'])), + }; + const authorization = envString(env, ['AGENTV_PHOENIX_AUTHORIZATION', 'PHOENIX_AUTHORIZATION']); + const apiKey = envString(env, ['AGENTV_PHOENIX_API_KEY', 'PHOENIX_API_KEY']); + const cookie = envString(env, ['AGENTV_PHOENIX_COOKIE', 'PHOENIX_COOKIE']); + if (authorization && !hasHeader(headers, 'authorization')) { + headers.Authorization = authorization; + } + if (apiKey && !hasHeader(headers, 'authorization')) { + headers.Authorization = `Bearer ${apiKey}`; + } + if (cookie && !hasHeader(headers, 'cookie')) { + headers.Cookie = cookie; + } + + return { + baseUrl, + graphqlUrl, + project: + externalTrace.projectId ?? + externalTrace.project ?? + envString(env, [ + 'AGENTV_PHOENIX_PROJECT_ID', + 'AGENTV_PHOENIX_PROJECT', + 'AGENTV_PHOENIX_PROJECT_NAME', + 'PHOENIX_PROJECT', + 'PHOENIX_PROJECT_ID', + 'PHOENIX_PROJECT_NAME', + 'OPENINFERENCE_PROJECT_NAME', + ]), + headers, + }; +} + +function response( + status: PhoenixReadStatus, + message: string, + externalTrace: ExternalTraceMetadataWire | undefined, + details: Omit = {}, +): PhoenixLinkedSessionResponse { + return { + schema_version: RESPONSE_SCHEMA_VERSION, + status, + message, + ...(externalTrace && { external_trace: externalTrace }), + ...details, + }; +} + +function encodePathSegment(value: string): string { + return encodeURIComponent(value).replace(/%2F/g, '%252F'); +} + +function appendQuery(url: URL, key: string, value: string | readonly string[] | undefined): void { + if (value === undefined) { + return; + } + if (Array.isArray(value)) { + for (const entry of value) { + url.searchParams.append(key, entry); + } + return; + } + if (typeof value === 'string') { + url.searchParams.set(key, value); + } +} + +async function fetchJson( + requestUrl: string, + init: RequestInit, + fetchImpl: typeof fetch, +): Promise { + let responseValue: Response; + try { + responseValue = await fetchImpl(requestUrl, init); + } catch (error) { + throw new PhoenixReadError( + 'unreachable', + `Phoenix request failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + if (responseValue.status === 404) { + throw new PhoenixReadError('unresolved', 'No matching Phoenix session or trace was found.'); + } + if (!responseValue.ok) { + throw new PhoenixReadError( + responseValue.status >= 500 ? 'unreachable' : 'schema_mismatch', + `Phoenix returned HTTP ${responseValue.status}.`, + ); + } + + try { + return await responseValue.json(); + } catch { + throw new PhoenixReadError('schema_mismatch', 'Phoenix returned a non-JSON response.'); + } +} + +async function phoenixRestGet( + config: PhoenixConfig, + pathName: string, + query: Record, + fetchImpl: typeof fetch, +): Promise { + const url = new URL( + pathName, + config.baseUrl.endsWith('/') ? config.baseUrl : `${config.baseUrl}/`, + ); + for (const [key, value] of Object.entries(query)) { + appendQuery(url, key, value); + } + return fetchJson( + url.toString(), + { + method: 'GET', + headers: { + Accept: 'application/json', + ...config.headers, + }, + }, + fetchImpl, + ); +} + +async function phoenixGraphql( + config: PhoenixConfig, + body: { query: string; variables: Record }, + fetchImpl: typeof fetch, +): Promise { + const payload = await fetchJson( + config.graphqlUrl, + { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + ...config.headers, + }, + body: JSON.stringify(body), + }, + fetchImpl, + ); + const record = isRecord(payload) ? payload : {}; + if (Array.isArray(record.errors) && record.errors.length > 0) { + throw new PhoenixReadError('schema_mismatch', 'Phoenix GraphQL returned schema errors.'); + } + return record.data; +} + +function normalizeTrace(raw: unknown): PhoenixRestTrace | undefined { + const record = isRecord(raw) ? raw : undefined; + if (!record) { + return undefined; + } + const traceId = stringValue(record.trace_id) ?? stringValue(record.traceId); + if (!traceId) { + return undefined; + } + return { + traceId, + startTime: stringValue(record.start_time) ?? stringValue(record.startTime), + endTime: stringValue(record.end_time) ?? stringValue(record.endTime), + raw: record, + }; +} + +function normalizeSession(raw: unknown): PhoenixRestSession | undefined { + const record = isRecord(raw) ? raw : undefined; + if (!record) { + return undefined; + } + const data = isRecord(record.data) ? record.data : record; + const traces = asArray(data.traces) + .map(normalizeTrace) + .filter((trace): trace is PhoenixRestTrace => trace !== undefined); + const sessionId = + stringValue(data.session_id) ?? stringValue(data.sessionId) ?? stringValue(data.id); + if (!sessionId && traces.length === 0) { + return undefined; + } + return { + id: stringValue(data.id), + sessionId, + projectId: stringValue(data.project_id) ?? stringValue(data.projectId), + startTime: stringValue(data.start_time) ?? stringValue(data.startTime), + endTime: stringValue(data.end_time) ?? stringValue(data.endTime), + traces, + raw: data, + }; +} + +function normalizeGraphqlSession(raw: unknown): PhoenixRestSession | undefined { + const node = sessionNodeFromGraphqlData(raw); + if (!node) { + return undefined; + } + + const traceNodes = connectionNodes(node.traces) + .map(normalizeTrace) + .filter((trace): trace is PhoenixRestTrace => trace !== undefined); + const sessionId = + stringValue(node.session_id) ?? stringValue(node.sessionId) ?? stringValue(node.id); + if (!sessionId && traceNodes.length === 0) { + return undefined; + } + + return { + id: stringValue(node.id), + sessionId, + projectId: projectIdFromGraphqlNode(node), + startTime: stringValue(node.start_time) ?? stringValue(node.startTime), + endTime: stringValue(node.end_time) ?? stringValue(node.endTime), + traces: traceNodes, + raw: node, + }; +} + +async function fetchSessionByNodeId( + config: PhoenixConfig, + sessionNodeId: string, + fetchImpl: typeof fetch, +): Promise { + const data = await phoenixGraphql( + config, + { + query: ` + query AgentVPhoenixLinkedSessionByNode($id: ID!, $traceFirst: Int!, $spanFirst: Int!) { + node(id: $id) { + __typename + ... on ProjectSession { + ...AgentVPhoenixSessionFields + } + } + } + ${PHOENIX_LINKED_SESSION_GRAPHQL_FIELDS} + `, + variables: { id: sessionNodeId, traceFirst: 100, spanFirst: DEFAULT_PAGE_LIMIT }, + }, + fetchImpl, + ); + return normalizeGraphqlSession(data); +} + +async function fetchSessionByIdGraphql( + config: PhoenixConfig, + projectIdentifier: string, + sessionId: string, + fetchImpl: typeof fetch, +): Promise { + const data = await phoenixGraphql( + config, + { + query: ` + query AgentVPhoenixLinkedSessionBySessionId( + $projectId: ID! + $sessionId: String! + $traceFirst: Int! + $spanFirst: Int! + ) { + project: node(id: $projectId) { + __typename + ... on Project { + sessions(first: 1, sessionId: $sessionId) { + edges { + node { + ...AgentVPhoenixSessionFields + } + } + } + } + } + } + ${PHOENIX_LINKED_SESSION_GRAPHQL_FIELDS} + `, + variables: { + projectId: projectIdentifier, + sessionId, + traceFirst: 100, + spanFirst: DEFAULT_PAGE_LIMIT, + }, + }, + fetchImpl, + ); + return normalizeGraphqlSession(data); +} + +async function fetchSessionByTraceIdGraphql( + config: PhoenixConfig, + projectIdentifier: string, + traceId: string, + fetchImpl: typeof fetch, +): Promise { + const data = await phoenixGraphql( + config, + { + query: ` + query AgentVPhoenixLinkedSessionByTraceId( + $projectId: ID! + $traceId: ID! + $traceFirst: Int! + $spanFirst: Int! + ) { + project: node(id: $projectId) { + __typename + ... on Project { + trace(traceId: $traceId) { + session { + ...AgentVPhoenixSessionFields + } + } + } + } + } + ${PHOENIX_LINKED_SESSION_GRAPHQL_FIELDS} + `, + variables: { + projectId: projectIdentifier, + traceId, + traceFirst: 100, + spanFirst: DEFAULT_PAGE_LIMIT, + }, + }, + fetchImpl, + ); + return normalizeGraphqlSession(data); +} + +async function fetchSessionByIdRest( + config: PhoenixConfig, + sessionId: string, + fetchImpl: typeof fetch, +): Promise { + return normalizeSession( + await phoenixRestGet(config, `v1/sessions/${encodePathSegment(sessionId)}`, {}, fetchImpl), + ); +} + +function extractRecords(payload: unknown): Record[] { + const record = isRecord(payload) ? payload : {}; + const data = isRecord(record.data) ? record.data.data : record.data; + return asArray(data) + .map((entry) => (isRecord(entry) ? entry : undefined)) + .filter((entry): entry is Record => entry !== undefined); +} + +function connectionNodes(value: unknown): Record[] { + if (Array.isArray(value)) { + return value.filter((entry): entry is Record => isRecord(entry)); + } + const record = isRecord(value) ? value : undefined; + return asArray(record?.edges) + .map((edge) => (isRecord(edge) && isRecord(edge.node) ? edge.node : edge)) + .filter((entry): entry is Record => isRecord(entry)); +} + +function projectIdFromGraphqlNode(node: Record): string | undefined { + const project = isRecord(node.project) ? node.project : undefined; + return stringValue(node.projectId) ?? stringValue(project?.id); +} + +function sessionNodeFromGraphqlData(raw: unknown): Record | undefined { + const data = isRecord(raw) ? raw : undefined; + if (!data) { + return undefined; + } + if (isRecord(data.node) && stringValue(data.node.sessionId)) { + return data.node; + } + if (isRecord(data.projectSession)) { + return data.projectSession; + } + const project = isRecord(data.project) ? data.project : undefined; + const sessionFromProject = connectionNodes(project?.sessions)[0]; + if (sessionFromProject) { + return sessionFromProject; + } + const trace = isRecord(project?.trace) ? project.trace : undefined; + return isRecord(trace?.session) ? trace.session : undefined; +} + +async function fetchTraceSpans( + config: PhoenixConfig, + projectIdentifier: string, + traceId: string, + fetchImpl: typeof fetch, +): Promise[]> { + return extractRecords( + await phoenixRestGet( + config, + `v1/projects/${encodePathSegment(projectIdentifier)}/spans`, + { limit: String(DEFAULT_PAGE_LIMIT), trace_id: traceId }, + fetchImpl, + ), + ); +} + +async function fetchAnnotations( + config: PhoenixConfig, + projectIdentifier: string, + kind: 'session' | 'trace' | 'span', + ids: readonly string[], + fetchImpl: typeof fetch, +): Promise { + if (ids.length === 0) { + return []; + } + const queryKey = kind === 'session' ? 'session_ids' : kind === 'trace' ? 'trace_ids' : 'span_ids'; + const pathName = `v1/projects/${encodePathSegment(projectIdentifier)}/${kind}_annotations`; + try { + return extractRecords( + await phoenixRestGet( + config, + pathName, + { [queryKey]: ids, limit: String(DEFAULT_PAGE_LIMIT) }, + fetchImpl, + ), + ).map((annotation) => normalizeAnnotation(annotation, kind)); + } catch (error) { + if (error instanceof PhoenixReadError && error.code === 'unresolved') { + return []; + } + throw error; + } +} + +function normalizeAnnotation( + annotation: Record, + target: 'session' | 'trace' | 'span', + targetId?: string, +): PhoenixAnnotation { + const result = isRecord(annotation.result) ? annotation.result : undefined; + return { + id: stringValue(annotation.id), + name: stringValue(annotation.name), + annotator_kind: stringValue(annotation.annotator_kind) ?? stringValue(annotation.annotatorKind), + label: stringValue(annotation.label) ?? stringValue(result?.label), + score: finiteNumber(annotation.score) ?? finiteNumber(result?.score), + explanation: stringValue(annotation.explanation) ?? stringValue(result?.explanation), + target, + target_id: + stringValue(annotation.session_id) ?? + stringValue(annotation.projectSessionId) ?? + stringValue(annotation.trace_id) ?? + stringValue(annotation.span_id) ?? + stringValue(annotation.spanId) ?? + targetId, + result: scrubCredentialFields(annotation.result ?? annotation.metadata), + }; +} + +function spanContext(span: Record): Record { + return isRecord(span.context) ? span.context : span; +} + +function spanAttributes(span: Record): Record | undefined { + if (isRecord(span.attributes)) { + return span.attributes; + } + const raw = stringValue(span.attributes); + if (!raw) { + return undefined; + } + try { + const parsed = JSON.parse(raw) as unknown; + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function attributeString( + attributes: Record | undefined, + keys: readonly string[], +): string | undefined { + if (!attributes) { + return undefined; + } + for (const key of keys) { + const value = stringValue(attributes[key]); + if (value) { + return value; + } + } + return undefined; +} + +function attributeNumber( + attributes: Record | undefined, + keys: readonly string[], +): number | undefined { + if (!attributes) { + return undefined; + } + for (const key of keys) { + const value = finiteNumber(attributes[key]); + if (value !== undefined) { + return value; + } + } + return undefined; +} + +function tokenUsageFromSpan( + span: Record, + attributes: Record | undefined, +): PhoenixTokenUsage | undefined { + const nested = isRecord(attributes?.token_usage) ? attributes.token_usage : {}; + return compactRecord({ + input: firstNumber( + nested.input, + nested.prompt, + span.input_tokens, + span.tokenCountPrompt, + span.cumulativeTokenCountPrompt, + attributeNumber(attributes, ['gen_ai.usage.input_tokens', 'llm.token_count.prompt']), + ), + output: firstNumber( + nested.output, + nested.completion, + span.output_tokens, + span.tokenCountCompletion, + span.cumulativeTokenCountCompletion, + attributeNumber(attributes, ['gen_ai.usage.output_tokens', 'llm.token_count.completion']), + ), + reasoning: firstNumber(nested.reasoning, attributes?.['gen_ai.usage.reasoning.output_tokens']), + cached: firstNumber(nested.cached, attributes?.['gen_ai.usage.cache_read.input_tokens']), + total: firstNumber( + nested.total, + span.total_tokens, + span.tokenCountTotal, + span.cumulativeTokenCountTotal, + attributes?.total_tokens, + ), + }) as PhoenixTokenUsage | undefined; +} + +function spanInput(span: Record, attributes: Record | undefined) { + const input = isRecord(span.input) ? span.input.value : span.input; + return ( + input ?? + attributes?.['input.value'] ?? + attributes?.input ?? + attributes?.['llm.input_messages'] ?? + attributes?.['gen_ai.prompt'] + ); +} + +function spanOutput( + span: Record, + attributes: Record | undefined, +) { + const output = isRecord(span.output) ? span.output.value : span.output; + return ( + output ?? + attributes?.['output.value'] ?? + attributes?.output ?? + attributes?.['llm.output_messages'] ?? + attributes?.['gen_ai.completion'] + ); +} + +function normalizeSpan( + span: Record, + annotationsBySpanId: Map, +): PhoenixSpanDetail | undefined { + const context = spanContext(span); + const attributes = spanAttributes(span); + const spanId = + stringValue(context.span_id) ?? + stringValue(span.span_id) ?? + stringValue(span.spanId) ?? + stringValue(span.id); + if (!spanId) { + return undefined; + } + const trace = isRecord(span.trace) ? span.trace : undefined; + const costSummary = isRecord(span.costSummary) ? span.costSummary : undefined; + const totalCostSummary = isRecord(costSummary?.total) ? costSummary.total : undefined; + const traceId = + stringValue(context.trace_id) ?? stringValue(span.trace_id) ?? stringValue(trace?.traceId); + const startTime = stringValue(span.start_time) ?? stringValue(span.startTime); + const endTime = stringValue(span.end_time) ?? stringValue(span.endTime); + const annotations = annotationsBySpanId.get(spanId); + + return compactRecord({ + span_id: spanId, + trace_id: traceId, + parent_span_id: + stringValue(span.parent_id) ?? stringValue(span.parent_span_id) ?? stringValue(span.parentId), + name: stringValue(span.name), + span_kind: + stringValue(span.span_kind) ?? + stringValue(span.spanKind) ?? + attributeString(attributes, ['openinference.span.kind', 'span.kind']), + status: + stringValue(span.status_code) ?? + stringValue(span.statusCode) ?? + stringValue(isRecord(span.status) ? span.status.code : undefined), + start_time: startTime, + end_time: endTime, + duration_ms: finiteNumber(span.latency_ms) ?? durationMs(startTime, endTime), + input: scrubCredentialFields(spanInput(span, attributes)), + output: scrubCredentialFields(spanOutput(span, attributes)), + token_usage: tokenUsageFromSpan(span, attributes), + cost_usd: firstNumber( + span.cost_usd, + span.cost, + totalCostSummary?.cost, + attributes?.cost, + attributes?.['llm.cost.usd'], + ), + attributes: scrubCredentialRecord(attributes), + ...(annotations && annotations.length > 0 ? { annotations } : {}), + }) as PhoenixSpanDetail | undefined; +} + +function buildSpanTree(spans: readonly PhoenixSpanDetail[]): PhoenixTraceTreeNode[] { + const childrenByParent = new Map(); + for (const span of spans) { + const parentId = span.parent_span_id; + const children = childrenByParent.get(parentId) ?? []; + children.push(span); + childrenByParent.set(parentId, children); + } + + const visited = new Set(); + const nodes: PhoenixTraceTreeNode[] = []; + const roots = spans.filter( + (span) => !span.parent_span_id || !spans.some((s) => s.span_id === span.parent_span_id), + ); + + function visit(span: PhoenixSpanDetail, depth: number): void { + if (visited.has(span.span_id)) { + return; + } + visited.add(span.span_id); + const children = childrenByParent.get(span.span_id) ?? []; + nodes.push({ ...span, depth, child_count: children.length }); + for (const child of children) { + visit(child, depth + 1); + } + } + + for (const root of roots) { + visit(root, 0); + } + for (const span of spans) { + visit(span, 0); + } + return nodes; +} + +function sumTokenUsage( + items: readonly { token_usage?: PhoenixTokenUsage }[], +): PhoenixTokenUsage | undefined { + const total: { + input?: number; + output?: number; + reasoning?: number; + cached?: number; + total?: number; + } = {}; + function add(key: keyof typeof total, value: number | undefined): void { + if (value === undefined) { + return; + } + total[key] = (total[key] ?? 0) + value; + } + for (const item of items) { + const usage = item.token_usage; + if (!usage) { + continue; + } + add('input', usage.input); + add('output', usage.output); + add('reasoning', usage.reasoning); + add('cached', usage.cached); + add('total', usage.total); + } + return compactRecord(total) as PhoenixTokenUsage | undefined; +} + +function tokenUsageFromGraphql(value: unknown): PhoenixTokenUsage | undefined { + const record = isRecord(value) ? value : undefined; + if (!record) { + return undefined; + } + return compactRecord({ + input: firstNumber(record.input, record.prompt), + output: firstNumber(record.output, record.completion), + reasoning: finiteNumber(record.reasoning), + cached: finiteNumber(record.cached), + total: finiteNumber(record.total), + }) as PhoenixTokenUsage | undefined; +} + +function costFromGraphqlSummary(value: unknown): number | undefined { + const record = isRecord(value) ? value : undefined; + const total = isRecord(record?.total) ? record.total : undefined; + return finiteNumber(total?.cost); +} + +function annotationsByTarget( + annotations: readonly PhoenixAnnotation[], + target: 'trace' | 'span', +): Map { + const byTarget = new Map(); + for (const annotation of annotations) { + if (annotation.target !== target || !annotation.target_id) { + continue; + } + const entries = byTarget.get(annotation.target_id) ?? []; + entries.push(annotation); + byTarget.set(annotation.target_id, entries); + } + return byTarget; +} + +function graphqlSpanId(span: Record): string | undefined { + const context = spanContext(span); + return stringValue(context.span_id) ?? stringValue(span.span_id) ?? stringValue(span.spanId); +} + +function graphqlTraceRows(session: PhoenixRestSession): Record[] { + const rows: Record[] = []; + const seen = new Set(); + for (const trace of session.traces) { + const rawTrace = trace.raw; + if (!rawTrace) { + continue; + } + const candidates = [ + ...(isRecord(rawTrace.rootSpan) ? [rawTrace.rootSpan] : []), + ...connectionNodes(rawTrace.spans), + ]; + for (const span of candidates) { + const spanId = graphqlSpanId(span); + const dedupeKey = `${trace.traceId ?? ''}:${spanId ?? JSON.stringify(span)}`; + if (seen.has(dedupeKey)) { + continue; + } + seen.add(dedupeKey); + rows.push(span); + } + } + return rows; +} + +function graphqlAnnotations( + entries: readonly unknown[], + target: 'session' | 'trace' | 'span', + targetId?: string, +): PhoenixAnnotation[] { + return entries + .filter((entry): entry is Record => isRecord(entry)) + .map((annotation) => normalizeAnnotation(annotation, target, targetId)); +} + +function graphqlSessionAnnotations(session: PhoenixRestSession): PhoenixAnnotation[] { + return graphqlAnnotations( + asArray(session.raw?.sessionAnnotations), + 'session', + session.sessionId ?? session.id, + ); +} + +function graphqlTraceAnnotations(session: PhoenixRestSession): PhoenixAnnotation[] { + return session.traces.flatMap((trace) => + graphqlAnnotations(asArray(trace.raw?.traceAnnotations), 'trace', trace.traceId), + ); +} + +function graphqlSpanAnnotations(spans: readonly Record[]): PhoenixAnnotation[] { + return spans.flatMap((span) => + graphqlAnnotations(asArray(span.spanAnnotations), 'span', graphqlSpanId(span)), + ); +} + +function sessionIdFromSpans(spans: readonly Record[]): string | undefined { + for (const span of spans) { + const attributes = spanAttributes(span); + const sessionId = + attributeString(attributes, [ + 'session.id', + 'session_id', + 'metadata.session_id', + 'openinference.session.id', + ]) ?? stringValue(span.session_id); + if (sessionId) { + return sessionId; + } + } + return undefined; +} + +function buildOpenInPhoenixUrl( + config: PhoenixConfig, + externalTrace: ExternalTraceMetadata, + session: PhoenixRestSession | undefined, +): string | undefined { + if (externalTrace.uiUrl) { + return externalTrace.uiUrl; + } + const sessionId = session?.sessionId ?? externalTrace.sessionId ?? externalTrace.sessionNodeId; + if (sessionId) { + return `${config.baseUrl}/sessions/${encodeURIComponent(sessionId)}`; + } + if (externalTrace.traceId) { + return `${config.baseUrl}/projects/${encodeURIComponent( + externalTrace.projectId ?? externalTrace.project ?? config.project ?? '', + )}/traces/${encodeURIComponent(externalTrace.traceId)}`; + } + return config.baseUrl; +} + +async function buildLinkedSessionResponse( + externalTrace: ExternalTraceMetadata, + externalTraceWire: ExternalTraceMetadataWire, + config: PhoenixConfig, + session: PhoenixRestSession, + fetchImpl: typeof fetch, +): Promise { + const projectIdentifier = + session.projectId ?? externalTrace.projectId ?? externalTrace.project ?? config.project; + const traceIds = session.traces.map((trace) => trace.traceId).filter((id): id is string => !!id); + const graphqlSpanRows = graphqlTraceRows(session); + const spanRows = + graphqlSpanRows.length > 0 + ? graphqlSpanRows + : projectIdentifier + ? ( + await Promise.all( + traceIds.map((traceId) => + fetchTraceSpans(config, projectIdentifier, traceId, fetchImpl), + ), + ) + ).flat() + : []; + + const graphqlSessionAnnotationRows = graphqlSessionAnnotations(session); + const graphqlTraceAnnotationRows = graphqlTraceAnnotations(session); + const graphqlSpanAnnotationRows = graphqlSpanAnnotations(spanRows); + const shouldFetchRestAnnotations = + projectIdentifier && + graphqlSessionAnnotationRows.length === 0 && + graphqlTraceAnnotationRows.length === 0 && + graphqlSpanAnnotationRows.length === 0; + + const [sessionAnnotations, traceAnnotations, spanAnnotations] = shouldFetchRestAnnotations + ? await Promise.all([ + fetchAnnotations( + config, + projectIdentifier, + 'session', + [session.sessionId, session.id].filter((id): id is string => !!id), + fetchImpl, + ), + fetchAnnotations(config, projectIdentifier, 'trace', traceIds, fetchImpl), + fetchAnnotations( + config, + projectIdentifier, + 'span', + spanRows.map(graphqlSpanId).filter((id): id is string => !!id), + fetchImpl, + ), + ]) + : [graphqlSessionAnnotationRows, graphqlTraceAnnotationRows, graphqlSpanAnnotationRows]; + + const spanAnnotationsById = annotationsByTarget(spanAnnotations, 'span'); + const spans = spanRows + .map((span) => normalizeSpan(span, spanAnnotationsById)) + .filter((span): span is PhoenixSpanDetail => span !== undefined); + const spansByTraceId = new Map(); + for (const span of spans) { + if (!span.trace_id) { + continue; + } + const traceSpans = spansByTraceId.get(span.trace_id) ?? []; + traceSpans.push(span); + spansByTraceId.set(span.trace_id, traceSpans); + } + const traceAnnotationsById = annotationsByTarget(traceAnnotations, 'trace'); + const turns = session.traces.map((trace, index) => { + const traceSpans = trace.traceId ? (spansByTraceId.get(trace.traceId) ?? []) : []; + const rootSpan = traceSpans.find((span) => !span.parent_span_id) ?? traceSpans[0]; + const startTime = trace.startTime ?? rootSpan?.start_time; + const endTime = trace.endTime ?? rootSpan?.end_time; + const annotations = trace.traceId ? traceAnnotationsById.get(trace.traceId) : undefined; + return compactRecord({ + index: index + 1, + trace_id: trace.traceId, + start_time: startTime, + end_time: endTime, + duration_ms: durationMs(startTime, endTime) ?? rootSpan?.duration_ms, + status: rootSpan?.status, + root_span_id: rootSpan?.span_id, + input: rootSpan?.input, + output: rootSpan?.output, + token_usage: sumTokenUsage(traceSpans), + cost_usd: traceSpans.reduce((sum, span) => sum + (span.cost_usd ?? 0), 0), + ...(annotations && annotations.length > 0 ? { annotations } : {}), + }) as PhoenixSessionTurn; + }); + + const summary: PhoenixSessionSummary = { + id: session.id, + session_id: session.sessionId, + project_id: projectIdentifier, + project: externalTrace.project, + start_time: session.startTime, + end_time: session.endTime, + duration_ms: durationMs(session.startTime, session.endTime), + trace_count: finiteNumber(session.raw?.numTraces) ?? traceIds.length, + token_usage: tokenUsageFromGraphql(session.raw?.tokenUsage) ?? sumTokenUsage(turns), + cost_usd: + costFromGraphqlSummary(session.raw?.costSummary) ?? + turns.reduce((sum, turn) => sum + (turn.cost_usd ?? 0), 0), + ...(sessionAnnotations.length > 0 ? { annotations: sessionAnnotations } : {}), + }; + const annotations = [...sessionAnnotations, ...traceAnnotations, ...spanAnnotations]; + + return response('ok', 'Linked Phoenix session loaded.', externalTraceWire, { + open_in_phoenix_url: buildOpenInPhoenixUrl(config, externalTrace, session), + session: summary, + turns, + spans, + trace_tree: buildSpanTree(spans), + ...(annotations.length > 0 ? { annotations } : {}), + }); +} + +async function resolveSession( + externalTrace: ExternalTraceMetadata, + config: PhoenixConfig, + fetchImpl: typeof fetch, +): Promise { + const projectIdentifier = externalTrace.projectId ?? externalTrace.project ?? config.project; + + if (externalTrace.sessionNodeId) { + try { + const session = await fetchSessionByNodeId(config, externalTrace.sessionNodeId, fetchImpl); + if (session) { + return session; + } + } catch (error) { + if (!(error instanceof PhoenixReadError && error.code === 'schema_mismatch')) { + throw error; + } + // REST is only a compatibility fallback when the configured Phoenix + // GraphQL schema lacks the web-equivalent fields AgentV needs. + } + return fetchSessionByIdRest(config, externalTrace.sessionNodeId, fetchImpl); + } + + if (externalTrace.sessionId) { + if (projectIdentifier) { + try { + const session = await fetchSessionByIdGraphql( + config, + projectIdentifier, + externalTrace.sessionId, + fetchImpl, + ); + if (session) { + return session; + } + } catch (error) { + if (!(error instanceof PhoenixReadError && error.code === 'schema_mismatch')) { + throw error; + } + // Project-scoped GraphQL lookup requires a Phoenix GraphQL project ID. + // REST remains a narrow fallback for configured project names or older + // Phoenix schemas without the session read fields used by the web UI. + } + } + return fetchSessionByIdRest(config, externalTrace.sessionId, fetchImpl); + } + + const traceId = externalTrace.traceId; + if (!traceId || !projectIdentifier) { + return undefined; + } + + try { + const session = await fetchSessionByTraceIdGraphql( + config, + projectIdentifier, + traceId, + fetchImpl, + ); + if (session) { + return session; + } + } catch (error) { + if (!(error instanceof PhoenixReadError && error.code === 'schema_mismatch')) { + throw error; + } + // Fallback path for project-name configs or older Phoenix GraphQL schemas: + // use the public spans API only to recover the session ID from trace attrs. + } + + const spans = await fetchTraceSpans(config, projectIdentifier, traceId, fetchImpl); + const resolvedSessionId = sessionIdFromSpans(spans); + if (!resolvedSessionId) { + return undefined; + } + return fetchSessionByIdRest(config, resolvedSessionId, fetchImpl); +} + +export async function readPhoenixLinkedSession( + externalTrace: ExternalTraceMetadata | undefined, + externalTraceWire: ExternalTraceMetadataWire | undefined, + options: ReadPhoenixLinkedSessionOptions = {}, +): Promise { + if (!externalTrace || !externalTraceWire) { + return response( + 'missing_external_trace', + 'This AgentV run does not include external_trace metadata.', + undefined, + ); + } + + const env = options.env ?? process.env; + const fetchImpl = options.fetch ?? fetch; + const config = resolvePhoenixConfig(externalTrace, env); + if (!config) { + return response( + 'not_configured', + 'Phoenix read-through is not configured. Set AGENTV_PHOENIX_ENDPOINT or PHOENIX_HOST on the Dashboard server.', + externalTraceWire, + { open_in_phoenix_url: externalTrace.uiUrl }, + ); + } + + try { + const session = await resolveSession(externalTrace, config, fetchImpl); + if (!session) { + return response( + 'unresolved', + 'No Phoenix session matched the external_trace correlation metadata.', + externalTraceWire, + { open_in_phoenix_url: buildOpenInPhoenixUrl(config, externalTrace, undefined) }, + ); + } + return buildLinkedSessionResponse(externalTrace, externalTraceWire, config, session, fetchImpl); + } catch (error) { + if (error instanceof PhoenixReadError) { + return response(error.code, error.message, externalTraceWire, { + open_in_phoenix_url: buildOpenInPhoenixUrl(config, externalTrace, undefined), + }); + } + return response( + 'unreachable', + `Phoenix read-through failed: ${error instanceof Error ? error.message : String(error)}`, + externalTraceWire, + { open_in_phoenix_url: buildOpenInPhoenixUrl(config, externalTrace, undefined) }, + ); + } +} diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 3cc537fd5..315ae4969 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -10,6 +10,7 @@ * - GET /api/runs — list available run workspaces with metadata * - GET /api/runs/:filename — load results from a specific run workspace * - GET /api/runs/:filename/log — stream the captured console.log for a run + * - GET /api/runs/:filename/phoenix-session — read linked Phoenix session data * - GET /api/runs/:filename/evals/:evalId/files/* — read artifact files as JSON, * or as raw/downloadable text with ?raw=1 / ?download=1 * - GET /api/feedback — read feedback reviews @@ -52,13 +53,17 @@ import { AGENTV_RESULTS_ARTIFACTS_REF, DEFAULT_CATEGORY, type EvaluationResult, + type ExternalTraceMetadata, + type ExternalTraceMetadataWire, addProject, + externalTraceMetadataFromRecord, getProject, loadConfig, loadProjectRegistry, readGitResultArtifact, removeProject, syncProjects, + toExternalTraceMetadataWire, touchProject, } from '@agentv/core'; import type { Context } from 'hono'; @@ -83,6 +88,7 @@ import { loadManifestResults, parseResultManifest, } from './manifest.js'; +import { readPhoenixLinkedSession } from './phoenix-readthrough.js'; import { type SourcedResultFileMeta, clearRemoteRunTags, @@ -1000,6 +1006,69 @@ async function handleRunDetail(c: C, { searchDir, projectId }: DataContext) { } } +interface ExternalTraceSelection { + readonly metadata: ExternalTraceMetadata; + readonly wire: ExternalTraceMetadataWire; +} + +function isPhoenixExternalTrace(metadata: ExternalTraceMetadata): boolean { + const provider = metadata.provider?.toLowerCase(); + return provider === undefined || provider === 'phoenix'; +} + +function externalTraceFromManifestRecord( + record: ResultManifestRecord, +): ExternalTraceSelection | undefined { + const topLevel = record.external_trace + ? externalTraceMetadataFromRecord({ external_trace: record.external_trace }) + : undefined; + const fromMetadata = externalTraceMetadataFromRecord(record.metadata); + const metadata = topLevel ?? fromMetadata; + if (!metadata || !isPhoenixExternalTrace(metadata)) { + return undefined; + } + try { + return { + metadata, + wire: toExternalTraceMetadataWire(metadata), + }; + } catch { + return undefined; + } +} + +function findExternalTrace( + records: readonly ResultManifestRecord[], +): ExternalTraceSelection | undefined { + for (const record of records) { + const externalTrace = externalTraceFromManifestRecord(record); + if (externalTrace) { + return externalTrace; + } + } + return undefined; +} + +async function handlePhoenixSession(c: C, { searchDir, projectId }: DataContext) { + const filename = c.req.param('filename') ?? ''; + const meta = await findRunById(searchDir, filename, projectId); + if (!meta) return c.json({ error: 'Run not found' }, 404); + + try { + const records = await parseManifestForMeta(searchDir, meta, projectId); + const externalTrace = findExternalTrace(records); + return c.json(await readPhoenixLinkedSession(externalTrace?.metadata, externalTrace?.wire)); + } catch (error) { + return c.json({ + schema_version: 'agentv.dashboard.phoenix_session.v1', + status: 'schema_mismatch', + message: `Failed to inspect AgentV run metadata: ${ + error instanceof Error ? error.message : String(error) + }`, + }); + } +} + /** * Compute `run_dir` (relative to cwd, snake_case) and `suite_filter` (the * eval file path stored in benchmark.json metadata) for a local run manifest. @@ -2374,6 +2443,7 @@ export function createApp( return handleRunDelete(c, defaultCtx); }); app.get('/api/runs/:filename', (c) => handleRunDetail(c, defaultCtx)); + app.get('/api/runs/:filename/phoenix-session', (c) => handlePhoenixSession(c, defaultCtx)); app.get('/api/runs/:filename/log', (c) => handleRunLog(c, defaultCtx)); app.get('/api/runs/:filename/suites', (c) => handleRunSuites(c, defaultCtx)); app.get('/api/runs/:filename/categories', (c) => handleRunCategories(c, defaultCtx)); @@ -2491,6 +2561,9 @@ export function createApp( return withProject(c, handleRunDelete); }); app.get('/api/projects/:projectId/runs/:filename', (c) => withProject(c, handleRunDetail)); + app.get('/api/projects/:projectId/runs/:filename/phoenix-session', (c) => + withProject(c, handlePhoenixSession), + ); app.get('/api/projects/:projectId/runs/:filename/log', (c) => withProject(c, handleRunLog)); app.get('/api/projects/:projectId/runs/:filename/suites', (c) => withProject(c, handleRunSuites)); app.get('/api/projects/:projectId/runs/:filename/categories', (c) => diff --git a/apps/cli/test/commands/eval/artifact-writer.test.ts b/apps/cli/test/commands/eval/artifact-writer.test.ts index 71c1e88e9..57376247b 100644 --- a/apps/cli/test/commands/eval/artifact-writer.test.ts +++ b/apps/cli/test/commands/eval/artifact-writer.test.ts @@ -1076,6 +1076,59 @@ describe('writeArtifactsFromResults', () => { expect(indexLine).not.toHaveProperty('transcript_json_path'); }); + it('writes safe external_trace metadata without persisting Phoenix credentials', async () => { + const results = [ + makeResult({ + testId: 'external-trace-case', + target: 'codex', + metadata: { + external_trace: { + provider: 'phoenix', + endpoint: 'https://phoenix.example/v1/traces?api_key=secret', + project: 'agentv-dogfood', + session_node_id: 'UHJvamVjdFNlc3Npb246MQ==', + trace_id: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + ui_url: 'https://phoenix.example/sessions/codex-session-1?authorization=secret', + api_key: 'secret', + }, + external_trace_token: 'secret-flat-token', + safe_note: 'kept', + }, + }), + ]; + + await writeArtifactsFromResults(results, testDir); + + const indexLine = JSON.parse( + (await readFile(path.join(testDir, 'index.jsonl'), 'utf8')).trim(), + ); + expect(indexLine.external_trace).toEqual({ + provider: 'phoenix', + source: 'codex', + endpoint: 'https://phoenix.example/', + project: 'agentv-dogfood', + session_node_id: 'UHJvamVjdFNlc3Npb246MQ==', + trace_id: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + ui_url: 'https://phoenix.example/sessions/codex-session-1', + run_id: path.basename(testDir), + test_id: 'external-trace-case', + target: 'codex', + }); + expect(indexLine.metadata).toEqual({ safe_note: 'kept' }); + expect(JSON.stringify(indexLine)).not.toContain('secret'); + expect(JSON.stringify(indexLine)).not.toContain('api_key'); + + const envelope = TraceEnvelopeWireSchema.parse( + JSON.parse( + await readFile(path.join(testDir, 'external-trace-case', 'outputs', 'trace.json'), 'utf8'), + ), + ); + expect(envelope.external_trace).toEqual(indexLine.external_trace); + expect(envelope.source.metadata ?? {}).not.toHaveProperty('external_trace'); + expect(JSON.stringify(envelope)).not.toContain('secret'); + expect(JSON.stringify(envelope)).not.toContain('api_key'); + }); + it('omits per-test transcript links when the execution trace has no transcript rows', async () => { const results = [ makeResult({ diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index e51c8d6b5..d2e992c8a 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -454,15 +454,40 @@ function createMockStudioDir(baseDir: string): string { // ── Hono app (Dashboard SPA + API) ───────────────────────────────────────── describe('serve app', () => { + const PHOENIX_ENV_KEYS = [ + 'AGENTV_PHOENIX_ENDPOINT', + 'AGENTV_PHOENIX_API_KEY', + 'AGENTV_PHOENIX_AUTHORIZATION', + 'AGENTV_PHOENIX_HEADERS', + 'AGENTV_PHOENIX_GRAPHQL_URL', + 'PHOENIX_HOST', + 'PHOENIX_API_KEY', + 'PHOENIX_CLIENT_HEADERS', + ] as const; let tempDir: string; let studioDir: string; + let originalFetch: typeof fetch; + let originalPhoenixEnv: Record<(typeof PHOENIX_ENV_KEYS)[number], string | undefined>; beforeEach(() => { tempDir = mkdtempSync(path.join(tmpdir(), 'agentv-serve-test-')); studioDir = createMockStudioDir(tempDir); + originalFetch = globalThis.fetch; + originalPhoenixEnv = Object.fromEntries( + PHOENIX_ENV_KEYS.map((key) => [key, process.env[key]]), + ) as Record<(typeof PHOENIX_ENV_KEYS)[number], string | undefined>; }); afterEach(() => { + globalThis.fetch = originalFetch; + for (const key of PHOENIX_ENV_KEYS) { + const value = originalPhoenixEnv[key]; + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } rmSync(tempDir, { recursive: true, force: true }); }); @@ -967,6 +992,383 @@ describe('serve app', () => { }); }); + it('reads a linked Phoenix session through narrow server-side GraphQL/API requests', async () => { + const filename = '2026-03-25T10-15-00-000Z'; + createLocalRun(tempDir, filename, { + ...RESULT_A, + external_trace: { + provider: 'phoenix', + source: 'codex', + endpoint: 'https://phoenix.example/v1/traces?api_key=artifact-secret', + project: 'agentv-dogfood', + session_node_id: 'UHJvamVjdFNlc3Npb246MQ==', + ui_url: 'https://phoenix.example/sessions/codex-session-1?token=artifact-secret', + }, + }); + process.env.AGENTV_PHOENIX_ENDPOINT = 'https://phoenix.example'; + process.env.AGENTV_PHOENIX_API_KEY = 'server-secret'; + + const requests: Request[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const request = new Request(input, init); + requests.push(request); + const url = new URL(request.url); + if (url.pathname === '/graphql') { + const body = (await request.json()) as { + query: string; + variables: Record; + }; + expect(body.query).toContain('AgentVPhoenixLinkedSessionByNode'); + expect(body.query).toContain('AgentVPhoenixSessionFields'); + expect(body.query).toContain('AgentVPhoenixTraceFields'); + expect(body.query).toContain('AgentVPhoenixSpanFields'); + expect(body.variables.id).toBe('UHJvamVjdFNlc3Npb246MQ=='); + expect(request.headers.get('authorization')).toBe('Bearer server-secret'); + return Response.json({ + data: { + node: { + __typename: 'ProjectSession', + id: 'UHJvamVjdFNlc3Npb246MQ==', + sessionId: 'codex-session-1', + project: { id: 'project-1', name: 'agentv-dogfood' }, + numTraces: 1, + tokenUsage: { prompt: 12, completion: 8, total: 20 }, + costSummary: { + prompt: { cost: 0.01, tokens: 12 }, + completion: { cost: 0.01, tokens: 8 }, + total: { cost: 0.02, tokens: 20 }, + }, + startTime: '2026-03-25T10:00:00.000Z', + endTime: '2026-03-25T10:00:04.000Z', + sessionAnnotations: [ + { + id: 'ann-session', + projectSessionId: 'codex-session-1', + name: 'review', + annotatorKind: 'HUMAN', + label: 'pass', + score: 1, + explanation: 'Looks good', + metadata: { cookie: 'annotation-cookie-secret' }, + identifier: 'session-review', + createdAt: '2026-03-25T10:00:05.000Z', + updatedAt: '2026-03-25T10:00:05.000Z', + }, + ], + traces: { + edges: [ + { + node: { + id: 'trace-node-1', + traceId: 'trace-1', + startTime: '2026-03-25T10:00:00.000Z', + endTime: '2026-03-25T10:00:04.000Z', + latencyMs: 4000, + projectId: 'project-1', + project: { id: 'project-1', name: 'agentv-dogfood' }, + projectSessionId: 'UHJvamVjdFNlc3Npb246MQ==', + costSummary: { + total: { cost: 0.02, tokens: 20 }, + }, + traceAnnotations: [], + rootSpan: { + id: 'span-node-root', + spanId: 'span-root', + parentId: null, + name: 'agent turn', + spanKind: 'agent', + statusCode: 'OK', + startTime: '2026-03-25T10:00:00.000Z', + endTime: '2026-03-25T10:00:04.000Z', + latencyMs: 4000, + input: { + value: 'summarize the repo', + truncatedValue: 'summarize the repo', + mimeType: 'text/plain', + }, + output: { + value: 'repo summary', + truncatedValue: 'repo summary', + mimeType: 'text/plain', + }, + attributes: JSON.stringify({ + 'input.value': 'summarize the repo', + 'output.value': 'repo summary', + authorization: 'span-authorization-secret', + nested: { api_key: 'span-api-key-secret', visible: 'safe' }, + }), + tokenCountTotal: 20, + tokenCountPrompt: 12, + tokenCountCompletion: 8, + cumulativeTokenCountTotal: 20, + cumulativeTokenCountPrompt: 12, + cumulativeTokenCountCompletion: 8, + costSummary: { + total: { cost: 0.02, tokens: 20 }, + }, + spanAnnotations: [ + { + id: 'ann-span', + spanId: 'span-root', + name: 'latency', + annotatorKind: 'CODE', + label: 'ok', + score: null, + explanation: null, + metadata: { + headers: { authorization: 'span-annotation-secret' }, + visible: 'safe', + }, + identifier: 'span-latency', + createdAt: '2026-03-25T10:00:05.000Z', + updatedAt: '2026-03-25T10:00:05.000Z', + }, + ], + trace: { + id: 'trace-node-1', + traceId: 'trace-1', + costSummary: { total: { cost: 0.02, tokens: 20 } }, + }, + project: { id: 'project-1', name: 'agentv-dogfood' }, + }, + spans: { + edges: [ + { + node: { + id: 'span-node-root', + spanId: 'span-root', + parentId: null, + name: 'agent turn', + spanKind: 'agent', + statusCode: 'OK', + startTime: '2026-03-25T10:00:00.000Z', + endTime: '2026-03-25T10:00:04.000Z', + latencyMs: 4000, + input: { + value: 'summarize the repo', + truncatedValue: 'summarize the repo', + mimeType: 'text/plain', + }, + output: { + value: 'repo summary', + truncatedValue: 'repo summary', + mimeType: 'text/plain', + }, + attributes: '{}', + tokenCountTotal: 20, + tokenCountPrompt: 12, + tokenCountCompletion: 8, + cumulativeTokenCountTotal: 20, + cumulativeTokenCountPrompt: 12, + cumulativeTokenCountCompletion: 8, + costSummary: { total: { cost: 0.02, tokens: 20 } }, + spanAnnotations: [ + { + id: 'ann-span', + spanId: 'span-root', + name: 'latency', + annotatorKind: 'CODE', + label: 'ok', + score: null, + explanation: null, + metadata: {}, + identifier: 'span-latency', + createdAt: '2026-03-25T10:00:05.000Z', + updatedAt: '2026-03-25T10:00:05.000Z', + }, + ], + trace: { + id: 'trace-node-1', + traceId: 'trace-1', + costSummary: { total: { cost: 0.02, tokens: 20 } }, + }, + project: { id: 'project-1', name: 'agentv-dogfood' }, + }, + }, + { + node: { + id: 'span-node-child', + spanId: 'span-child', + parentId: 'span-root', + name: 'tool call', + spanKind: 'tool', + statusCode: 'OK', + startTime: '2026-03-25T10:00:01.000Z', + endTime: '2026-03-25T10:00:02.000Z', + latencyMs: 1000, + input: null, + output: null, + attributes: JSON.stringify({ 'openinference.span.kind': 'TOOL' }), + tokenCountTotal: null, + tokenCountPrompt: null, + tokenCountCompletion: null, + cumulativeTokenCountTotal: null, + cumulativeTokenCountPrompt: null, + cumulativeTokenCountCompletion: null, + costSummary: null, + spanAnnotations: [], + trace: { + id: 'trace-node-1', + traceId: 'trace-1', + costSummary: { total: { cost: 0.02, tokens: 20 } }, + }, + project: { id: 'project-1', name: 'agentv-dogfood' }, + }, + }, + ], + }, + }, + }, + ], + }, + }, + }, + }); + } + return Response.json({ error: 'unexpected request' }, { status: 500 }); + }) as typeof fetch; + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request(`/api/runs/${encodeURIComponent(filename)}/phoenix-session`); + expect(res.status).toBe(200); + const data = (await res.json()) as { + status: string; + external_trace: { endpoint: string; ui_url: string }; + session: { session_id: string; trace_count: number; token_usage?: { input?: number } }; + turns: Array<{ root_span_id?: string; input?: string; output?: string; cost_usd?: number }>; + trace_tree: Array<{ span_id: string; depth: number }>; + annotations: Array<{ name?: string; target?: string }>; + }; + + expect(data.status).toBe('ok'); + expect(data.external_trace).toMatchObject({ + endpoint: 'https://phoenix.example/', + ui_url: 'https://phoenix.example/sessions/codex-session-1', + }); + expect(data.session).toMatchObject({ + session_id: 'codex-session-1', + trace_count: 1, + token_usage: { input: 12 }, + }); + expect(data.turns[0]).toMatchObject({ + root_span_id: 'span-root', + input: 'summarize the repo', + output: 'repo summary', + cost_usd: 0.02, + }); + expect(data.trace_tree.map(({ span_id, depth }) => ({ span_id, depth }))).toEqual([ + { span_id: 'span-root', depth: 0 }, + { span_id: 'span-child', depth: 1 }, + ]); + expect(data.annotations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'review', target: 'session' }), + expect.objectContaining({ name: 'latency', target: 'span' }), + ]), + ); + expect(requests.map((request) => new URL(request.url).pathname)).toEqual(['/graphql']); + const serialized = JSON.stringify(data); + expect(serialized).not.toContain('server-secret'); + expect(serialized).not.toContain('artifact-secret'); + expect(serialized).not.toContain('annotation-cookie-secret'); + expect(serialized).not.toContain('span-authorization-secret'); + expect(serialized).not.toContain('span-api-key-secret'); + expect(serialized).not.toContain('span-annotation-secret'); + expect(serialized).not.toContain('Authorization'); + expect(serialized).toContain('safe'); + }); + + it('does not use artifact Phoenix endpoints as server network config', async () => { + const filename = '2026-03-25T10-17-00-000Z'; + createLocalRun(tempDir, filename, { + ...RESULT_A, + external_trace: { + provider: 'phoenix', + endpoint: 'https://artifact-controlled.example', + session_node_id: 'UHJvamVjdFNlc3Npb246MQ==', + ui_url: 'https://artifact-controlled.example/sessions/codex-session-1', + }, + }); + process.env.AGENTV_PHOENIX_API_KEY = 'server-secret'; + const requests: Request[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const request = new Request(input, init); + requests.push(request); + return Response.json({ error: 'unexpected request' }, { status: 500 }); + }) as typeof fetch; + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request(`/api/runs/${encodeURIComponent(filename)}/phoenix-session`); + expect(res.status).toBe(200); + const data = (await res.json()) as { + status: string; + message: string; + open_in_phoenix_url?: string; + }; + expect(data.status).toBe('not_configured'); + expect(data.message).toContain('AGENTV_PHOENIX_ENDPOINT'); + expect(data.open_in_phoenix_url).toBe( + 'https://artifact-controlled.example/sessions/codex-session-1', + ); + expect(requests).toHaveLength(0); + }); + + it('reports missing Phoenix configuration without contacting Phoenix', async () => { + const filename = '2026-03-25T10-20-00-000Z'; + createLocalRun(tempDir, filename, { + ...RESULT_A, + external_trace: { + provider: 'phoenix', + session_id: 'codex-session-1', + }, + }); + const requests: Request[] = []; + globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + requests.push(new Request(input, init)); + return Response.json({ error: 'unexpected request' }, { status: 500 }); + }) as typeof fetch; + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const res = await app.request(`/api/runs/${encodeURIComponent(filename)}/phoenix-session`); + expect(res.status).toBe(200); + const data = (await res.json()) as { status: string; message: string }; + + expect(data.status).toBe('not_configured'); + expect(data.message).toContain('AGENTV_PHOENIX_ENDPOINT'); + expect(requests).toEqual([]); + }); + + it('keeps run detail readable when Phoenix is unreachable', async () => { + const filename = '2026-03-25T10-25-00-000Z'; + createLocalRun(tempDir, filename, { + ...RESULT_A, + external_trace: { + provider: 'phoenix', + endpoint: 'https://phoenix.example', + trace_id: 'trace-1', + project: 'agentv-dogfood', + }, + }); + process.env.AGENTV_PHOENIX_ENDPOINT = 'https://phoenix.example'; + globalThis.fetch = (async () => { + throw new Error('connect ECONNREFUSED'); + }) as typeof fetch; + + const app = createApp([], tempDir, tempDir, undefined, { studioDir }); + const phoenixRes = await app.request( + `/api/runs/${encodeURIComponent(filename)}/phoenix-session`, + ); + expect(phoenixRes.status).toBe(200); + const phoenixData = (await phoenixRes.json()) as { status: string; message: string }; + expect(phoenixData.status).toBe('unreachable'); + expect(phoenixData.message).toContain('connect ECONNREFUSED'); + + const detailRes = await app.request(`/api/runs/${encodeURIComponent(filename)}`); + expect(detailRes.status).toBe(200); + const detailData = (await detailRes.json()) as { results: Array<{ testId: string }> }; + expect(detailData.results[0].testId).toBe('test-greeting'); + }); + it('exposes materialized final state and oplog watermark for local run tags', async () => { const runsDir = path.join(tempDir, '.agentv', 'results', 'runs'); mkdirSync(runsDir, { recursive: true }); diff --git a/apps/dashboard/src/components/PhoenixLinkedSessionPanel.test.tsx b/apps/dashboard/src/components/PhoenixLinkedSessionPanel.test.tsx new file mode 100644 index 000000000..8cb84b66f --- /dev/null +++ b/apps/dashboard/src/components/PhoenixLinkedSessionPanel.test.tsx @@ -0,0 +1,102 @@ +import { describe, expect, it } from 'bun:test'; +import { renderToStaticMarkup } from 'react-dom/server'; + +import type { PhoenixLinkedSessionResponse } from '~/lib/types'; + +import { PhoenixLinkedSessionPanelContent } from './PhoenixLinkedSessionPanel'; + +describe('PhoenixLinkedSessionPanelContent', () => { + it('renders linked session summary, turns, trace tree, and selected span detail', () => { + const response: PhoenixLinkedSessionResponse = { + schema_version: 'agentv.dashboard.phoenix_session.v1', + status: 'ok', + open_in_phoenix_url: 'https://phoenix.example/sessions/codex-session-1', + external_trace: { + provider: 'phoenix', + session_id: 'codex-session-1', + }, + session: { + session_id: 'codex-session-1', + project_id: 'project-1', + trace_count: 1, + duration_ms: 4200, + token_usage: { input: 12, output: 8, total: 20 }, + cost_usd: 0.02, + start_time: '2026-03-25T10:00:00.000Z', + }, + turns: [ + { + index: 1, + trace_id: 'trace-1', + duration_ms: 4200, + root_span_id: 'span-root', + input: 'summarize the repo', + output: 'repo summary', + }, + ], + spans: [ + { + span_id: 'span-root', + trace_id: 'trace-1', + name: 'agent turn', + status: 'OK', + input: 'summarize the repo', + output: 'repo summary', + token_usage: { total: 20 }, + attributes: { 'openinference.span.kind': 'LLM' }, + }, + ], + trace_tree: [ + { + span_id: 'span-root', + trace_id: 'trace-1', + name: 'agent turn', + depth: 0, + child_count: 1, + duration_ms: 4200, + }, + { + span_id: 'span-child', + trace_id: 'trace-1', + parent_span_id: 'span-root', + name: 'tool call', + depth: 1, + child_count: 0, + duration_ms: 900, + }, + ], + annotations: [{ name: 'review', target: 'session', label: 'pass' }], + }; + + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain('Phoenix Session'); + expect(html).toContain('codex-session-1'); + expect(html).toContain('Open in Phoenix'); + expect(html).toContain('summarize the repo'); + expect(html).toContain('repo summary'); + expect(html).toContain('agent turn'); + expect(html).toContain('tool call'); + expect(html).toContain('openinference.span.kind'); + }); + + it('renders recoverable unresolved state', () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain('Phoenix Session'); + expect(html).toContain('unresolved'); + expect(html).toContain('No Phoenix session matched'); + expect(html).toContain('Open in Phoenix'); + }); +}); diff --git a/apps/dashboard/src/components/PhoenixLinkedSessionPanel.tsx b/apps/dashboard/src/components/PhoenixLinkedSessionPanel.tsx new file mode 100644 index 000000000..32ac3123e --- /dev/null +++ b/apps/dashboard/src/components/PhoenixLinkedSessionPanel.tsx @@ -0,0 +1,322 @@ +import { useState } from 'react'; + +import { usePhoenixLinkedSession } from '~/lib/api'; +import type { + PhoenixLinkedSessionResponse, + PhoenixLinkedSessionSpan, + PhoenixLinkedSessionTokenUsage, + PhoenixLinkedSessionTraceNode, +} from '~/lib/types'; + +interface PhoenixLinkedSessionPanelProps { + runId: string; + projectId?: string; +} + +interface PhoenixLinkedSessionPanelContentProps { + response: PhoenixLinkedSessionResponse; + initialSpanId?: string; +} + +function formatDuration(ms: number | undefined): string { + if (ms === undefined) return 'n/a'; + if (ms < 1000) return `${Math.round(ms)} ms`; + const seconds = ms / 1000; + if (seconds < 60) return `${seconds.toFixed(seconds >= 10 ? 0 : 1)} s`; + const minutes = Math.floor(seconds / 60); + const remainder = Math.round(seconds % 60); + return `${minutes}m ${remainder}s`; +} + +function formatCost(value: number | undefined): string { + if (value === undefined || value <= 0) return 'n/a'; + return `$${value.toFixed(value < 0.01 ? 4 : 2)}`; +} + +function formatTokenUsage(usage: PhoenixLinkedSessionTokenUsage | undefined): string { + if (!usage) return 'n/a'; + const total = + usage.total ?? + (usage.input ?? 0) + (usage.output ?? 0) + (usage.reasoning ?? 0) + (usage.cached ?? 0); + return total > 0 ? total.toLocaleString() : 'n/a'; +} + +function formatDateTime(value: string | undefined): string { + if (!value) return 'n/a'; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : date.toLocaleString(); +} + +function valuePreview(value: unknown): string { + if (value === undefined || value === null) return ''; + if (typeof value === 'string') return value; + try { + return JSON.stringify(value, null, 2); + } catch { + return String(value); + } +} + +function compactPreview(value: unknown): string { + return valuePreview(value).replace(/\s+/g, ' ').trim(); +} + +function statusClass(status: PhoenixLinkedSessionResponse['status']): string { + if (status === 'ok') return 'border-emerald-800 bg-emerald-950/40 text-emerald-300'; + if (status === 'not_configured' || status === 'unresolved') { + return 'border-amber-800 bg-amber-950/40 text-amber-300'; + } + return 'border-red-800 bg-red-950/40 text-red-300'; +} + +function Metric({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
+ {value} +
+
+ ); +} + +function OpenInPhoenixLink({ href }: { href?: string }) { + if (!href) return null; + return ( + + Open in Phoenix + + ); +} + +function ErrorState({ response }: { response: PhoenixLinkedSessionResponse }) { + return ( +
+
+
+

Phoenix Session

+

{response.message}

+
+
+ + {response.status} + + +
+
+
+ ); +} + +function SpanTree({ + nodes, + selectedSpanId, + onSelect, +}: { + nodes: readonly PhoenixLinkedSessionTraceNode[]; + selectedSpanId?: string; + onSelect: (spanId: string) => void; +}) { + if (nodes.length === 0) { + return
No span tree returned.
; + } + + return ( +
+ {nodes.map((node) => { + const active = node.span_id === selectedSpanId; + return ( + + ); + })} +
+ ); +} + +function SpanDetail({ span }: { span?: PhoenixLinkedSessionSpan }) { + if (!span) { + return
No span selected.
; + } + + return ( +
+
+ + + +
+
+
+
Input
+ 
+            {valuePreview(span.input)}
+
+
+
+
Output
+ 
+            {valuePreview(span.output)}
+
+
+
+ {span.attributes ? ( +
+
Attributes
+ 
+            {valuePreview(span.attributes)}
+
+
+ ) : null} +
+ ); +} + +export function PhoenixLinkedSessionPanelContent({ + response, + initialSpanId, +}: PhoenixLinkedSessionPanelContentProps) { + const tree = response.trace_tree ?? []; + const spans = response.spans ?? []; + const [selectedSpanId, setSelectedSpanId] = useState( + initialSpanId ?? tree[0]?.span_id ?? spans[0]?.span_id, + ); + + if (response.status === 'missing_external_trace') { + return null; + } + + if (response.status !== 'ok') { + return ; + } + + const selectedSpan = + spans.find((span) => span.span_id === selectedSpanId) ?? + tree.find((span) => span.span_id === selectedSpanId) ?? + spans[0] ?? + tree[0]; + const session = response.session; + + return ( +
+
+
+
+

Phoenix Session

+ + linked + +
+

+ {session?.session_id ?? response.external_trace?.session_id ?? 'session'} +

+
+ +
+ +
+ + + + + +
+ +
+
+
+

Turns

+
+ {(response.turns ?? []).map((turn) => ( +
+ #{turn.index} + + + {compactPreview(turn.input) || turn.trace_id || 'trace'} + + + {compactPreview(turn.output)} + + + + {formatDuration(turn.duration_ms)} + +
+ ))} +
+
+ +
+

Trace Tree

+ +
+
+ +
+

Span Details

+ +
+
+
+ ); +} + +export function PhoenixLinkedSessionPanel({ runId, projectId }: PhoenixLinkedSessionPanelProps) { + const { data, isLoading, error } = usePhoenixLinkedSession(runId, projectId); + + if (data?.status === 'missing_external_trace') { + return null; + } + + if (isLoading) { + return ( +
+ Loading Phoenix session... +
+ ); + } + + if (error) { + return ( + + ); + } + + return data ? : null; +} diff --git a/apps/dashboard/src/components/RunDetail.tsx b/apps/dashboard/src/components/RunDetail.tsx index 387f9834a..08693f36a 100644 --- a/apps/dashboard/src/components/RunDetail.tsx +++ b/apps/dashboard/src/components/RunDetail.tsx @@ -26,6 +26,7 @@ import { summarizeQuality } from '~/lib/result-summary'; import { formatCategoryDisplay } from '~/lib/run-detail-context'; import { PassRatePill } from './PassRatePill'; +import { PhoenixLinkedSessionPanel } from './PhoenixLinkedSessionPanel'; import { ResultTable } from './ResultTable'; import { StatsCards } from './StatsCards'; @@ -124,6 +125,7 @@ export function RunDetail({ results, runId, projectId }: RunDetailProps) { return (
+

No evaluations found

This run has no results yet.

@@ -143,6 +145,8 @@ export function RunDetail({ results, runId, projectId }: RunDetailProps) { totalCost={totalCost > 0 ? totalCost : undefined} /> + + {/* Category Breakdown */}

Category Breakdown

diff --git a/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts b/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts index ad501dbff..d62895d3f 100644 --- a/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts +++ b/apps/dashboard/src/lib/__fixtures__/trace-session-read-model.ts @@ -100,9 +100,13 @@ export const traceSessionEnvelopeFixture = { external_trace: { provider: 'phoenix', source: 'codex', + endpoint: 'https://phoenix.example/v1/traces?authorization=secret', project: 'agentv-dogfood', + project_id: 'project-1', session_id: 'codex-session-123', + session_node_id: 'UHJvamVjdFNlc3Npb246MQ==', trace_id: 'phoenix-trace-456', + traceparent: '00-11111111111111111111111111111111-2222222222222222-01', ui_url: 'https://phoenix.example/projects/agentv-dogfood/traces/phoenix-trace-456?api_key=secret', run_id: '2026-06-21T10-00-00-000Z', diff --git a/apps/dashboard/src/lib/api.ts b/apps/dashboard/src/lib/api.ts index 869e1e81d..4d5ed06ce 100644 --- a/apps/dashboard/src/lib/api.ts +++ b/apps/dashboard/src/lib/api.ts @@ -32,6 +32,7 @@ import type { FilesystemBrowseResponse, FilesystemBrowseResponseWire, IndexResponse, + PhoenixLinkedSessionResponse, ProjectEntry, ProjectEntryWire, ProjectListResponse, @@ -152,10 +153,26 @@ export function runLogOptions(filename: string, projectId?: string) { }); } +export function phoenixLinkedSessionOptions(filename: string, projectId?: string) { + const url = projectId + ? `${projectApiBase(projectId)}/runs/${encodeURIComponent(filename)}/phoenix-session` + : `/api/runs/${encodeURIComponent(filename)}/phoenix-session`; + return queryOptions({ + queryKey: ['runs', filename, 'phoenix-session', projectId ?? ''], + queryFn: () => fetchJson(url), + enabled: !!filename, + staleTime: 30_000, + }); +} + export function useRunLog(filename: string, projectId?: string) { return useQuery(runLogOptions(filename, projectId)); } +export function usePhoenixLinkedSession(filename: string, projectId?: string) { + return useQuery(phoenixLinkedSessionOptions(filename, projectId)); +} + export function runSuitesOptions(runId: string) { return queryOptions({ queryKey: ['runs', runId, 'suites'], diff --git a/apps/dashboard/src/lib/trace-read-model.test.ts b/apps/dashboard/src/lib/trace-read-model.test.ts index 0bf2015f3..2f19aa46e 100644 --- a/apps/dashboard/src/lib/trace-read-model.test.ts +++ b/apps/dashboard/src/lib/trace-read-model.test.ts @@ -157,9 +157,13 @@ describe('trace session read model', () => { expect(session.external_trace).toEqual({ provider: 'phoenix', source: 'codex', + endpoint: 'https://phoenix.example/v1/traces', project: 'agentv-dogfood', + project_id: 'project-1', session_id: 'codex-session-123', + session_node_id: 'UHJvamVjdFNlc3Npb246MQ==', trace_id: 'phoenix-trace-456', + traceparent: '00-11111111111111111111111111111111-2222222222222222-01', ui_url: 'https://phoenix.example/projects/agentv-dogfood/traces/phoenix-trace-456', run_id: '2026-06-21T10-00-00-000Z', test_id: 'nested-session', diff --git a/apps/dashboard/src/lib/trace-read-model.ts b/apps/dashboard/src/lib/trace-read-model.ts index 9c8e0a5a3..ec34982a4 100644 --- a/apps/dashboard/src/lib/trace-read-model.ts +++ b/apps/dashboard/src/lib/trace-read-model.ts @@ -386,9 +386,18 @@ function projectScores(scores: unknown): TraceSessionScore[] | undefined { const EXTERNAL_TRACE_KEYS = [ 'provider', 'source', + 'endpoint', + 'profile', 'project', + 'project_id', 'session_id', + 'session_node_id', 'trace_id', + 'trace_node_id', + 'span_id', + 'span_node_id', + 'traceparent', + 'tracestate', 'ui_url', 'run_id', 'test_id', @@ -422,9 +431,21 @@ function sanitizeExternalTrace(value: unknown): ExternalTraceMetadata | undefine const sanitized = compactRecord({ provider: stringValue(record.provider), source: stringValue(record.source), + endpoint: sanitizeUrl(record.endpoint), + profile: stringValue(record.profile), project: stringValue(record.project), + project_id: stringValue(record.project_id) ?? stringValue(record.projectId), session_id: stringValue(record.session_id) ?? stringValue(record.session), + session_node_id: + stringValue(record.session_node_id) ?? + stringValue(record.session_node) ?? + stringValue(record.node_id), trace_id: stringValue(record.trace_id) ?? stringValue(record.trace), + trace_node_id: stringValue(record.trace_node_id) ?? stringValue(record.trace_node), + span_id: stringValue(record.span_id) ?? stringValue(record.span), + span_node_id: stringValue(record.span_node_id) ?? stringValue(record.span_node), + traceparent: stringValue(record.traceparent), + tracestate: stringValue(record.tracestate), ui_url: sanitizeUrl(record.ui_url ?? record.url ?? record.href), run_id: stringValue(record.run_id), test_id: stringValue(record.test_id), @@ -445,17 +466,35 @@ function externalTraceFromFlatMetadata( return sanitizeExternalTrace({ provider: metadata.external_trace_provider ?? metadata['external_trace.provider'], source: metadata.external_trace_source ?? metadata['external_trace.source'], + endpoint: metadata.external_trace_endpoint ?? metadata['external_trace.endpoint'], + profile: metadata.external_trace_profile ?? metadata['external_trace.profile'], project: metadata.external_trace_project ?? metadata['external_trace.project'], + project_id: metadata.external_trace_project_id ?? metadata['external_trace.project_id'], session_id: metadata.external_trace_session_id ?? metadata.external_trace_session ?? metadata['external_trace.session_id'] ?? metadata['external_trace.session'], + session_node_id: + metadata.external_trace_session_node_id ?? + metadata.external_trace_node_id ?? + metadata['external_trace.session_node_id'] ?? + metadata['external_trace.node_id'], trace_id: metadata.external_trace_trace_id ?? metadata.external_trace_trace ?? metadata['external_trace.trace_id'] ?? metadata['external_trace.trace'], + trace_node_id: + metadata.external_trace_trace_node_id ?? metadata['external_trace.trace_node_id'], + span_id: + metadata.external_trace_span_id ?? + metadata.external_trace_span ?? + metadata['external_trace.span_id'] ?? + metadata['external_trace.span'], + span_node_id: metadata.external_trace_span_node_id ?? metadata['external_trace.span_node_id'], + traceparent: metadata.external_trace_traceparent ?? metadata['external_trace.traceparent'], + tracestate: metadata.external_trace_tracestate ?? metadata['external_trace.tracestate'], ui_url: metadata.external_trace_ui_url ?? metadata.external_trace_url ?? diff --git a/apps/dashboard/src/lib/types.ts b/apps/dashboard/src/lib/types.ts index 55781a900..a6efab3ac 100644 --- a/apps/dashboard/src/lib/types.ts +++ b/apps/dashboard/src/lib/types.ts @@ -142,15 +142,115 @@ export interface ExternalTraceMetadata { */ provider?: string; source?: string; + endpoint?: string; + profile?: string; project?: string; + project_id?: string; session_id?: string; + session_node_id?: string; trace_id?: string; + trace_node_id?: string; + span_id?: string; + span_node_id?: string; + traceparent?: string; + tracestate?: string; ui_url?: string; run_id?: string; test_id?: string; target?: string; } +export interface PhoenixLinkedSessionTokenUsage { + input?: number; + output?: number; + reasoning?: number; + cached?: number; + total?: number; +} + +export interface PhoenixLinkedSessionAnnotation { + id?: string; + name?: string; + annotator_kind?: string; + label?: string; + score?: number; + explanation?: string; + target?: 'session' | 'trace' | 'span'; + target_id?: string; + result?: unknown; +} + +export interface PhoenixLinkedSessionSpan { + span_id: string; + trace_id?: string; + parent_span_id?: string; + name?: string; + span_kind?: string; + status?: string; + start_time?: string; + end_time?: string; + duration_ms?: number; + input?: unknown; + output?: unknown; + token_usage?: PhoenixLinkedSessionTokenUsage; + cost_usd?: number; + attributes?: Record; + annotations?: PhoenixLinkedSessionAnnotation[]; +} + +export interface PhoenixLinkedSessionTraceNode extends PhoenixLinkedSessionSpan { + depth: number; + child_count: number; +} + +export interface PhoenixLinkedSessionTurn { + index: number; + trace_id?: string; + start_time?: string; + end_time?: string; + duration_ms?: number; + status?: string; + root_span_id?: string; + input?: unknown; + output?: unknown; + token_usage?: PhoenixLinkedSessionTokenUsage; + cost_usd?: number; + annotations?: PhoenixLinkedSessionAnnotation[]; +} + +export interface PhoenixLinkedSessionSummary { + id?: string; + session_id?: string; + project_id?: string; + project?: string; + start_time?: string; + end_time?: string; + duration_ms?: number; + trace_count: number; + token_usage?: PhoenixLinkedSessionTokenUsage; + cost_usd?: number; + annotations?: PhoenixLinkedSessionAnnotation[]; +} + +export interface PhoenixLinkedSessionResponse { + schema_version: 'agentv.dashboard.phoenix_session.v1'; + status: + | 'ok' + | 'missing_external_trace' + | 'not_configured' + | 'unresolved' + | 'unreachable' + | 'schema_mismatch'; + message?: string; + external_trace?: ExternalTraceMetadata; + open_in_phoenix_url?: string; + session?: PhoenixLinkedSessionSummary; + turns?: PhoenixLinkedSessionTurn[]; + spans?: PhoenixLinkedSessionSpan[]; + trace_tree?: PhoenixLinkedSessionTraceNode[]; + annotations?: PhoenixLinkedSessionAnnotation[]; +} + export interface TraceSessionTokenUsage { input?: number; output?: number; diff --git a/apps/web/src/content/docs/docs/integrations/phoenix.mdx b/apps/web/src/content/docs/docs/integrations/phoenix.mdx index 65953a735..da25d0816 100644 --- a/apps/web/src/content/docs/docs/integrations/phoenix.mdx +++ b/apps/web/src/content/docs/docs/integrations/phoenix.mdx @@ -23,9 +23,10 @@ Phoenix can still appear in AgentV workflows in two narrow ways: already emitted spans independently. When an AgentV run artifact includes safe `external_trace` metadata, AgentV may -link to that external Phoenix session. A future read-through can use Phoenix -GraphQL/API to read the referenced session, but AgentV must not write AgentV -artifacts into Phoenix as part of that flow. +link to that external Phoenix session. Dashboard can also read the linked +session through Phoenix GraphQL/API from the AgentV server process and show a +compact session panel. AgentV must not write AgentV artifacts into Phoenix as +part of that flow. ## Local Inspection @@ -40,6 +41,32 @@ workspaces, remote results repositories, trace sidecars, transcripts, and artifact manifests. It does not require Phoenix, the `px` CLI, Phoenix database tables, or any Phoenix runtime process. +If a run has `external_trace` metadata and the Dashboard server has a Phoenix +endpoint configured, the run detail page can show the linked Phoenix session +summary, turns, trace IDs, root span input/output, timing, token usage, costs, +annotations, a trace tree, and selected span details. Missing configuration, +unreachable Phoenix, schema mismatches, or no matching session render as +recoverable empty/error states; AgentV run detail still loads from AgentV +artifacts. + +Configure the Dashboard server with one of: + +```bash +AGENTV_PHOENIX_ENDPOINT=https://phoenix.example +PHOENIX_HOST=https://phoenix.example +``` + +Optional server-side auth can be supplied with: + +```bash +AGENTV_PHOENIX_API_KEY=... +AGENTV_PHOENIX_AUTHORIZATION="Bearer ..." +PHOENIX_CLIENT_HEADERS='{"Authorization":"Bearer ..."}' +``` + +These credentials stay server-side and are not written to AgentV artifacts or +sent to the browser. + ## External Trace Metadata AgentV artifacts may carry metadata such as: @@ -49,9 +76,12 @@ AgentV artifacts may carry metadata such as: "external_trace": { "provider": "phoenix", "source": "codex", + "endpoint": "https://phoenix.example", "project": "agentv-dogfood", + "session_node_id": "UHJvamVjdFNlc3Npb246MQ==", "session_id": "codex-session-123", "trace_id": "phoenix-trace-456", + "traceparent": "00-11111111111111111111111111111111-2222222222222222-01", "ui_url": "https://phoenix.example/projects/agentv-dogfood/traces/phoenix-trace-456" } } @@ -78,3 +108,5 @@ already-existing session. - No Dashboard runtime dependency on Phoenix or `px`. - No direct Dashboard access to Phoenix database tables. - No Phoenix dataset or experiment creation as part of the zero-infra local path. +- No browser-side exposure of Phoenix API keys, authorization headers, cookies, + or tokens. diff --git a/apps/web/src/content/docs/docs/tools/dashboard.mdx b/apps/web/src/content/docs/docs/tools/dashboard.mdx index 5635615b7..409a3fdbc 100644 --- a/apps/web/src/content/docs/docs/tools/dashboard.mdx +++ b/apps/web/src/content/docs/docs/tools/dashboard.mdx @@ -50,7 +50,17 @@ service. If a trace artifact includes safe `external_trace` metadata for spans that were already emitted to Phoenix by Codex, Arize, or another hook, Dashboard may show -that external reference. AgentV still treats the local/Git-backed run artifacts +that external reference and, when the Dashboard server has a Phoenix endpoint +configured, read the linked Phoenix session through Phoenix GraphQL/API. The +run detail page renders this as optional external context with session summary, +turns, trace IDs, root span input/output, timing, token usage, costs, +annotations, trace tree, selected span details, and an **Open in Phoenix** link. + +Phoenix endpoint and auth configuration stay on the server process +(`AGENTV_PHOENIX_ENDPOINT`, `PHOENIX_HOST`, `AGENTV_PHOENIX_API_KEY`, +`AGENTV_PHOENIX_AUTHORIZATION`, or `PHOENIX_CLIENT_HEADERS`). API keys, +authorization headers, cookies, and tokens are not written to AgentV artifacts +or sent to the browser. AgentV still treats the local/Git-backed run artifacts as canonical and does not export or project completed runs, transcripts, datasets, experiments, or indexes into Phoenix. diff --git a/packages/core/src/evaluation/external-trace.ts b/packages/core/src/evaluation/external-trace.ts new file mode 100644 index 000000000..61e88cd82 --- /dev/null +++ b/packages/core/src/evaluation/external-trace.ts @@ -0,0 +1,482 @@ +/** + * Safe external trace correlation metadata. + * + * AgentV artifacts remain the canonical source for eval results. This shape is + * only a non-secret pointer to spans that another system already emitted, so + * artifact writers keep it deliberately small and credential-free. + */ + +import { z } from 'zod'; +import type { EvaluationResult } from './types.js'; + +export interface ExternalTraceMetadata { + readonly provider?: string; + readonly source?: string; + readonly endpoint?: string; + readonly profile?: string; + readonly project?: string; + readonly projectId?: string; + readonly sessionId?: string; + readonly sessionNodeId?: string; + readonly traceId?: string; + readonly traceNodeId?: string; + readonly spanId?: string; + readonly spanNodeId?: string; + readonly traceparent?: string; + readonly tracestate?: string; + readonly uiUrl?: string; + readonly runId?: string; + readonly testId?: string; + readonly target?: string; +} + +export const ExternalTraceMetadataWireSchema = z + .object({ + provider: z.string().optional(), + source: z.string().optional(), + endpoint: z.string().optional(), + profile: z.string().optional(), + project: z.string().optional(), + project_id: z.string().optional(), + session_id: z.string().optional(), + session_node_id: z.string().optional(), + trace_id: z.string().optional(), + trace_node_id: z.string().optional(), + span_id: z.string().optional(), + span_node_id: z.string().optional(), + traceparent: z.string().optional(), + tracestate: z.string().optional(), + ui_url: z.string().optional(), + run_id: z.string().optional(), + test_id: z.string().optional(), + target: z.string().optional(), + }) + .strict(); + +export type ExternalTraceMetadataWire = z.infer; + +const EXTERNAL_TRACE_KEYS = [ + 'provider', + 'source', + 'endpoint', + 'profile', + 'project', + 'projectId', + 'sessionId', + 'sessionNodeId', + 'traceId', + 'traceNodeId', + 'spanId', + 'spanNodeId', + 'traceparent', + 'tracestate', + 'uiUrl', + 'runId', + 'testId', + 'target', +] as const; + +const TRACEPARENT_RE = /^[\da-f]{2}-[\da-f]{32}-[\da-f]{16}-[\da-f]{2}$/i; + +type ExternalTraceKey = (typeof EXTERNAL_TRACE_KEYS)[number]; + +type ExternalTraceInput = Partial>; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function isCredentialLikeKey(key: string): boolean { + const normalized = key.toLowerCase(); + if (normalized === 'traceparent' || normalized === 'tracestate') { + return false; + } + return /(^|[._-])(api[._-]?key|authorization|bearer|password|secret|private[._-]?key|access[._-]?token|auth[._-]?token|client[._-]?secret|cookie|id[._-]?token|refresh[._-]?token|session[._-]?token|token)($|[._-])/.test( + normalized, + ); +} + +function sanitizeUrl(value: unknown, options?: { stripOtlpPath?: boolean }): string | undefined { + const raw = stringValue(value); + if (!raw) { + return undefined; + } + try { + const url = new URL(raw); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { + return undefined; + } + url.search = ''; + url.hash = ''; + if (options?.stripOtlpPath) { + url.pathname = url.pathname + .replace(/\/v1\/traces\/?$/i, '') + .replace(/\/v1\/?$/i, '') + .replace(/\/+$/g, ''); + if (!url.pathname) { + url.pathname = '/'; + } + } + return url.toString(); + } catch { + return undefined; + } +} + +function sanitizeTraceparent(value: unknown): string | undefined { + const raw = stringValue(value); + return raw && TRACEPARENT_RE.test(raw) ? raw.toLowerCase() : undefined; +} + +function traceIdFromTraceparent(traceparent: string | undefined): string | undefined { + return traceparent?.split('-')[1]; +} + +function sanitizeTracestate(value: unknown): string | undefined { + const raw = stringValue(value); + if (!raw) { + return undefined; + } + const entries = raw + .split(',') + .map((entry) => entry.trim()) + .filter((entry) => { + const separator = entry.indexOf('='); + const key = separator >= 0 ? entry.slice(0, separator) : entry; + return key.length > 0 && !isCredentialLikeKey(key); + }); + const sanitized = entries.join(','); + return sanitized.length > 0 && sanitized.length <= 512 ? sanitized : undefined; +} + +function compactExternalTrace(value: ExternalTraceMetadata): ExternalTraceMetadata | undefined { + const compacted = Object.fromEntries( + Object.entries(value).filter(([, entry]) => entry !== undefined), + ) as ExternalTraceMetadata; + return EXTERNAL_TRACE_KEYS.some((key) => compacted[key] !== undefined) ? compacted : undefined; +} + +function sanitizeExternalTraceObject(value: ExternalTraceInput): ExternalTraceMetadata | undefined { + const traceparent = sanitizeTraceparent(value.traceparent); + return compactExternalTrace({ + provider: stringValue(value.provider), + source: stringValue(value.source), + endpoint: sanitizeUrl(value.endpoint, { stripOtlpPath: true }), + profile: stringValue(value.profile), + project: stringValue(value.project), + projectId: stringValue(value.projectId), + sessionId: stringValue(value.sessionId), + sessionNodeId: stringValue(value.sessionNodeId), + traceId: stringValue(value.traceId) ?? traceIdFromTraceparent(traceparent), + traceNodeId: stringValue(value.traceNodeId), + spanId: stringValue(value.spanId), + spanNodeId: stringValue(value.spanNodeId), + traceparent, + tracestate: sanitizeTracestate(value.tracestate), + uiUrl: sanitizeUrl(value.uiUrl), + runId: stringValue(value.runId), + testId: stringValue(value.testId), + target: stringValue(value.target), + }); +} + +function externalTraceInputFromRecord(record: Record): ExternalTraceInput { + return { + provider: record.provider, + source: record.source, + endpoint: record.endpoint, + profile: record.profile, + project: record.project ?? record.project_name, + projectId: record.project_id ?? record.projectId, + sessionId: record.session_id ?? record.session ?? record.sessionId, + sessionNodeId: + record.session_node_id ?? record.session_node ?? record.node_id ?? record.sessionNodeId, + traceId: record.trace_id ?? record.trace ?? record.traceId, + traceNodeId: record.trace_node_id ?? record.trace_node ?? record.traceNodeId, + spanId: record.span_id ?? record.span ?? record.spanId, + spanNodeId: record.span_node_id ?? record.span_node ?? record.spanNodeId, + traceparent: record.traceparent, + tracestate: record.tracestate, + uiUrl: record.ui_url ?? record.url ?? record.href ?? record.uiUrl, + runId: record.run_id ?? record.runId, + testId: record.test_id ?? record.testId, + target: record.target, + }; +} + +function externalTraceFromFlatRecord( + record: Record | undefined, +): ExternalTraceMetadata | undefined { + if (!record) { + return undefined; + } + return sanitizeExternalTraceObject({ + provider: record.external_trace_provider ?? record['external_trace.provider'], + source: record.external_trace_source ?? record['external_trace.source'], + endpoint: record.external_trace_endpoint ?? record['external_trace.endpoint'], + profile: record.external_trace_profile ?? record['external_trace.profile'], + project: + record.external_trace_project ?? + record.external_trace_project_name ?? + record['external_trace.project'] ?? + record['external_trace.project_name'], + projectId: record.external_trace_project_id ?? record['external_trace.project_id'], + sessionId: + record.external_trace_session_id ?? + record.external_trace_session ?? + record['external_trace.session_id'] ?? + record['external_trace.session'], + sessionNodeId: + record.external_trace_session_node_id ?? + record.external_trace_node_id ?? + record['external_trace.session_node_id'] ?? + record['external_trace.node_id'], + traceId: + record.external_trace_trace_id ?? + record.external_trace_trace ?? + record['external_trace.trace_id'] ?? + record['external_trace.trace'], + traceNodeId: record.external_trace_trace_node_id ?? record['external_trace.trace_node_id'], + spanId: + record.external_trace_span_id ?? + record.external_trace_span ?? + record['external_trace.span_id'] ?? + record['external_trace.span'], + spanNodeId: record.external_trace_span_node_id ?? record['external_trace.span_node_id'], + traceparent: record.external_trace_traceparent ?? record['external_trace.traceparent'], + tracestate: record.external_trace_tracestate ?? record['external_trace.tracestate'], + uiUrl: + record.external_trace_ui_url ?? + record.external_trace_url ?? + record['external_trace.ui_url'] ?? + record['external_trace.url'], + runId: record.external_trace_run_id ?? record['external_trace.run_id'], + testId: record.external_trace_test_id ?? record['external_trace.test_id'], + target: record.external_trace_target ?? record['external_trace.target'], + }); +} + +export function isExternalTraceMetadataKey(key: string): boolean { + return ( + key === 'external_trace' || + key.startsWith('external_trace_') || + key.startsWith('external_trace.') + ); +} + +export function omitExternalTraceMetadataKeys( + value: Record | undefined, +): Record | undefined { + if (!value) { + return undefined; + } + const entries = Object.entries(value).filter(([key]) => !isExternalTraceMetadataKey(key)); + return entries.length > 0 ? Object.fromEntries(entries) : undefined; +} + +export function sanitizeExternalTraceMetadata(value: unknown): ExternalTraceMetadata | undefined { + if (!isRecord(value)) { + return undefined; + } + return sanitizeExternalTraceObject(externalTraceInputFromRecord(value)); +} + +export function externalTraceMetadataFromRecord( + value: Record | undefined, +): ExternalTraceMetadata | undefined { + if (!value) { + return undefined; + } + return sanitizeExternalTraceMetadata(value.external_trace) ?? externalTraceFromFlatRecord(value); +} + +function envString( + env: Readonly>, + keys: readonly string[], +): string | undefined { + for (const key of keys) { + const value = stringValue(env[key]); + if (value) { + return value; + } + } + return undefined; +} + +export function externalTraceMetadataFromEnv( + env: Readonly> = process.env, +): ExternalTraceMetadata | undefined { + const traceparent = envString(env, ['AGENTV_EXTERNAL_TRACE_TRACEPARENT', 'TRACEPARENT']); + const endpoint = envString(env, [ + 'AGENTV_EXTERNAL_TRACE_ENDPOINT', + 'AGENTV_PHOENIX_ENDPOINT', + 'AGENTV_PHOENIX_BASE_URL', + 'PHOENIX_BASE_URL', + 'PHOENIX_ENDPOINT', + 'PHOENIX_HOST', + 'PHOENIX_COLLECTOR_ENDPOINT', + 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', + 'OTEL_EXPORTER_OTLP_ENDPOINT', + ]); + const project = envString(env, [ + 'AGENTV_EXTERNAL_TRACE_PROJECT', + 'AGENTV_EXTERNAL_TRACE_PROJECT_NAME', + 'AGENTV_PHOENIX_PROJECT', + 'AGENTV_PHOENIX_PROJECT_NAME', + 'PHOENIX_PROJECT_NAME', + 'OPENINFERENCE_PROJECT_NAME', + ]); + + return sanitizeExternalTraceObject({ + provider: + envString(env, ['AGENTV_EXTERNAL_TRACE_PROVIDER']) ?? + (endpoint || project ? 'phoenix' : undefined), + source: envString(env, ['AGENTV_EXTERNAL_TRACE_SOURCE']), + endpoint, + profile: envString(env, ['AGENTV_EXTERNAL_TRACE_PROFILE', 'AGENTV_PHOENIX_PROFILE']), + project, + projectId: envString(env, [ + 'AGENTV_EXTERNAL_TRACE_PROJECT_ID', + 'AGENTV_PHOENIX_PROJECT_ID', + 'PHOENIX_PROJECT_ID', + ]), + sessionId: envString(env, [ + 'AGENTV_EXTERNAL_TRACE_SESSION_ID', + 'PHOENIX_SESSION_ID', + 'ARIZE_SESSION_ID', + 'CODEX_SESSION_ID', + ]), + sessionNodeId: envString(env, [ + 'AGENTV_EXTERNAL_TRACE_SESSION_NODE_ID', + 'PHOENIX_SESSION_NODE_ID', + ]), + traceId: envString(env, [ + 'AGENTV_EXTERNAL_TRACE_TRACE_ID', + 'PHOENIX_TRACE_ID', + 'ARIZE_TRACE_ID', + ]), + traceNodeId: envString(env, ['AGENTV_EXTERNAL_TRACE_TRACE_NODE_ID', 'PHOENIX_TRACE_NODE_ID']), + spanId: envString(env, ['AGENTV_EXTERNAL_TRACE_SPAN_ID', 'PHOENIX_SPAN_ID']), + spanNodeId: envString(env, ['AGENTV_EXTERNAL_TRACE_SPAN_NODE_ID', 'PHOENIX_SPAN_NODE_ID']), + traceparent, + tracestate: envString(env, ['AGENTV_EXTERNAL_TRACE_TRACESTATE', 'TRACESTATE']), + uiUrl: envString(env, ['AGENTV_EXTERNAL_TRACE_UI_URL', 'PHOENIX_UI_URL']), + runId: envString(env, ['AGENTV_EXTERNAL_TRACE_RUN_ID']), + testId: envString(env, ['AGENTV_EXTERNAL_TRACE_TEST_ID']), + target: envString(env, ['AGENTV_EXTERNAL_TRACE_TARGET']), + }); +} + +function mergeExternalTraceMetadata( + values: readonly (ExternalTraceMetadata | undefined)[], +): ExternalTraceMetadata | undefined { + const merged = Object.assign({}, ...values.filter(Boolean)) as ExternalTraceMetadata; + return compactExternalTrace(merged); +} + +function hasExternalTraceSignal(metadata: ExternalTraceMetadata | undefined): boolean { + if (!metadata) { + return false; + } + return [ + metadata.endpoint, + metadata.profile, + metadata.project, + metadata.projectId, + metadata.sessionId, + metadata.sessionNodeId, + metadata.traceId, + metadata.traceNodeId, + metadata.spanId, + metadata.spanNodeId, + metadata.traceparent, + metadata.tracestate, + metadata.uiUrl, + metadata.runId, + ].some((value) => value !== undefined); +} + +export function externalTraceMetadataForResult( + result: EvaluationResult, + options: { runId?: string; env?: Readonly> } = {}, +): ExternalTraceMetadata | undefined { + const resultMetadata = externalTraceMetadataFromRecord(result.metadata); + const traceMetadata = externalTraceMetadataFromRecord(result.trace.metadata); + const envMetadata = externalTraceMetadataFromEnv(options.env); + const externalMetadata = mergeExternalTraceMetadata([envMetadata, traceMetadata, resultMetadata]); + + if (!hasExternalTraceSignal(externalMetadata)) { + return undefined; + } + + return mergeExternalTraceMetadata([ + externalMetadata, + sanitizeExternalTraceObject({ + runId: externalMetadata?.runId ?? options.runId, + testId: result.testId, + target: result.target, + source: + externalMetadata?.source ?? + (typeof result.trace.metadata?.provider === 'string' + ? result.trace.metadata.provider + : result.target), + }), + ]); +} + +export function toExternalTraceMetadataWire( + metadata: ExternalTraceMetadata, +): ExternalTraceMetadataWire { + const wire = { + provider: metadata.provider, + source: metadata.source, + endpoint: metadata.endpoint, + profile: metadata.profile, + project: metadata.project, + project_id: metadata.projectId, + session_id: metadata.sessionId, + session_node_id: metadata.sessionNodeId, + trace_id: metadata.traceId, + trace_node_id: metadata.traceNodeId, + span_id: metadata.spanId, + span_node_id: metadata.spanNodeId, + traceparent: metadata.traceparent, + tracestate: metadata.tracestate, + ui_url: metadata.uiUrl, + run_id: metadata.runId, + test_id: metadata.testId, + target: metadata.target, + }; + return ExternalTraceMetadataWireSchema.parse( + Object.fromEntries(Object.entries(wire).filter(([, value]) => value !== undefined)), + ); +} + +export function fromExternalTraceMetadataWire( + input: ExternalTraceMetadataWire, +): ExternalTraceMetadata { + return { + provider: input.provider, + source: input.source, + endpoint: input.endpoint, + profile: input.profile, + project: input.project, + projectId: input.project_id, + sessionId: input.session_id, + sessionNodeId: input.session_node_id, + traceId: input.trace_id, + traceNodeId: input.trace_node_id, + spanId: input.span_id, + spanNodeId: input.span_node_id, + traceparent: input.traceparent, + tracestate: input.tracestate, + uiUrl: input.ui_url, + runId: input.run_id, + testId: input.test_id, + target: input.target, + }; +} diff --git a/packages/core/src/evaluation/run-artifacts.ts b/packages/core/src/evaluation/run-artifacts.ts index af93cc692..af07822db 100644 --- a/packages/core/src/evaluation/run-artifacts.ts +++ b/packages/core/src/evaluation/run-artifacts.ts @@ -12,6 +12,12 @@ import { copyFile, mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { traceEnvelopeToTranscriptJsonLines } from '../import/types.js'; +import { + type ExternalTraceMetadataWire, + externalTraceMetadataForResult, + omitExternalTraceMetadataKeys, + toExternalTraceMetadataWire, +} from './external-trace.js'; import { DEFAULT_THRESHOLD } from './graders/scoring.js'; import { type ExportDuplicatePolicy, @@ -229,6 +235,7 @@ export interface IndexArtifactEntry { readonly targets_path?: string; readonly files_path?: string; readonly graders_path?: string; + readonly external_trace?: ExternalTraceMetadataWire; readonly projection_identity?: ProjectionIdentityWire; readonly export_metadata?: { readonly duplicate_policy: ExportDuplicatePolicy; @@ -439,19 +446,31 @@ function toIndexMetadata( if (!metadata) { return undefined; } + const safeMetadata = omitExternalTraceMetadataKeys(metadata); + if (!safeMetadata) { + return undefined; + } const rerunSource = toIndexRerunSource(metadata.rerunSource); const preparedAttempt = toIndexPreparedAttempt(metadata.preparedAttempt); if (!rerunSource && !preparedAttempt) { - return { ...metadata }; + return { ...safeMetadata }; } const reservedKeys = new Set(['rerunSource', 'preparedAttempt']); return { - ...Object.fromEntries(Object.entries(metadata).filter(([key]) => !reservedKeys.has(key))), + ...Object.fromEntries(Object.entries(safeMetadata).filter(([key]) => !reservedKeys.has(key))), ...(rerunSource ? { rerun_source: rerunSource } : {}), ...(preparedAttempt ? { prepared_attempt: preparedAttempt } : {}), }; } +function toIndexExternalTrace( + result: EvaluationResult, + runId: string | undefined, +): ExternalTraceMetadataWire | undefined { + const externalTrace = externalTraceMetadataForResult(result, { runId }); + return externalTrace ? toExternalTraceMetadataWire(externalTrace) : undefined; +} + function buildExportMetadata( duplicatePolicy: ExportDuplicatePolicy | undefined, projectionIdentity: ProjectionIdentity | undefined, @@ -946,6 +965,7 @@ export function buildIndexArtifactEntry( ? toRelativeArtifactPath(options.outputDir, options.responsePath) : undefined, ...options.extraIndexFields, + external_trace: toIndexExternalTrace(result, options.projectionIdentity?.dimensions.runId), projection_identity: options.projectionIdentity ? toProjectionIdentityWire(options.projectionIdentity) : undefined, @@ -1005,6 +1025,7 @@ export function buildResultIndexArtifact( ? path.posix.join(artifactSubdir, 'outputs', 'response.md') : undefined, ...extraIndexFields, + external_trace: toIndexExternalTrace(result, options?.projectionIdentity?.dimensions.runId), projection_identity: options?.projectionIdentity ? toProjectionIdentityWire(options.projectionIdentity) : undefined, diff --git a/packages/core/src/evaluation/trace-envelope.ts b/packages/core/src/evaluation/trace-envelope.ts index c1fbb961e..6f01fdb51 100644 --- a/packages/core/src/evaluation/trace-envelope.ts +++ b/packages/core/src/evaluation/trace-envelope.ts @@ -23,6 +23,15 @@ import { createHash } from 'node:crypto'; import { z } from 'zod'; +import { + type ExternalTraceMetadata, + type ExternalTraceMetadataWire, + ExternalTraceMetadataWireSchema, + externalTraceMetadataForResult, + fromExternalTraceMetadataWire, + omitExternalTraceMetadataKeys, + toExternalTraceMetadataWire, +} from './external-trace.js'; import { type ExportDuplicatePolicy, type ProjectionIdentity, @@ -173,6 +182,7 @@ export interface TraceEnvelope { readonly artifactId: string; readonly createdAt: string; readonly eval: TraceEnvelopeEval; + readonly externalTrace?: ExternalTraceMetadata; readonly projectionIdentity?: ProjectionIdentity; readonly exportMetadata?: TraceEnvelopeExportMetadata; readonly replay?: TraceEnvelopeReplay; @@ -338,6 +348,7 @@ export const TraceEnvelopeWireSchema = z artifact_id: z.string(), created_at: z.string(), eval: TraceEnvelopeEvalWireSchema, + external_trace: ExternalTraceMetadataWireSchema.optional(), projection_identity: ProjectionIdentityWireSchema.optional(), export_metadata: TraceEnvelopeExportMetadataWireSchema.optional(), replay: TraceEnvelopeReplayWireSchema.optional(), @@ -357,6 +368,7 @@ export type TraceEnvelopeBodyWire = z.infer; export type TraceEnvelopeSpanWire = z.infer; export type TraceEnvelopeSourceWire = z.infer; export type TraceEnvelopeCaptureWire = z.infer; +export type TraceEnvelopeExternalTraceWire = ExternalTraceMetadataWire; export type TraceEnvelopeConversionWarningWire = z.infer< typeof TraceEnvelopeConversionWarningWireSchema >; @@ -372,6 +384,7 @@ export interface BuildTraceEnvelopeOptions { readonly runId?: string; readonly experiment?: string; readonly source?: Partial; + readonly externalTrace?: ExternalTraceMetadata; readonly replay?: TraceEnvelopeReplay; readonly capture?: Partial; readonly artifacts?: Readonly>; @@ -609,8 +622,8 @@ function sourceFromResult( ? result.trace.metadata.provider : undefined; const metadata = dropUndefined({ - ...result.trace.metadata, - ...options.source?.metadata, + ...omitExternalTraceMetadataKeys(result.trace.metadata), + ...omitExternalTraceMetadataKeys(options.source?.metadata), }); return { kind: options.source?.kind ?? 'agentv_run', @@ -970,6 +983,7 @@ export function buildTraceEnvelopeFromEvaluationResult( const attempt = options.attempt ?? 0; const variant = options.variant ?? null; const runId = options.runId ?? result.timestamp; + const externalTrace = options.externalTrace ?? externalTraceMetadataForResult(result, { runId }); const evalIdentity: TraceEnvelopeEval = { evalId: options.evalId, evalPath: options.evalPath, @@ -1004,6 +1018,7 @@ export function buildTraceEnvelopeFromEvaluationResult( artifactId, createdAt: now.toISOString(), eval: evalIdentity, + externalTrace, projectionIdentity, exportMetadata: options.duplicatePolicy ? { @@ -1037,6 +1052,9 @@ export function toTraceEnvelopeWire(envelope: TraceEnvelope): TraceEnvelopeWire artifact_id: envelope.artifactId, created_at: envelope.createdAt, eval: toTraceEnvelopeEvalWire(envelope.eval), + external_trace: envelope.externalTrace + ? toExternalTraceMetadataWire(envelope.externalTrace) + : undefined, projection_identity: envelope.projectionIdentity ? toProjectionIdentityWire(envelope.projectionIdentity) : undefined, @@ -1061,6 +1079,9 @@ export function fromTraceEnvelopeWire(input: unknown): TraceEnvelope { artifactId: wire.artifact_id, createdAt: wire.created_at, eval: fromTraceEnvelopeEvalWire(wire.eval), + externalTrace: wire.external_trace + ? fromExternalTraceMetadataWire(wire.external_trace) + : undefined, projectionIdentity: wire.projection_identity ? fromProjectionIdentityWire(wire.projection_identity) : undefined, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 498285a01..ff73abe87 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,6 +2,7 @@ export * from './evaluation/content.js'; export * from './evaluation/types.js'; export * from './evaluation/trace.js'; export * from './evaluation/trace-envelope.js'; +export * from './evaluation/external-trace.js'; export * from './evaluation/projection-identity.js'; export * from './evaluation/replay-fixtures.js'; export * from './evaluation/replay-trace-envelopes.js'; diff --git a/packages/core/test/evaluation/trace-envelope.test.ts b/packages/core/test/evaluation/trace-envelope.test.ts index 30754c76f..e4beafb6a 100644 --- a/packages/core/test/evaluation/trace-envelope.test.ts +++ b/packages/core/test/evaluation/trace-envelope.test.ts @@ -83,22 +83,44 @@ function makeResult(overrides: Partial = {}): EvaluationResult describe('execution trace artifact v1', () => { it('validates and round-trips the explicit snake_case wire shape', () => { - const envelope = buildTraceEnvelopeFromEvaluationResult(makeResult(), { - evalPath: 'examples/showcase/trace-evaluation/evals/coding-agent-replay.eval.yaml', - runId: 'run-123', - experiment: 'execution-trace-v1', - now: () => new Date('2026-06-15T12:00:05.000Z'), - source: { + const envelope = buildTraceEnvelopeFromEvaluationResult( + makeResult({ metadata: { - source_provider: 'replay', - providerCamelKey: 'kept', + external_trace: { + provider: 'phoenix', + source: 'codex', + endpoint: 'https://phoenix.example/v1/traces?api_key=secret', + project: 'agentv-dogfood', + session_id: 'codex-session-123', + traceparent: '00-11111111111111111111111111111111-2222222222222222-01', + tracestate: 'congo=t61rcWkgMzE,api_key=secret,rojo=00f067aa0ba902b7', + ui_url: 'https://phoenix.example/projects/agentv-dogfood/traces/trace-123?token=secret', + api_key: 'secret', + }, + external_trace_token: 'secret-flat-token', + }, + }), + { + evalPath: 'examples/showcase/trace-evaluation/evals/coding-agent-replay.eval.yaml', + runId: 'run-123', + experiment: 'execution-trace-v1', + now: () => new Date('2026-06-15T12:00:05.000Z'), + source: { + metadata: { + source_provider: 'replay', + providerCamelKey: 'kept', + external_trace: { + api_key: 'source-secret', + ui_url: 'https://phoenix.example/source?api_key=source-secret', + }, + }, + }, + artifacts: { + trace_path: 'outputs/trace.json', + transcript_path: 'outputs/transcript.jsonl', }, }, - artifacts: { - trace_path: 'outputs/trace.json', - transcript_path: 'outputs/transcript.jsonl', - }, - }); + ); const wire = toTraceEnvelopeWire(envelope); @@ -123,6 +145,20 @@ describe('execution trace artifact v1', () => { }); expect(wire.created_at).toBe('2026-06-15T12:00:05.000Z'); expect(wire.eval.eval_path).toContain('coding-agent-replay.eval.yaml'); + expect(wire.external_trace).toEqual({ + provider: 'phoenix', + source: 'codex', + endpoint: 'https://phoenix.example/', + project: 'agentv-dogfood', + session_id: 'codex-session-123', + trace_id: '11111111111111111111111111111111', + traceparent: '00-11111111111111111111111111111111-2222222222222222-01', + tracestate: 'congo=t61rcWkgMzE,rojo=00f067aa0ba902b7', + ui_url: 'https://phoenix.example/projects/agentv-dogfood/traces/trace-123', + run_id: 'run-123', + test_id: 'trace-case', + target: 'replay_coding_agent', + }); expect(wire.trace.format).toBe('otlp_openinference_spans'); expect(wire.artifacts?.trace_path).toBe('outputs/trace.json'); expect(wire.artifacts).not.toHaveProperty('execution_trace_path'); @@ -132,6 +168,9 @@ describe('execution trace artifact v1', () => { source_provider: 'replay', providerCamelKey: 'kept', }); + expect(wire.source.metadata).not.toHaveProperty('external_trace'); + expect(JSON.stringify(wire)).not.toContain('secret'); + expect(JSON.stringify(wire)).not.toContain('api_key'); expect(wire.source.metadata).not.toHaveProperty('sourceProvider'); expect(wire.source.metadata).not.toHaveProperty('provider_camel_key'); expect(TraceEnvelopeWireSchema.parse(wire).trace.spans.length).toBeGreaterThanOrEqual(2);