diff --git a/apps/cli/src/commands/results/serve.ts b/apps/cli/src/commands/results/serve.ts index 2c7dcc980..72674e8ac 100644 --- a/apps/cli/src/commands/results/serve.ts +++ b/apps/cli/src/commands/results/serve.ts @@ -63,12 +63,12 @@ import { getProject, loadConfig, loadProjectRegistry, + normalizeTraceArtifactToTraceSessionResponse, readGitResultArtifact, removeProject, syncProjects, toExternalTraceMetadataWire, touchProject, - traceEnvelopeToTraceSessionResponse, } from '@agentv/core'; import type { Context } from 'hono'; import { Hono } from 'hono'; @@ -661,15 +661,6 @@ function parseTraceArtifactJson(content: string): { value?: unknown; message?: s } } -function supportedTraceSessionEnvelope(value: unknown): value is Record { - if (!isRecord(value)) { - return false; - } - const schemaVersion = nonEmptyString(value.schema_version); - const trace = isRecord(value.trace) ? value.trace : undefined; - return schemaVersion === 'agentv.trace.v1' && Array.isArray(trace?.spans); -} - function stripHeavyFields(results: readonly EvaluationResult[]) { return results.map((r) => { const { requests, trace, ...rest } = r as EvaluationResult & Record; @@ -1468,13 +1459,20 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex ); } - if (!supportedTraceSessionEnvelope(parsed.value)) { + const normalized = normalizeTraceArtifactToTraceSessionResponse(parsed.value, { + runId: filename, + testId: record.test_id, + suite: record.suite, + target: record.target, + artifactPath: trace.path, + }); + + if (normalized.status === 'unsupported') { return c.json( traceSessionArtifactResponse({ status: 'unsupported', trace_path: trace.path, - message: - 'Trace artifact is not an agentv.trace.v1 envelope with trace.spans. Normalization is handled by the trace normalization pipeline.', + message: normalized.message, ...(trace.description && { pointer: trace.description }), }), ); @@ -1484,10 +1482,7 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex traceSessionArtifactResponse({ status: 'ok', trace_path: trace.path, - trace_session: traceEnvelopeToTraceSessionResponse(parsed.value, { - runId: filename, - artifactPath: trace.path, - }), + trace_session: normalized.traceSession, ...(trace.description && { pointer: trace.description }), }), ); diff --git a/apps/cli/test/commands/results/serve.test.ts b/apps/cli/test/commands/results/serve.test.ts index 1af8b1b8b..fc4c65940 100644 --- a/apps/cli/test/commands/results/serve.test.ts +++ b/apps/cli/test/commands/results/serve.test.ts @@ -3332,14 +3332,48 @@ describe('serve app', () => { expect(data.message).toContain('not available'); }); - it('returns unsupported for trace artifacts that still need normalization', async () => { + it('normalizes supported OTLP trace artifacts instead of returning unsupported', async () => { const traceArtifactPath = 'demo/test-greeting/outputs/trace.json'; const runId = writeLocalTraceRun( tempDir, - 'unsupported-trace', + 'otlp-trace', '2026-03-25T10-50-00-000Z', traceArtifactPath, - `${JSON.stringify({ resourceSpans: [] })}\n`, + `${JSON.stringify({ + resourceSpans: [ + { + resource: { + attributes: [{ key: 'service.name', value: { stringValue: 'agentv' } }], + }, + scopeSpans: [ + { + spans: [ + { + traceId: 'trace-otlp', + spanId: 'root', + name: 'invoke_agent codex', + startTimeUnixNano: '1000000000', + endTimeUnixNano: '2000000000', + attributes: [ + { + key: 'gen_ai.operation.name', + value: { stringValue: 'invoke_agent' }, + }, + ], + events: [ + { + name: 'agentv.score', + timeUnixNano: '1900000000', + attributes: [{ key: 'agentv.grader.score', value: { doubleValue: 0.9 } }], + }, + ], + }, + ], + }, + ], + }, + ], + })}\n`, ); const app = createApp([], tempDir, tempDir, undefined, { studioDir }); @@ -3351,11 +3385,30 @@ describe('serve app', () => { const data = (await res.json()) as { status: string; trace_path: string; - message: string; + trace_session: { + test_id?: string; + target?: string; + resource_attributes?: Record; + artifact_links?: Array<{ name: string; path: string }>; + spans: Array<{ span_id: string; name: string }>; + events: Array<{ kind: string; score?: number }>; + }; }; - expect(data.status).toBe('unsupported'); + expect(data.status).toBe('ok'); expect(data.trace_path).toBe(traceArtifactPath); - expect(data.message).toContain('trace normalization pipeline'); + expect(data.trace_session).toMatchObject({ + test_id: 'test-greeting', + target: 'gpt-4o', + resource_attributes: { 'service.name': 'agentv' }, + }); + expect(data.trace_session.artifact_links).toEqual([ + { name: 'raw_trace_path', path: traceArtifactPath }, + ]); + expect(data.trace_session.spans[0]).toMatchObject({ + span_id: 'root', + name: 'invoke_agent codex', + }); + expect(data.trace_session.events[0]).toMatchObject({ kind: 'score', score: 0.9 }); }); it('rejects trace artifact paths that escape the run workspace', async () => { diff --git a/packages/core/src/evaluation/dashboard-trace-read-model.ts b/packages/core/src/evaluation/dashboard-trace-read-model.ts index b6b2ae147..0bc44e921 100644 --- a/packages/core/src/evaluation/dashboard-trace-read-model.ts +++ b/packages/core/src/evaluation/dashboard-trace-read-model.ts @@ -52,6 +52,7 @@ export interface TraceSessionSpan { end_time?: string; duration_ms?: number; token_usage?: TraceSessionTokenUsage; + resource_attributes?: Record; attributes?: Record; events?: TraceSessionEvent[]; } @@ -116,6 +117,7 @@ export interface TraceSessionResponse { source?: TraceSessionSource; external_trace?: ExternalTraceMetadataWire; artifact_links?: TraceSessionArtifactLink[]; + resource_attributes?: Record; conversion_warnings?: TraceSessionConversionWarning[]; spans: TraceSessionSpan[]; events: TraceSessionEvent[]; @@ -337,9 +339,13 @@ function eventKind( const lowerName = name.toLowerCase(); if ( lowerName.includes('score') || - finiteNumber(attributes?.score) !== undefined || - finiteNumber(attributes?.['agentv.score']) !== undefined || - finiteNumber(attributes?.['agentv.grader.score']) !== undefined + numberFromAttributes(attributes ?? {}, [ + 'score', + 'agentv.score', + 'agentv.grader.score', + 'gen_ai.evaluation.score.value', + 'openinference.evaluation.score', + ]) !== undefined ) { return 'score'; } @@ -364,7 +370,9 @@ function scoreFromEvent(attributes: Record | undefined): number return ( finiteNumber(attributes.score) ?? finiteNumber(attributes['agentv.score']) ?? - finiteNumber(attributes['agentv.grader.score']) + finiteNumber(attributes['agentv.grader.score']) ?? + finiteNumber(attributes['gen_ai.evaluation.score.value']) ?? + finiteNumber(attributes['openinference.evaluation.score']) ); } @@ -376,6 +384,8 @@ function textFromEvent(attributes: Record | undefined): string stringValue(attributes.text) ?? stringValue(attributes.annotation) ?? stringValue(attributes['agentv.annotation.text']) ?? + stringValue(attributes['gen_ai.evaluation.explanation']) ?? + stringValue(attributes['gen_ai.evaluation.score.label']) ?? stringValue(attributes['exception.message']) ); } @@ -384,7 +394,12 @@ function passedFromEvent(attributes: Record | undefined): boole if (!attributes) { return undefined; } - return boolValue(attributes.passed) ?? boolValue(attributes['agentv.annotation.passed']); + return ( + boolValue(attributes.passed) ?? + boolValue(attributes['agentv.annotation.passed']) ?? + boolValue(attributes['agentv.grader.passed']) ?? + boolValue(attributes['gen_ai.evaluation.passed']) + ); } function eventId( @@ -460,6 +475,7 @@ function projectSpan(span: unknown, index: number): TraceSessionSpan | undefined end_time: unixNanoToIso(endTimeUnixNano), duration_ms: durationMsFromNanos(startTimeUnixNano, endTimeUnixNano), token_usage: tokenUsageFromAttributes(attributes), + resource_attributes: sanitizeAttributeMap(asRecord(record.resource_attributes)), attributes: safeAttributes, events: events.length > 0 ? events : undefined, }); @@ -770,6 +786,7 @@ export function traceEnvelopeToTraceSessionResponse( const envelope = asRecord(input) ?? {}; const evaluation = asRecord(envelope.eval); const trace = asRecord(envelope.trace); + const traceResource = asRecord(trace?.resource); const spans = asArray(trace?.spans) .map(projectSpan) .filter((span): span is TraceSessionSpan => span !== undefined); @@ -788,6 +805,7 @@ export function traceEnvelopeToTraceSessionResponse( source: sourceFromEnvelope(asRecord(envelope.source), options.artifactPath), external_trace: externalTraceFromEnvelope(envelope), artifact_links: projectArtifactLinks(envelope.artifacts), + resource_attributes: sanitizeAttributeMap(asRecord(traceResource?.attributes)), conversion_warnings: projectConversionWarnings(envelope.conversion_warnings), spans, events, diff --git a/packages/core/src/evaluation/trace-normalization.ts b/packages/core/src/evaluation/trace-normalization.ts new file mode 100644 index 000000000..546152d23 --- /dev/null +++ b/packages/core/src/evaluation/trace-normalization.ts @@ -0,0 +1,1039 @@ +/** + * Dashboard trace normalization. + * + * This module accepts raw trace artifacts that are already owned by AgentV + * (`agentv.trace.v1`) or standards-shaped OTLP/OpenInference JSON, then projects + * them into the shared Dashboard trace/session read model. It deliberately + * avoids Phoenix-specific concepts; backend-specific mapping belongs in + * adapters before or after this generic normalization boundary. + */ + +import { + type TraceSessionConversionWarning, + type TraceSessionProjectionOptions, + type TraceSessionResponse, + traceEnvelopeToTraceSessionResponse, +} from './dashboard-trace-read-model.js'; +import { EXECUTION_TRACE_SCHEMA_VERSION } from './trace-envelope.js'; + +const TRACE_ENVELOPE_FORMAT = 'otlp_openinference_spans' as const; + +export interface TraceArtifactNormalizationOptions extends TraceSessionProjectionOptions { + runId?: string; + testId?: string; + suite?: string; + target?: string; + now?: () => Date; +} + +export type TraceArtifactNormalizationResult = + | { + status: 'ok'; + format: 'agentv_trace_v1' | 'otlp_json'; + traceSession: TraceSessionResponse; + warnings?: readonly TraceSessionConversionWarning[]; + } + | { + status: 'unsupported'; + message: string; + warnings?: readonly TraceSessionConversionWarning[]; + }; + +type WarningSink = TraceSessionConversionWarning[]; + +interface WarningContext { + readonly rawKind?: string; + readonly path?: string; + readonly spanId?: string; + readonly details?: Record; +} + +interface NormalizedOtlpSpan { + readonly trace_id?: string; + readonly span_id: string; + readonly parent_span_id?: string | null; + readonly name: string; + readonly kind?: string; + readonly start_time_unix_nano?: string; + readonly end_time_unix_nano?: string; + readonly status?: { + readonly code?: string; + readonly message?: string; + }; + readonly resource_attributes?: Record; + readonly attributes?: Record; + readonly events?: readonly { + readonly name: string; + readonly time_unix_nano?: string; + readonly attributes?: Record; + }[]; +} + +interface ParsedOtlpValue { + readonly ok: boolean; + readonly value?: unknown; +} + +const OTLP_ROOT_FIELDS = new Set(['resourceSpans']); +const OTLP_RESOURCE_SPAN_FIELDS = new Set(['resource', 'scopeSpans', 'schemaUrl']); +const OTLP_RESOURCE_FIELDS = new Set(['attributes', 'droppedAttributesCount']); +const OTLP_SCOPE_SPAN_FIELDS = new Set(['scope', 'spans', 'schemaUrl']); +const OTLP_SCOPE_FIELDS = new Set(['name', 'version', 'attributes', 'droppedAttributesCount']); +const OTLP_SPAN_FIELDS = new Set([ + 'traceId', + 'spanId', + 'parentSpanId', + 'traceState', + 'name', + 'kind', + 'startTimeUnixNano', + 'endTimeUnixNano', + 'attributes', + 'droppedAttributesCount', + 'events', + 'droppedEventsCount', + 'links', + 'droppedLinksCount', + 'status', +]); +const OTLP_EVENT_FIELDS = new Set(['timeUnixNano', 'name', 'attributes', 'droppedAttributesCount']); +const OTLP_STATUS_FIELDS = new Set(['code', 'message']); +const OTLP_ATTRIBUTE_FIELDS = new Set(['key', 'value']); +const OTLP_ANY_VALUE_FIELDS = new Set([ + 'stringValue', + 'boolValue', + 'intValue', + 'doubleValue', + 'arrayValue', + 'kvlistValue', + 'bytesValue', +]); + +export function normalizeTraceArtifactToTraceSessionResponse( + input: unknown, + options: TraceArtifactNormalizationOptions = {}, +): TraceArtifactNormalizationResult { + const artifact = asRecord(input); + if (!artifact) { + return { + status: 'unsupported', + message: 'Trace artifact is not a JSON object.', + }; + } + + if (stringValue(artifact.schema_version) === EXECUTION_TRACE_SCHEMA_VERSION) { + return normalizeAgentVTraceEnvelope(artifact, options); + } + + if (Object.hasOwn(artifact, 'resourceSpans')) { + return normalizeOtlpJson(artifact, options); + } + + return { + status: 'unsupported', + message: 'Trace artifact is not an agentv.trace.v1 envelope or OTLP JSON resourceSpans body.', + }; +} + +function normalizeAgentVTraceEnvelope( + artifact: Record, + options: TraceArtifactNormalizationOptions, +): TraceArtifactNormalizationResult { + const trace = asRecord(artifact.trace); + if (!Array.isArray(trace?.spans)) { + return { + status: 'unsupported', + message: 'Trace artifact is not an agentv.trace.v1 envelope with trace.spans.', + }; + } + + const traceSession = traceEnvelopeToTraceSessionResponse(artifact, options); + return { + status: 'ok', + format: 'agentv_trace_v1', + traceSession, + warnings: traceSession.conversion_warnings, + }; +} + +function normalizeOtlpJson( + artifact: Record, + options: TraceArtifactNormalizationOptions, +): TraceArtifactNormalizationResult { + const warnings: WarningSink = []; + warnUnknownFields(artifact, OTLP_ROOT_FIELDS, warnings, { + rawKind: 'otlp_json', + path: options.artifactPath, + }); + + const resourceSpans = Array.isArray(artifact.resourceSpans) ? artifact.resourceSpans : []; + if (!Array.isArray(artifact.resourceSpans)) { + addWarning(warnings, 'malformed_otlp_resource_spans', 'OTLP resourceSpans must be an array.', { + rawKind: 'otlp_json', + path: options.artifactPath, + }); + } + + const spans: NormalizedOtlpSpan[] = []; + const resourceAttributeSets: Record[] = []; + let scopeSpanCount = 0; + let firstScope: { name?: string; version?: string } | undefined; + + resourceSpans.forEach((resourceSpan, resourceSpanIndex) => { + const resourceRecord = asRecord(resourceSpan); + if (!resourceRecord) { + addWarning( + warnings, + 'malformed_otlp_resource_span', + 'OTLP resourceSpans entry must be an object and was skipped.', + { + rawKind: 'otlp_resource_span', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex }, + }, + ); + return; + } + + warnUnknownFields(resourceRecord, OTLP_RESOURCE_SPAN_FIELDS, warnings, { + rawKind: 'otlp_resource_span', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex }, + }); + + const resource = asRecord(resourceRecord.resource); + if (resource) { + warnUnknownFields(resource, OTLP_RESOURCE_FIELDS, warnings, { + rawKind: 'otlp_resource', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex }, + }); + } + const resourceAttributes = parseOtlpAttributes(resource?.attributes, warnings, { + rawKind: 'otlp_resource_attributes', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex }, + }); + if (resourceAttributes && Object.keys(resourceAttributes).length > 0) { + resourceAttributeSets.push(resourceAttributes); + } + + const scopeSpans = Array.isArray(resourceRecord.scopeSpans) ? resourceRecord.scopeSpans : []; + if (!Array.isArray(resourceRecord.scopeSpans)) { + addWarning( + warnings, + 'malformed_otlp_scope_spans', + 'OTLP resourceSpans.scopeSpans must be an array and was skipped.', + { + rawKind: 'otlp_resource_span', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex }, + }, + ); + } + + scopeSpans.forEach((scopeSpan, scopeSpanIndex) => { + const scopeSpanRecord = asRecord(scopeSpan); + if (!scopeSpanRecord) { + addWarning( + warnings, + 'malformed_otlp_scope_span', + 'OTLP scopeSpans entry must be an object and was skipped.', + { + rawKind: 'otlp_scope_span', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex, scope_span_index: scopeSpanIndex }, + }, + ); + return; + } + + scopeSpanCount += 1; + warnUnknownFields(scopeSpanRecord, OTLP_SCOPE_SPAN_FIELDS, warnings, { + rawKind: 'otlp_scope_span', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex, scope_span_index: scopeSpanIndex }, + }); + + const scope = normalizeOtlpScope(scopeSpanRecord.scope, warnings, { + rawKind: 'otlp_scope', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex, scope_span_index: scopeSpanIndex }, + }); + firstScope ??= scope; + + const otlpSpans = Array.isArray(scopeSpanRecord.spans) ? scopeSpanRecord.spans : []; + if (!Array.isArray(scopeSpanRecord.spans)) { + addWarning( + warnings, + 'malformed_otlp_spans', + 'OTLP scopeSpans.spans must be an array and was skipped.', + { + rawKind: 'otlp_scope_span', + path: options.artifactPath, + details: { resource_span_index: resourceSpanIndex, scope_span_index: scopeSpanIndex }, + }, + ); + } + + otlpSpans.forEach((span, spanIndex) => { + const normalized = normalizeOtlpSpan(span, warnings, { + resourceAttributes, + path: options.artifactPath, + resourceSpanIndex, + scopeSpanIndex, + spanIndex, + }); + if (normalized) { + spans.push(normalized); + } + }); + }); + }); + + if (spans.length === 0) { + addWarning(warnings, 'empty_otlp_trace', 'OTLP JSON did not contain any readable spans.', { + rawKind: 'otlp_json', + path: options.artifactPath, + }); + } + + const traceId = firstString(spans.map((span) => span.trace_id)); + const rootSpanId = firstRootSpanId(spans); + const resourceAttributes = singleDistinctRecord(resourceAttributeSets); + const createdAt = (options.now?.() ?? new Date()).toISOString(); + const artifactPath = options.artifactPath; + const envelope = dropUndefined({ + schema_version: EXECUTION_TRACE_SCHEMA_VERSION, + artifact_id: `otlp-trace-${traceId ?? rootSpanId ?? 'unknown'}`, + created_at: createdAt, + eval: dropUndefined({ + run_id: options.runId, + test_id: options.testId ?? traceId ?? 'otlp-trace', + suite: options.suite, + target: options.target ?? 'unknown', + }), + trace: dropUndefined({ + format: TRACE_ENVELOPE_FORMAT, + trace_id: traceId, + root_span_id: rootSpanId, + resource: resourceAttributes ? { attributes: resourceAttributes } : undefined, + scope: firstScope, + spans, + }), + source: dropUndefined({ + kind: 'otlp', + path: artifactPath, + format: 'otlp_json', + version: '1', + metadata: dropUndefined({ + resource_spans_count: resourceSpans.length, + scope_spans_count: scopeSpanCount, + }), + }), + capture: { + content: 'metadata', + redaction_level: 'partial', + }, + conversion_warnings: warnings.length > 0 ? warnings : undefined, + artifacts: artifactPath ? { raw_trace_path: artifactPath } : undefined, + }); + + const traceSession = traceEnvelopeToTraceSessionResponse(envelope, options); + return { + status: 'ok', + format: 'otlp_json', + traceSession, + warnings: traceSession.conversion_warnings, + }; +} + +function normalizeOtlpScope( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): { name?: string; version?: string } | undefined { + if (value === undefined) { + return undefined; + } + const scope = asRecord(value); + if (!scope) { + addWarning(warnings, 'malformed_otlp_scope', 'OTLP scope must be an object and was ignored.', { + ...context, + rawKind: context.rawKind ?? 'otlp_scope', + }); + return undefined; + } + warnUnknownFields(scope, OTLP_SCOPE_FIELDS, warnings, context); + const normalized = dropUndefined({ + name: stringValue(scope.name), + version: stringValue(scope.version), + }); + return Object.keys(normalized).length > 0 ? normalized : undefined; +} + +function normalizeOtlpSpan( + value: unknown, + warnings: WarningSink, + context: { + readonly resourceAttributes?: Record; + readonly path?: string; + readonly resourceSpanIndex: number; + readonly scopeSpanIndex: number; + readonly spanIndex: number; + }, +): NormalizedOtlpSpan | undefined { + const span = asRecord(value); + const details = { + resource_span_index: context.resourceSpanIndex, + scope_span_index: context.scopeSpanIndex, + span_index: context.spanIndex, + }; + if (!span) { + addWarning(warnings, 'malformed_otlp_span', 'OTLP span must be an object and was skipped.', { + rawKind: 'otlp_span', + path: context.path, + details, + }); + return undefined; + } + + warnUnknownFields(span, OTLP_SPAN_FIELDS, warnings, { + rawKind: 'otlp_span', + path: context.path, + details, + }); + + const spanId = + stringValue(span.spanId) ?? + `missing-span-${context.resourceSpanIndex}-${context.scopeSpanIndex}-${context.spanIndex}`; + if (!stringValue(span.spanId)) { + addWarning( + warnings, + 'missing_span_id', + 'OTLP span was missing spanId; a stable ID was assigned.', + { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }, + ); + } + + const traceId = stringValue(span.traceId); + if (!traceId) { + addWarning(warnings, 'missing_trace_id', 'OTLP span was missing traceId.', { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }); + } + + const name = stringValue(span.name) ?? spanId; + if (!stringValue(span.name)) { + addWarning(warnings, 'missing_span_name', 'OTLP span was missing name; spanId was used.', { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }); + } + + const parentSpanId = + span.parentSpanId === undefined || span.parentSpanId === null || span.parentSpanId === '' + ? null + : stringValue(span.parentSpanId); + if ( + span.parentSpanId !== undefined && + span.parentSpanId !== null && + span.parentSpanId !== '' && + !parentSpanId + ) { + addWarning( + warnings, + 'malformed_parent_span_id', + 'OTLP span parentSpanId was not a string and was ignored.', + { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }, + ); + } + + const attributes = parseOtlpAttributes(span.attributes, warnings, { + rawKind: 'otlp_span_attributes', + path: context.path, + spanId, + details, + }); + const events = normalizeOtlpEvents(span.events, warnings, { + path: context.path, + spanId, + details, + }); + + return dropUndefined({ + trace_id: traceId, + span_id: spanId, + parent_span_id: parentSpanId, + name, + kind: normalizeOtlpSpanKind(span.kind, warnings, { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }), + start_time_unix_nano: normalizeUnixNano(span.startTimeUnixNano, 'startTimeUnixNano', warnings, { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }), + end_time_unix_nano: normalizeUnixNano(span.endTimeUnixNano, 'endTimeUnixNano', warnings, { + rawKind: 'otlp_span', + path: context.path, + spanId, + details, + }), + status: normalizeOtlpStatus(span.status, warnings, { + rawKind: 'otlp_span_status', + path: context.path, + spanId, + details, + }), + resource_attributes: context.resourceAttributes, + attributes, + events, + }); +} + +function normalizeOtlpEvents( + value: unknown, + warnings: WarningSink, + context: { + readonly path?: string; + readonly spanId: string; + readonly details: Record; + }, +): + | readonly { + readonly name: string; + readonly time_unix_nano?: string; + readonly attributes?: Record; + }[] + | undefined { + if (value === undefined) { + return undefined; + } + if (!Array.isArray(value)) { + addWarning(warnings, 'malformed_otlp_events', 'OTLP span events must be an array.', { + rawKind: 'otlp_span_events', + path: context.path, + spanId: context.spanId, + details: context.details, + }); + return undefined; + } + + const events = value.flatMap((event, eventIndex) => { + const eventRecord = asRecord(event); + const details = { ...context.details, event_index: eventIndex }; + if (!eventRecord) { + addWarning( + warnings, + 'malformed_otlp_event', + 'OTLP span event must be an object and was skipped.', + { + rawKind: 'otlp_span_event', + path: context.path, + spanId: context.spanId, + details, + }, + ); + return []; + } + + warnUnknownFields(eventRecord, OTLP_EVENT_FIELDS, warnings, { + rawKind: 'otlp_span_event', + path: context.path, + spanId: context.spanId, + details, + }); + + const name = stringValue(eventRecord.name) ?? `event-${eventIndex}`; + if (!stringValue(eventRecord.name)) { + addWarning( + warnings, + 'missing_event_name', + 'OTLP span event was missing name; a stable name was assigned.', + { + rawKind: 'otlp_span_event', + path: context.path, + spanId: context.spanId, + details, + }, + ); + } + + return [ + dropUndefined({ + name, + time_unix_nano: normalizeUnixNano(eventRecord.timeUnixNano, 'timeUnixNano', warnings, { + rawKind: 'otlp_span_event', + path: context.path, + spanId: context.spanId, + details, + }), + attributes: parseOtlpAttributes(eventRecord.attributes, warnings, { + rawKind: 'otlp_event_attributes', + path: context.path, + spanId: context.spanId, + details, + }), + }), + ]; + }); + + return events.length > 0 ? events : undefined; +} + +function parseOtlpAttributes( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): Record | undefined { + if (value === undefined) { + return undefined; + } + if (asRecord(value)) { + addWarning( + warnings, + 'nonstandard_otlp_attribute_map', + 'OTLP attributes were an object map; values were preserved as-is.', + context, + ); + return value as Record; + } + if (!Array.isArray(value)) { + addWarning(warnings, 'malformed_otlp_attributes', 'OTLP attributes must be an array.', context); + return undefined; + } + + const attributes: Record = {}; + value.forEach((attribute, attributeIndex) => { + const attributeRecord = asRecord(attribute); + const details = { ...(context.details ?? {}), attribute_index: attributeIndex }; + if (!attributeRecord) { + addWarning( + warnings, + 'malformed_otlp_attribute', + 'OTLP attribute must be an object and was skipped.', + { ...context, details }, + ); + return; + } + + warnUnknownFields(attributeRecord, OTLP_ATTRIBUTE_FIELDS, warnings, { + ...context, + rawKind: context.rawKind ?? 'otlp_attribute', + details, + }); + + const key = stringValue(attributeRecord.key); + if (!key) { + addWarning( + warnings, + 'malformed_otlp_attribute', + 'OTLP attribute was missing key and was skipped.', + { + ...context, + details, + }, + ); + return; + } + + if (Object.hasOwn(attributes, key)) { + addWarning( + warnings, + 'duplicate_otlp_attribute', + `OTLP attribute "${key}" appeared more than once; the last value was used.`, + { + ...context, + details: { ...details, key }, + }, + ); + } + + const parsed = parseOtlpAnyValue(attributeRecord.value, warnings, { + ...context, + rawKind: 'otlp_attribute_value', + details: { ...details, key }, + }); + if (parsed.ok) { + attributes[key] = parsed.value; + } + }); + + return Object.keys(attributes).length > 0 ? attributes : undefined; +} + +function parseOtlpAnyValue( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): ParsedOtlpValue { + if (isPrimitive(value)) { + addWarning( + warnings, + 'nonstandard_otlp_value', + 'OTLP attribute value was primitive; value was preserved as-is.', + context, + ); + return { ok: true, value }; + } + + const record = asRecord(value); + if (!record) { + addWarning( + warnings, + 'malformed_otlp_value', + 'OTLP attribute value must be an object.', + context, + ); + return { ok: false }; + } + + const presentFields = Object.keys(record).filter((key) => OTLP_ANY_VALUE_FIELDS.has(key)); + const unknownFields = Object.keys(record).filter((key) => !OTLP_ANY_VALUE_FIELDS.has(key)); + if (unknownFields.length > 0) { + addWarning( + warnings, + 'unknown_otlp_value_field', + `Unknown OTLP attribute value field(s) ignored: ${unknownFields.join(', ')}.`, + { ...context, details: { ...(context.details ?? {}), fields: unknownFields } }, + ); + } + if (presentFields.length > 1) { + addWarning( + warnings, + 'ambiguous_otlp_value', + `OTLP attribute value had multiple value fields; ${presentFields[0]} was used.`, + { ...context, details: { ...(context.details ?? {}), fields: presentFields } }, + ); + } + + const field = presentFields[0]; + if (!field) { + addWarning( + warnings, + 'malformed_otlp_value', + 'OTLP attribute value did not contain a recognized value field; raw value was preserved.', + context, + ); + return { ok: true, value }; + } + + switch (field) { + case 'stringValue': + case 'bytesValue': + return { ok: true, value: stringValue(record[field]) ?? String(record[field] ?? '') }; + case 'boolValue': + return { ok: true, value: Boolean(record.boolValue) }; + case 'intValue': + return { ok: true, value: integerOrString(record.intValue) }; + case 'doubleValue': + return { ok: true, value: numberOrString(record.doubleValue) }; + case 'arrayValue': + return parseOtlpArrayValue(record.arrayValue, warnings, context); + case 'kvlistValue': + return parseOtlpKeyValueList(record.kvlistValue, warnings, context); + default: + return { ok: true, value }; + } +} + +function parseOtlpArrayValue( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): ParsedOtlpValue { + const arrayValue = asRecord(value); + if (!arrayValue || !Array.isArray(arrayValue.values)) { + addWarning( + warnings, + 'malformed_otlp_array_value', + 'OTLP arrayValue.values must be an array.', + context, + ); + return { ok: false }; + } + return { + ok: true, + value: arrayValue.values.map( + (entry, index) => + parseOtlpAnyValue(entry, warnings, { + ...context, + details: { ...(context.details ?? {}), value_index: index }, + }).value, + ), + }; +} + +function parseOtlpKeyValueList( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): ParsedOtlpValue { + const kvlistValue = asRecord(value); + if (!kvlistValue || !Array.isArray(kvlistValue.values)) { + addWarning( + warnings, + 'malformed_otlp_kvlist_value', + 'OTLP kvlistValue.values must be an array.', + context, + ); + return { ok: false }; + } + return { + ok: true, + value: parseOtlpAttributes(kvlistValue.values, warnings, context) ?? {}, + }; +} + +function normalizeOtlpSpanKind( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value === 'string' && value.length > 0) { + return value.replace(/^SPAN_KIND_/, ''); + } + if (typeof value !== 'number' || !Number.isInteger(value)) { + addWarning( + warnings, + 'malformed_span_kind', + 'OTLP span kind was not a string or integer.', + context, + ); + return undefined; + } + if (value === 0) return 'INTERNAL'; + if (value === 1) return 'SERVER'; + if (value === 2) return 'CLIENT'; + if (value === 3) return 'PRODUCER'; + if (value === 4 || value === 5) return 'CONSUMER'; + addWarning( + warnings, + 'unknown_span_kind', + `Unknown OTLP span kind ${value} was preserved as a string.`, + context, + ); + return String(value); +} + +function normalizeOtlpStatus( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): { code?: string; message?: string } | undefined { + if (value === undefined) { + return undefined; + } + const status = asRecord(value); + if (!status) { + addWarning(warnings, 'malformed_otlp_status', 'OTLP span status must be an object.', context); + return undefined; + } + warnUnknownFields(status, OTLP_STATUS_FIELDS, warnings, context); + return dropUndefined({ + code: normalizeStatusCode(status.code, warnings, context), + message: stringValue(status.message), + }); +} + +function normalizeStatusCode( + value: unknown, + warnings: WarningSink, + context: WarningContext, +): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value === 'string' && value.length > 0) { + return value.replace(/^STATUS_CODE_/, ''); + } + if (typeof value === 'number' && Number.isInteger(value)) { + if (value === 0) return 'UNSET'; + if (value === 1) return 'OK'; + if (value === 2) return 'ERROR'; + addWarning( + warnings, + 'unknown_status_code', + `Unknown OTLP status code ${value} was preserved as a string.`, + context, + ); + return String(value); + } + addWarning( + warnings, + 'malformed_status_code', + 'OTLP status code was not a string or integer.', + context, + ); + return undefined; +} + +function normalizeUnixNano( + value: unknown, + field: string, + warnings: WarningSink, + context: WarningContext, +): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value === 'string' && value.length > 0) { + if (!/^\d+$/.test(value)) { + addWarning( + warnings, + 'malformed_unix_nano', + `OTLP ${field} was not an unsigned integer string.`, + context, + ); + } + return value; + } + if (typeof value === 'number' && Number.isFinite(value) && value >= 0) { + return String(Math.trunc(value)); + } + addWarning(warnings, 'malformed_unix_nano', `OTLP ${field} was not a string or number.`, context); + return undefined; +} + +function firstRootSpanId(spans: readonly NormalizedOtlpSpan[]): string | undefined { + const spanIds = new Set(spans.map((span) => span.span_id)); + return ( + spans.find( + (span) => + !span.parent_span_id || + (typeof span.parent_span_id === 'string' && !spanIds.has(span.parent_span_id)), + )?.span_id ?? spans[0]?.span_id + ); +} + +function singleDistinctRecord( + records: readonly Record[], +): Record | undefined { + if (records.length === 0) { + return undefined; + } + const unique = new Map(records.map((record) => [stableRecordKey(record), record])); + return unique.size === 1 ? [...unique.values()][0] : undefined; +} + +function stableRecordKey(record: Record): string { + return JSON.stringify( + Object.keys(record) + .sort() + .map((key) => [key, record[key]]), + ); +} + +function firstString(values: readonly (string | undefined)[]): string | undefined { + return values.find((value): value is string => typeof value === 'string' && value.length > 0); +} + +function integerOrString(value: unknown): number | string { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'string') { + const numberValue = Number(value); + return Number.isSafeInteger(numberValue) ? numberValue : value; + } + return String(value ?? ''); +} + +function numberOrString(value: unknown): number | string { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'string') { + const numberValue = Number(value); + return Number.isFinite(numberValue) ? numberValue : value; + } + return String(value ?? ''); +} + +function warnUnknownFields( + record: Record, + allowed: ReadonlySet, + warnings: WarningSink, + context: WarningContext, +): void { + const unknownFields = Object.keys(record).filter((key) => !allowed.has(key)); + for (const field of unknownFields) { + addWarning( + warnings, + 'unknown_otlp_field', + `Unknown OTLP ${context.rawKind ?? 'object'} field "${field}" was ignored.`, + { ...context, details: { ...(context.details ?? {}), field } }, + ); + } +} + +function addWarning( + warnings: WarningSink, + code: string, + message: string, + context: WarningContext = {}, +): void { + warnings.push( + dropUndefined({ + code, + severity: 'warning', + span_id: context.spanId, + source_ref: + context.path || context.rawKind + ? dropUndefined({ + path: context.path, + raw_kind: context.rawKind, + span_id: context.spanId, + }) + : undefined, + message, + details: context.details, + }) as TraceSessionConversionWarning, + ); +} + +function asRecord(value: unknown): Record | undefined { + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function isPrimitive(value: unknown): boolean { + return ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' || + value === null + ); +} + +function dropUndefined>(value: T): T { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 074d3505c..53f653b53 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,7 @@ export * from './evaluation/types.js'; export * from './evaluation/trace.js'; export * from './evaluation/trace-envelope.js'; export * from './evaluation/dashboard-trace-read-model.js'; +export * from './evaluation/trace-normalization.js'; export * from './evaluation/external-trace.js'; export * from './evaluation/projection-identity.js'; export * from './evaluation/replay-fixtures.js'; diff --git a/packages/core/test/evaluation/trace-normalization.test.ts b/packages/core/test/evaluation/trace-normalization.test.ts new file mode 100644 index 000000000..5f619a036 --- /dev/null +++ b/packages/core/test/evaluation/trace-normalization.test.ts @@ -0,0 +1,319 @@ +import { describe, expect, it } from 'bun:test'; + +import { buildTraceSpanTree } from '../../src/evaluation/dashboard-trace-read-model.js'; +import { normalizeTraceArtifactToTraceSessionResponse } from '../../src/evaluation/trace-normalization.js'; + +function otlpValue(value: unknown): Record { + if (typeof value === 'string') return { stringValue: value }; + if (typeof value === 'boolean') return { boolValue: value }; + if (typeof value === 'number') { + return Number.isInteger(value) ? { intValue: value } : { doubleValue: value }; + } + if (Array.isArray(value)) { + return { arrayValue: { values: value.map(otlpValue) } }; + } + return { stringValue: JSON.stringify(value) }; +} + +function attr(key: string, value: unknown): Record { + return { key, value: otlpValue(value) }; +} + +function fixedNow(): Date { + return new Date('2026-06-22T10:00:00.000Z'); +} + +describe('Dashboard trace artifact normalization', () => { + it('normalizes nested OTLP invoke_agent/chat/execute_tool spans into the trace-session read model', () => { + const otlp = { + resourceSpans: [ + { + resource: { + attributes: [attr('service.name', 'agentv'), attr('deployment.environment', 'test')], + }, + scopeSpans: [ + { + scope: { name: 'agentv', version: '1.0.0' }, + spans: [ + { + traceId: 'trace-1', + spanId: 'root', + name: 'invoke_agent codex', + kind: 0, + startTimeUnixNano: '1000000000', + endTimeUnixNano: '5000000000', + attributes: [ + attr('gen_ai.operation.name', 'invoke_agent'), + attr('openinference.span.kind', 'AGENT'), + ], + status: { code: 1 }, + events: [ + { + name: 'agentv.grader.match', + timeUnixNano: '4900000000', + attributes: [ + attr('agentv.grader.score', 0.87), + attr('agentv.annotation.text', 'Matched trajectory'), + attr('agentv.grader.passed', true), + ], + }, + ], + }, + { + traceId: 'trace-1', + spanId: 'chat', + parentSpanId: 'root', + name: 'chat gpt-4o', + kind: 0, + startTimeUnixNano: '2000000000', + endTimeUnixNano: '3000000000', + attributes: [ + attr('gen_ai.operation.name', 'chat'), + attr('gen_ai.usage.input_tokens', 12), + attr('gen_ai.usage.output_tokens', 5), + attr('openinference.span.kind', 'LLM'), + ], + status: { code: 1 }, + }, + { + traceId: 'trace-1', + spanId: 'tool', + parentSpanId: 'chat', + name: 'execute_tool Read', + kind: 0, + startTimeUnixNano: '3100000000', + endTimeUnixNano: '3300000000', + attributes: [ + attr('gen_ai.operation.name', 'execute_tool'), + attr('gen_ai.tool.name', 'Read'), + attr('openinference.span.kind', 'TOOL'), + ], + status: { code: 1 }, + }, + ], + }, + ], + }, + ], + }; + + const result = normalizeTraceArtifactToTraceSessionResponse(otlp, { + runId: 'run-1', + testId: 'test-1', + suite: 'demo', + target: 'codex', + artifactPath: 'demo/test-1/outputs/trace.otlp.json', + now: fixedNow, + }); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') throw new Error(result.message); + + const session = result.traceSession; + const tree = buildTraceSpanTree(session.spans); + expect(session).toMatchObject({ + schema_version: 'agentv.dashboard.trace_session.v1', + artifact_id: 'otlp-trace-trace-1', + created_at: '2026-06-22T10:00:00.000Z', + run_id: 'run-1', + test_id: 'test-1', + suite: 'demo', + target: 'codex', + trace_id: 'trace-1', + root_span_id: 'root', + resource_attributes: { + 'service.name': 'agentv', + 'deployment.environment': 'test', + }, + }); + expect(session.artifact_links).toEqual([ + { name: 'raw_trace_path', path: 'demo/test-1/outputs/trace.otlp.json' }, + ]); + expect(session.spans.find((span) => span.span_id === 'chat')?.token_usage).toEqual({ + input: 12, + output: 5, + }); + expect(session.spans[0]?.resource_attributes).toEqual({ + 'service.name': 'agentv', + 'deployment.environment': 'test', + }); + expect(session.events).toEqual([ + expect.objectContaining({ + span_id: 'root', + kind: 'score', + score: 0.87, + text: 'Matched trajectory', + passed: true, + }), + ]); + expect(tree).toHaveLength(1); + expect(tree[0]?.span.name).toBe('invoke_agent codex'); + expect(tree[0]?.children[0]?.span.name).toBe('chat gpt-4o'); + expect(tree[0]?.children[0]?.children[0]?.span.name).toBe('execute_tool Read'); + }); + + it('preserves multiple OTLP roots as multiple Dashboard tree roots', () => { + const result = normalizeTraceArtifactToTraceSessionResponse( + { + resourceSpans: [ + { + scopeSpans: [ + { + spans: [ + { + traceId: 'trace-2', + spanId: 'root-b', + name: 'invoke_agent b', + startTimeUnixNano: '2000000000', + }, + { + traceId: 'trace-2', + spanId: 'root-a', + name: 'invoke_agent a', + startTimeUnixNano: '1000000000', + }, + ], + }, + ], + }, + ], + }, + { testId: 'multi-root', target: 'codex', now: fixedNow }, + ); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') throw new Error(result.message); + + const tree = buildTraceSpanTree(result.traceSession.spans); + expect(result.traceSession.root_span_id).toBe('root-b'); + expect(tree.map((node) => node.spanId)).toEqual(['root-a', 'root-b']); + }); + + it('returns conversion warnings for malformed and unknown OTLP fields while keeping the raw link', () => { + const result = normalizeTraceArtifactToTraceSessionResponse( + { + resourceSpans: [ + { + ignoredResourceSpanField: true, + resource: { + ignoredResourceField: true, + attributes: [{ value: otlpValue('missing key') }, attr('service.name', 'agentv')], + }, + scopeSpans: [ + { + ignoredScopeSpanField: true, + scope: { name: 'agentv', ignoredScopeField: true }, + spans: [ + { + traceId: 'trace-warnings', + name: 'span without id', + startTimeUnixNano: 'not-a-nano', + mysterySpanField: true, + attributes: [ + { + key: 'custom.unknown', + value: { unknownValue: 'preserved' }, + }, + ], + status: { code: 99, ignoredStatusField: true }, + events: [{ attributes: 'not attributes' }], + }, + ], + }, + ], + }, + ], + ignoredTopLevelField: true, + }, + { + testId: 'warnings', + target: 'codex', + artifactPath: 'demo/warnings/outputs/trace.otlp.json', + now: fixedNow, + }, + ); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') throw new Error(result.message); + + const warningCodes = result.traceSession.conversion_warnings?.map((warning) => warning.code); + expect(warningCodes).toContain('unknown_otlp_field'); + expect(warningCodes).toContain('malformed_otlp_attribute'); + expect(warningCodes).toContain('unknown_otlp_value_field'); + expect(warningCodes).toContain('missing_span_id'); + expect(warningCodes).toContain('malformed_unix_nano'); + expect(warningCodes).toContain('unknown_status_code'); + expect(result.traceSession.artifact_links).toEqual([ + { name: 'raw_trace_path', path: 'demo/warnings/outputs/trace.otlp.json' }, + ]); + expect(result.traceSession.spans[0]).toMatchObject({ + span_id: 'missing-span-0-0-0', + name: 'span without id', + attributes: { + 'custom.unknown': { unknownValue: 'preserved' }, + }, + }); + }); + + it('normalizes AgentV trace sidecars through the same entry point', () => { + const result = normalizeTraceArtifactToTraceSessionResponse( + { + schema_version: 'agentv.trace.v1', + artifact_id: 'agentv-trace-1', + created_at: '2026-06-22T09:00:00.000Z', + eval: { + run_id: 'run-agentv', + test_id: 'agentv-case', + suite: 'demo', + target: 'codex', + }, + trace: { + format: 'otlp_openinference_spans', + trace_id: 'agentv-trace-id', + root_span_id: 'agentv-root', + resource: { + attributes: { 'service.name': 'agentv-sidecar' }, + }, + spans: [ + { + trace_id: 'agentv-trace-id', + span_id: 'agentv-root', + parent_span_id: null, + name: 'invoke_agent codex', + kind: 'INTERNAL', + start_time_unix_nano: '1000000000', + end_time_unix_nano: '2000000000', + status: { code: 'OK' }, + attributes: {}, + }, + ], + }, + source: { kind: 'agentv_run', format: 'agentv_result' }, + capture: { content: 'metadata', redaction_level: 'partial' }, + conversion_warnings: [ + { + code: 'source_warning', + severity: 'warning', + message: 'Preserved warning', + }, + ], + artifacts: { trace_path: 'outputs/trace.json' }, + }, + { runId: 'run-agentv', artifactPath: 'demo/agentv/outputs/trace.json' }, + ); + + expect(result.status).toBe('ok'); + if (result.status !== 'ok') throw new Error(result.message); + + expect(result.format).toBe('agentv_trace_v1'); + expect(result.traceSession.resource_attributes).toEqual({ + 'service.name': 'agentv-sidecar', + }); + expect(result.traceSession.conversion_warnings).toEqual([ + expect.objectContaining({ code: 'source_warning', message: 'Preserved warning' }), + ]); + expect(result.traceSession.artifact_links).toEqual([ + { name: 'trace_path', path: 'outputs/trace.json' }, + ]); + }); +});