Skip to content
This repository was archived by the owner on Oct 16, 2026. It is now read-only.

feat(core): normalize trace artifacts for dashboard #1474

Merged
christso merged 1 commit into from
Jun 22, 2026
Merged

feat(core): normalize trace artifacts for dashboard #1474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 12 additions & 17 deletions apps/cli/src/commands/results/serve.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@ import {
getProject,
loadConfig,
loadProjectRegistry,
normalizeTraceArtifactToTraceSessionResponse,
readGitResultArtifact,
removeProject,
syncProjects,
toExternalTraceMetadataWire,
touchProject,
traceEnvelopeToTraceSessionResponse,
} from '@agentv/core';
import type { Context } from 'hono';
import { Hono } from 'hono';
Expand Down Expand Up @@ -661,15 +661,6 @@ function parseTraceArtifactJson(content: string): { value?: unknown; message?: s
}
}

function supportedTraceSessionEnvelope(value: unknown): value is Record<string, unknown> {
if (!isRecord(value)) {
return false;
}
const schemaVersion = nonEmptyString(value.schema_version);
const trace = isRecord(value.trace) ? value.trace : undefined;
return schemaVersion === 'agentv.trace.v1' && Array.isArray(trace?.spans);
}

function stripHeavyFields(results: readonly EvaluationResult[]) {
return results.map((r) => {
const { requests, trace, ...rest } = r as EvaluationResult & Record<string, unknown>;
Expand Down Expand Up @@ -1468,13 +1459,20 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex
);
}

if (!supportedTraceSessionEnvelope(parsed.value)) {
const normalized = normalizeTraceArtifactToTraceSessionResponse(parsed.value, {
runId: filename,
testId: record.test_id,
suite: record.suite,
target: record.target,
artifactPath: trace.path,
});

if (normalized.status === 'unsupported') {
return c.json(
traceSessionArtifactResponse({
status: 'unsupported',
trace_path: trace.path,
message:
'Trace artifact is not an agentv.trace.v1 envelope with trace.spans. Normalization is handled by the trace normalization pipeline.',
message: normalized.message,
...(trace.description && { pointer: trace.description }),
}),
);
Expand All @@ -1484,10 +1482,7 @@ async function handleEvalTraceSession(c: C, { searchDir, projectId }: DataContex
traceSessionArtifactResponse({
status: 'ok',
trace_path: trace.path,
trace_session: traceEnvelopeToTraceSessionResponse(parsed.value, {
runId: filename,
artifactPath: trace.path,
}),
trace_session: normalized.traceSession,
...(trace.description && { pointer: trace.description }),
}),
);
Expand Down
65 changes: 59 additions & 6 deletions apps/cli/test/commands/results/serve.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3332,14 +3332,48 @@ describe('serve app', () => {
expect(data.message).toContain('not available');
});

it('returns unsupported for trace artifacts that still need normalization', async () => {
it('normalizes supported OTLP trace artifacts instead of returning unsupported', async () => {
const traceArtifactPath = 'demo/test-greeting/outputs/trace.json';
const runId = writeLocalTraceRun(
tempDir,
'unsupported-trace',
'otlp-trace',
'2026-03-25T10-50-00-000Z',
traceArtifactPath,
`${JSON.stringify({ resourceSpans: [] })}\n`,
`${JSON.stringify({
resourceSpans: [
{
resource: {
attributes: [{ key: 'service.name', value: { stringValue: 'agentv' } }],
},
scopeSpans: [
{
spans: [
{
traceId: 'trace-otlp',
spanId: 'root',
name: 'invoke_agent codex',
startTimeUnixNano: '1000000000',
endTimeUnixNano: '2000000000',
attributes: [
{
key: 'gen_ai.operation.name',
value: { stringValue: 'invoke_agent' },
},
],
events: [
{
name: 'agentv.score',
timeUnixNano: '1900000000',
attributes: [{ key: 'agentv.grader.score', value: { doubleValue: 0.9 } }],
},
],
},
],
},
],
},
],
})}\n`,
);

const app = createApp([], tempDir, tempDir, undefined, { studioDir });
Expand All @@ -3351,11 +3385,30 @@ describe('serve app', () => {
const data = (await res.json()) as {
status: string;
trace_path: string;
message: string;
trace_session: {
test_id?: string;
target?: string;
resource_attributes?: Record<string, unknown>;
artifact_links?: Array<{ name: string; path: string }>;
spans: Array<{ span_id: string; name: string }>;
events: Array<{ kind: string; score?: number }>;
};
};
expect(data.status).toBe('unsupported');
expect(data.status).toBe('ok');
expect(data.trace_path).toBe(traceArtifactPath);
expect(data.message).toContain('trace normalization pipeline');
expect(data.trace_session).toMatchObject({
test_id: 'test-greeting',
target: 'gpt-4o',
resource_attributes: { 'service.name': 'agentv' },
});
expect(data.trace_session.artifact_links).toEqual([
{ name: 'raw_trace_path', path: traceArtifactPath },
]);
expect(data.trace_session.spans[0]).toMatchObject({
span_id: 'root',
name: 'invoke_agent codex',
});
expect(data.trace_session.events[0]).toMatchObject({ kind: 'score', score: 0.9 });
});

it('rejects trace artifact paths that escape the run workspace', async () => {
Expand Down
28 changes: 23 additions & 5 deletions packages/core/src/evaluation/dashboard-trace-read-model.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export interface TraceSessionSpan {
end_time?: string;
duration_ms?: number;
token_usage?: TraceSessionTokenUsage;
resource_attributes?: Record<string, unknown>;
attributes?: Record<string, unknown>;
events?: TraceSessionEvent[];
}
Expand Down Expand Up @@ -116,6 +117,7 @@ export interface TraceSessionResponse {
source?: TraceSessionSource;
external_trace?: ExternalTraceMetadataWire;
artifact_links?: TraceSessionArtifactLink[];
resource_attributes?: Record<string, unknown>;
conversion_warnings?: TraceSessionConversionWarning[];
spans: TraceSessionSpan[];
events: TraceSessionEvent[];
Expand Down Expand Up @@ -337,9 +339,13 @@ function eventKind(
const lowerName = name.toLowerCase();
if (
lowerName.includes('score') ||
finiteNumber(attributes?.score) !== undefined ||
finiteNumber(attributes?.['agentv.score']) !== undefined ||
finiteNumber(attributes?.['agentv.grader.score']) !== undefined
numberFromAttributes(attributes ?? {}, [
'score',
'agentv.score',
'agentv.grader.score',
'gen_ai.evaluation.score.value',
'openinference.evaluation.score',
]) !== undefined
) {
return 'score';
}
Expand All @@ -364,7 +370,9 @@ function scoreFromEvent(attributes: Record<string, unknown> | undefined): number
return (
finiteNumber(attributes.score) ??
finiteNumber(attributes['agentv.score']) ??
finiteNumber(attributes['agentv.grader.score'])
finiteNumber(attributes['agentv.grader.score']) ??
finiteNumber(attributes['gen_ai.evaluation.score.value']) ??
finiteNumber(attributes['openinference.evaluation.score'])
);
}

Expand All @@ -376,6 +384,8 @@ function textFromEvent(attributes: Record<string, unknown> | undefined): string
stringValue(attributes.text) ??
stringValue(attributes.annotation) ??
stringValue(attributes['agentv.annotation.text']) ??
stringValue(attributes['gen_ai.evaluation.explanation']) ??
stringValue(attributes['gen_ai.evaluation.score.label']) ??
stringValue(attributes['exception.message'])
);
}
Expand All @@ -384,7 +394,12 @@ function passedFromEvent(attributes: Record<string, unknown> | undefined): boole
if (!attributes) {
return undefined;
}
return boolValue(attributes.passed) ?? boolValue(attributes['agentv.annotation.passed']);
return (
boolValue(attributes.passed) ??
boolValue(attributes['agentv.annotation.passed']) ??
boolValue(attributes['agentv.grader.passed']) ??
boolValue(attributes['gen_ai.evaluation.passed'])
);
}

function eventId(
Expand Down Expand Up @@ -460,6 +475,7 @@ function projectSpan(span: unknown, index: number): TraceSessionSpan | undefined
end_time: unixNanoToIso(endTimeUnixNano),
duration_ms: durationMsFromNanos(startTimeUnixNano, endTimeUnixNano),
token_usage: tokenUsageFromAttributes(attributes),
resource_attributes: sanitizeAttributeMap(asRecord(record.resource_attributes)),
attributes: safeAttributes,
events: events.length > 0 ? events : undefined,
});
Expand Down Expand Up @@ -770,6 +786,7 @@ export function traceEnvelopeToTraceSessionResponse(
const envelope = asRecord(input) ?? {};
const evaluation = asRecord(envelope.eval);
const trace = asRecord(envelope.trace);
const traceResource = asRecord(trace?.resource);
const spans = asArray(trace?.spans)
.map(projectSpan)
.filter((span): span is TraceSessionSpan => span !== undefined);
Expand All @@ -788,6 +805,7 @@ export function traceEnvelopeToTraceSessionResponse(
source: sourceFromEnvelope(asRecord(envelope.source), options.artifactPath),
external_trace: externalTraceFromEnvelope(envelope),
artifact_links: projectArtifactLinks(envelope.artifacts),
resource_attributes: sanitizeAttributeMap(asRecord(traceResource?.attributes)),
conversion_warnings: projectConversionWarnings(envelope.conversion_warnings),
spans,
events,
Expand Down
Loading
Loading