From e4ba68557027040280a95d5b01afe211314024cf Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:16:31 -0700 Subject: [PATCH 01/10] Add LangSmith trace normalization --- PARITY.md | 9 + README.md | 12 + fixtures/langsmith/cleanup/expected.json | 14 + fixtures/langsmith/cleanup/input.jsonl | 4 + fixtures/langsmith/tool-call/expected.json | 10 + fixtures/langsmith/tool-call/input.jsonl | 3 + src/adapters/langsmith.ts | 667 +++++++++++++++++++++ src/index.ts | 2 + src/types.ts | 7 +- test/normalize.test.ts | 206 ++++++- 10 files changed, 932 insertions(+), 2 deletions(-) create mode 100644 fixtures/langsmith/cleanup/expected.json create mode 100644 fixtures/langsmith/cleanup/input.jsonl create mode 100644 fixtures/langsmith/tool-call/expected.json create mode 100644 fixtures/langsmith/tool-call/input.jsonl create mode 100644 src/adapters/langsmith.ts diff --git a/PARITY.md b/PARITY.md index f22585a..3fa8ec9 100644 --- a/PARITY.md +++ b/PARITY.md @@ -52,3 +52,12 @@ OpenHands message, action, observation, agent-error, and user-rejection event shapes were checked against the `dream-pipeline` OpenHands source. Both the array and `{items: [...]}` input forms produced exact production-equivalent records in the compatibility fixtures. + +## LangSmith adapter + +The LangSmith adapter is covered with synthetic runs matching the published +LangSmith run and Messages-view formats. Fixtures exercise LangChain +constructor messages, Vercel AI SDK content blocks, repeated history snapshots, +tool-result matching by ID and tool name, run ordering, metadata, malformed +JSONL cleanup, and missing-timestamp repair. No private LangSmith corpus or +reference implementation was provided for differential testing. diff --git a/README.md b/README.md index 0a74b8a..86a5bae 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ and is empty when the transcript required no recoverable cleanup. | --- | --- | --- | | `claude-code` | Native Claude Code JSONL | `claude-code` | | `codex` | Native Codex rollout JSONL | `codex` | +| `langsmith` | LangSmith run object, run array, `{ "runs": [...] }`, or run JSONL | `langsmith` | | `letta` | Native Letta transcript JSON | `letta` | | `openhands` | JSON event array or an events-API `{ "items": [...] }` envelope | `openhands` | @@ -116,6 +117,17 @@ ignores system and approval-control records. OpenHands inputs are serialized exports; when a native store uses individual event files, assembling the event array remains the caller's responsibility. +LangSmith inputs contain the runs for one trace. The output of +`langsmith trace export --project --full` can be passed +directly as one transcript per exported JSONL file. Runs are ordered by +`dotted_order` and `start_time`; nested `child_runs` are flattened. The adapter +decodes the LangChain/LangGraph, OpenAI Chat Completions and Responses, +Anthropic Messages, and Vercel AI SDK message envelopes documented by +LangSmith. Repeated message-history snapshots from successive LLM runs are +deduplicated, while tool runs are linked to the earlier model tool call by call +ID and then by tool name when an integration omits the ID. Fetching or exporting +runs from LangSmith remains the caller's responsibility. + ## Normalized records A trajectory is an ordered array containing: diff --git a/fixtures/langsmith/cleanup/expected.json b/fixtures/langsmith/cleanup/expected.json new file mode 100644 index 0000000..843ae7f --- /dev/null +++ b/fixtures/langsmith/cleanup/expected.json @@ -0,0 +1,14 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "cwd": "/workspace", "git_branch": "feature/weather", "model": "claude-sonnet-4" }, + { "role": "user", "content": "Check Paris weather.", "timestamp": "2026-07-10T13:00:00.000Z" }, + { "role": "assistant", "content": "I'll check.", "timestamp": "2026-07-10T13:00:01.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "call-vercel", "name": "weather", "args": "{\"city\":\"Paris\"}" }], "timestamp": "2026-07-10T13:00:01.000Z" }, + { "role": "tool", "tool_call_id": "call-vercel", "content": "{\"condition\":\"sunny\"}", "timestamp": "2026-07-10T13:00:03.000Z" }, + { "role": "assistant", "content": "It is sunny in Paris.", "timestamp": "2026-07-10T13:00:04.000Z" } + ], + "diagnostics": [ + { "code": "invalid_json_line", "message": "Skipped invalid JSON on line 1.", "inputLine": 1 }, + { "code": "timestamps_interpolated", "message": "Interpolated timestamps for 1 normalized records.", "count": 1 } + ] +} diff --git a/fixtures/langsmith/cleanup/input.jsonl b/fixtures/langsmith/cleanup/input.jsonl new file mode 100644 index 0000000..2bc39f9 --- /dev/null +++ b/fixtures/langsmith/cleanup/input.jsonl @@ -0,0 +1,4 @@ +not-json +{"run_id":"vercel-1","trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","start_time":"2026-07-10T13:00:00Z","end_time":"2026-07-10T13:00:01Z","custom_metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4","cwd":"/workspace","git_branch":"feature/weather"},"inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]}} +{"run_id":"vercel-tool","trace_id":"trace-2","parent_run_id":"vercel-1","run_type":"tool","name":"weather","start_time":"2026-07-10T13:00:02Z","end_time":"2026-07-10T13:00:03Z","inputs":{"args":{"city":"Paris"}},"outputs":{"result":{"condition":"sunny"}}} +{"run_id":"vercel-2","trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","custom_metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4"},"inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]},{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]},{"role":"tool","toolCallId":"call-vercel","content":{"condition":"sunny"}}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"It is sunny in Paris."}]}} diff --git a/fixtures/langsmith/tool-call/expected.json b/fixtures/langsmith/tool-call/expected.json new file mode 100644 index 0000000..b50ad68 --- /dev/null +++ b/fixtures/langsmith/tool-call/expected.json @@ -0,0 +1,10 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "model": "gpt-4o" }, + { "role": "user", "content": "What is the weather?", "timestamp": "2026-07-10T12:00:00.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "call-weather", "name": "get_weather", "args": "{\"city\":\"Paris\"}" }], "timestamp": "2026-07-10T12:00:01.000Z" }, + { "role": "tool", "tool_call_id": "call-weather", "content": "Sunny, 22C", "timestamp": "2026-07-10T12:00:03.000Z" }, + { "role": "assistant", "content": "It is sunny and 22C in Paris.", "timestamp": "2026-07-10T12:00:05.000Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/tool-call/input.jsonl b/fixtures/langsmith/tool-call/input.jsonl new file mode 100644 index 0000000..d5f0281 --- /dev/null +++ b/fixtures/langsmith/tool-call/input.jsonl @@ -0,0 +1,3 @@ +{"id":"llm-2","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:04Z","end_time":"2026-07-10T12:00:05Z","dotted_order":"20260710T120004000000Zllm-2","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"It is sunny and 22C in Paris.","id":"ai-2"}}}]]}} +{"id":"llm-1","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:00Z","end_time":"2026-07-10T12:00:01Z","dotted_order":"20260710T120000000000Zllm-1","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}}}]]}} +{"id":"tool-run-1","trace_id":"trace-1","parent_run_id":"llm-1","run_type":"tool","name":"get_weather","start_time":"2026-07-10T12:00:02Z","end_time":"2026-07-10T12:00:03Z","dotted_order":"20260710T120002000000Ztool-run-1","inputs":{"city":"Paris"},"outputs":{"output":{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}}} diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts new file mode 100644 index 0000000..84f646e --- /dev/null +++ b/src/adapters/langsmith.ts @@ -0,0 +1,667 @@ +import type { + DecodedEvent, + DecodedSession, + SourceAdapter, +} from "../internal.js"; +import type { Diagnostic } from "../types.js"; +import { NormalizationError } from "../types.js"; +import { + blocksText, + isObject, + jsonString, + parseTimestamp, +} from "./shared.js"; + +interface OrderedRun { + run: Record; + index: number; +} + +interface ConversationItem { + key: string; + stable?: boolean; + event?: DecodedEvent; +} + +interface PendingCall { + id?: string; + name?: string; + consumed: boolean; +} + +interface DecodeState { + events: DecodedEvent[]; + history: string[]; + pendingCalls: PendingCall[]; +} + +export const langSmithAdapter: SourceAdapter = { + source: "langsmith", + + decode(transcript: string): DecodedSession { + const diagnostics: Diagnostic[] = []; + const runs = parseRuns(transcript, diagnostics).sort(compareRuns); + const state: DecodeState = { + events: [], + history: [], + pendingCalls: [], + }; + let model: string | undefined; + let cwd: string | undefined; + let gitBranch: string | undefined; + let createdAt: Date | undefined; + + for (const { run } of runs) { + const metadata = runMetadata(run); + model ??= firstString(metadata.ls_model_name, metadata.model); + cwd ??= firstString(metadata.cwd, metadata.working_directory); + gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); + const start = parseTimestamp(run.start_time); + const end = parseTimestamp(run.end_time) ?? start; + createdAt ??= start; + + if (run.run_type === "llm") { + const runModel = firstString( + metadata.ls_model_name, + metadata.model, + model, + ); + mergeItems( + state, + decodeMessages(inputMessages(run.inputs), start, runModel), + ); + mergeItems( + state, + decodeMessages(outputMessages(run.outputs), end, runModel, "assistant"), + ); + continue; + } + + if (run.run_type === "tool") { + const result = decodeToolRun(run, end, state.pendingCalls); + if (result) mergeItems(state, [result]); + } + } + + return { + events: state.events, + context: { + source: "langsmith", + ...(cwd ? { cwd } : {}), + ...(gitBranch ? { gitBranch } : {}), + ...(model ? { model } : {}), + ...(createdAt ? { createdAt } : {}), + }, + diagnostics, + }; + }, +}; + +function parseRuns( + transcript: string, + diagnostics: Diagnostic[], +): OrderedRun[] { + let parsed: unknown; + try { + parsed = JSON.parse(transcript); + } catch { + return parseRunJsonLines(transcript, diagnostics); + } + + const roots = runContainer(parsed); + if (!roots) throw invalidTranscript(); + const flattened: Record[] = []; + for (const root of roots) flattenRun(root, flattened); + if (flattened.length === 0) throw invalidTranscript(); + return flattened.map((run, index) => ({ run, index })); +} + +function parseRunJsonLines( + transcript: string, + diagnostics: Diagnostic[], +): OrderedRun[] { + const flattened: Record[] = []; + for (const [index, raw] of transcript.split("\n").entries()) { + if (!raw.trim()) continue; + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + diagnostics.push({ + code: "invalid_json_line", + message: `Skipped invalid JSON on line ${index + 1}.`, + inputLine: index + 1, + }); + continue; + } + const roots = runContainer(value); + if (!roots) { + diagnostics.push({ + code: "non_object_json_line", + message: `Skipped a non-run JSON value on line ${index + 1}.`, + inputLine: index + 1, + }); + continue; + } + for (const root of roots) flattenRun(root, flattened); + } + if (flattened.length === 0) throw invalidTranscript(); + return flattened.map((run, index) => ({ run, index })); +} + +function runContainer(value: unknown): Record[] | undefined { + if (Array.isArray(value)) { + return value.every(isObject) ? value : undefined; + } + if (!isObject(value)) return undefined; + if (Array.isArray(value.runs)) { + return value.runs.every(isObject) ? value.runs : undefined; + } + if (typeof value.run_type === "string") return [value]; + return undefined; +} + +function flattenRun( + run: Record, + output: Record[], +): void { + output.push(run); + if (!Array.isArray(run.child_runs)) return; + for (const child of run.child_runs) { + if (isObject(child)) flattenRun(child, output); + } +} + +function invalidTranscript(): NormalizationError { + return new NormalizationError( + "invalid_input", + "LangSmith transcript must be a run object, a JSON run array, an object with a runs array, or JSONL containing runs.", + ); +} + +function compareRuns(left: OrderedRun, right: OrderedRun): number { + const leftOrder = left.run.dotted_order; + const rightOrder = right.run.dotted_order; + if (typeof leftOrder === "string" && typeof rightOrder === "string") { + const order = leftOrder.localeCompare(rightOrder); + if (order !== 0) return order; + } + const leftTime = parseTimestamp(left.run.start_time)?.getTime(); + const rightTime = parseTimestamp(right.run.start_time)?.getTime(); + if (leftTime !== undefined && rightTime !== undefined && leftTime !== rightTime) { + return leftTime - rightTime; + } + return left.index - right.index; +} + +function runMetadata(run: Record): Record { + const topLevel = isObject(run.metadata) ? run.metadata : {}; + const custom = isObject(run.custom_metadata) ? run.custom_metadata : {}; + const extra = isObject(run.extra) ? run.extra : {}; + const nested = isObject(extra.metadata) ? extra.metadata : {}; + return { ...topLevel, ...custom, ...nested }; +} + +function inputMessages(value: unknown): unknown[] { + if (!isObject(value)) return []; + const messages: unknown[] = []; + if (value.system !== undefined) { + messages.push({ role: "system", content: value.system }); + } + if (value.instructions !== undefined) { + messages.push({ role: "system", content: value.instructions }); + } + + const candidate = firstNonEmpty(value.messages, value.input, value.prompt); + if (candidate !== undefined) { + messages.push(...messageArray(candidate, "user")); + } else if (Array.isArray(value.prompts)) { + for (const prompt of value.prompts) { + if (typeof prompt === "string") messages.push({ role: "user", content: prompt }); + } + } + return messages; +} + +function outputMessages(value: unknown): unknown[] { + if (!isObject(value)) { + return typeof value === "string" ? [{ role: "assistant", content: value }] : []; + } + + if (Array.isArray(value.generations)) { + const firstBatch = Array.isArray(value.generations[0]) + ? value.generations[0] + : value.generations; + const messages: unknown[] = []; + for (const generation of firstBatch) { + if (!isObject(generation)) continue; + if (generation.message !== undefined) messages.push(generation.message); + else if (typeof generation.text === "string") { + messages.push({ role: "assistant", content: generation.text }); + } + } + if (messages.length > 0) return messages; + } + + if (Array.isArray(value.choices)) { + const messages = value.choices + .map((choice) => (isObject(choice) ? choice.message : undefined)) + .filter((message) => message !== undefined); + if (messages.length > 0) return messages; + } + + if (Array.isArray(value.messages)) return value.messages; + if (isObject(value.message)) return [value.message]; + if (Array.isArray(value.output)) return value.output; + if (isObject(value.output)) { + if (Array.isArray(value.output.messages)) return value.output.messages; + if (isObject(value.output.update) && Array.isArray(value.output.update.messages)) { + return value.output.update.messages; + } + } + if (value.role !== undefined || value.type === "message") return [value]; + if (typeof value.content === "string" || Array.isArray(value.content)) { + return [{ role: "assistant", content: value.content }]; + } + return []; +} + +function messageArray(value: unknown, fallbackRole: string): unknown[] { + if (typeof value === "string") return [{ role: fallbackRole, content: value }]; + if (isObject(value)) return [value]; + if (!Array.isArray(value)) return []; + if (value.length === 1 && Array.isArray(value[0])) return value[0]; + return value; +} + +function decodeMessages( + messages: unknown[], + timestamp?: Date, + model?: string, + fallbackRole?: string, +): ConversationItem[] { + const items: ConversationItem[] = []; + for (const message of messages) { + items.push(...decodeMessage(message, timestamp, model, fallbackRole)); + } + return items; +} + +function decodeMessage( + value: unknown, + timestamp?: Date, + model?: string, + fallbackRole?: string, +): ConversationItem[] { + if (typeof value === "string") { + return [messageItem(fallbackRole ?? "assistant", value, undefined, 0, timestamp, model)]; + } + if (!isObject(value)) return []; + + const constructorClass = constructorName(value); + const body = isObject(value.kwargs) ? value.kwargs : value; + const stableId = firstString(body.id, value.id); + const role = canonicalRole( + firstString(body.role, body.type, fallbackRole), + constructorClass, + ); + const type = firstString(body.type, value.type); + + if (type === "function_call") { + return [toolCallItem(body, stableId, 0, timestamp, model)]; + } + if (type === "function_call_output") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + if (type === "reasoning") { + const text = reasoningText(body); + return text ? [reasoningItem(text, stableId, 0, timestamp, model)] : []; + } + + if (role === "tool") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + + const items: ConversationItem[] = []; + const content = body.content; + if (Array.isArray(content)) { + for (let index = 0; index < content.length; index += 1) { + const block = content[index]; + if (typeof block === "string") { + items.push(messageItem(role, block, stableId, index, timestamp, model)); + } else if (isObject(block)) { + items.push(...decodeContentBlock(block, role, stableId, index, timestamp, model)); + } + } + } else { + const text = contentText(content); + if (text) items.push(messageItem(role, text, stableId, 0, timestamp, model)); + } + + if (role === "assistant" && Array.isArray(body.tool_calls)) { + for (let index = 0; index < body.tool_calls.length; index += 1) { + const call = body.tool_calls[index]; + if (isObject(call)) { + items.push(toolCallItem(call, stableId, index + items.length, timestamp, model)); + } + } + } + return items; +} + +function decodeContentBlock( + block: Record, + role: string, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem[] { + const type = block.type; + if (type === "tool_use" || type === "tool-call" || type === "function_call") { + return [toolCallItem(block, stableId, index, timestamp, model)]; + } + if ( + type === "tool_result" || + type === "tool-result" || + type === "function_call_output" + ) { + return [toolResultItem(block, stableId, index, timestamp)]; + } + if (type === "thinking" || type === "reasoning" || type === "redacted_thinking") { + const text = reasoningText(block); + return text ? [reasoningItem(text, stableId, index, timestamp, model)] : []; + } + if (type === "image" || type === "image_url") { + return [messageItem(role, "[image]", stableId, index, timestamp, model)]; + } + const text = contentText(block.text ?? block.content); + return text ? [messageItem(role, text, stableId, index, timestamp, model)] : []; +} + +function messageItem( + role: string, + content: string, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem { + const canonical = role === "assistant" ? "assistant" : role === "user" ? "user" : role; + const key = stableId + ? `message:${stableId}:${index}` + : `message:${canonical}:${jsonString(content)}`; + if (canonical !== "user" && canonical !== "assistant") return { key }; + return { + key, + ...(stableId ? { stable: true } : {}), + event: { + type: "message", + role: canonical, + content, + ...(timestamp ? { timestamp } : {}), + ...(model ? { model } : {}), + }, + }; +} + +function reasoningItem( + content: string, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem { + return { + key: stableId + ? `reasoning:${stableId}:${index}` + : `reasoning:${jsonString(content)}`, + ...(stableId ? { stable: true } : {}), + event: { + type: "reasoning", + content, + ...(timestamp ? { timestamp } : {}), + ...(model ? { model } : {}), + }, + }; +} + +function toolCallItem( + value: Record, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem { + const fn = isObject(value.function) ? value.function : {}; + const id = firstString(value.id, value.call_id, value.toolCallId); + const name = firstString(value.name, value.toolName, fn.name); + const rawArgs = firstDefined(value.arguments, value.args, value.input, fn.arguments); + const args = typeof rawArgs === "string" ? rawArgs : jsonString(rawArgs); + return { + key: id + ? `tool-call:${id}` + : `tool-call:${stableId ?? ""}:${index}:${name ?? ""}:${args}`, + ...(id || stableId ? { stable: true } : {}), + event: { + type: "tool_call", + args, + ...(id ? { id } : {}), + ...(name ? { name } : {}), + ...(timestamp ? { timestamp } : {}), + ...(model ? { model } : {}), + }, + }; +} + +function toolResultItem( + value: Record, + stableId: string | undefined, + index: number, + timestamp?: Date, +): ConversationItem { + const id = firstString( + value.tool_call_id, + value.tool_use_id, + value.call_id, + value.toolCallId, + ); + const raw = firstDefined(value.output, value.result, value.content); + const content = resultText(raw); + return { + key: id + ? `tool-result:${id}` + : `tool-result:${stableId ?? ""}:${index}:${content}`, + ...(id || stableId ? { stable: true } : {}), + event: { + type: "tool_result", + content, + ...(id ? { callId: id } : {}), + ...(timestamp ? { timestamp } : {}), + }, + }; +} + +function decodeToolRun( + run: Record, + timestamp: Date | undefined, + pendingCalls: PendingCall[], +): ConversationItem | undefined { + const inputs = isObject(run.inputs) ? run.inputs : {}; + const outputs = isObject(run.outputs) ? run.outputs : {}; + const embedded = embeddedToolMessage(outputs); + const explicitId = firstString( + inputs.toolCallId, + inputs.tool_call_id, + inputs.call_id, + outputs.toolCallId, + outputs.tool_call_id, + outputs.call_id, + embedded?.tool_call_id, + embedded?.tool_use_id, + embedded?.call_id, + ); + const name = firstString(inputs.toolName, inputs.tool_name, run.name); + const call = matchPendingCall(pendingCalls, explicitId, name); + const callId = explicitId ?? call?.id; + const rawResult = firstDefined( + embedded?.content, + embedded?.output, + embedded?.result, + outputs.result, + outputs.output, + outputs.content, + run.error, + ); + if (rawResult === undefined) return undefined; + let content = resultText(rawResult); + if (run.error && rawResult === run.error && !/^error/i.test(content)) { + content = `Error: ${content}`; + } + return { + key: callId + ? `tool-result:${callId}` + : `tool-run:${firstString(run.id, run.run_id) ?? "unknown"}`, + stable: true, + event: { + type: "tool_result", + content, + ...(callId ? { callId } : {}), + ...(timestamp ? { timestamp } : {}), + }, + }; +} + +function embeddedToolMessage( + outputs: Record, +): Record | undefined { + const output = isObject(outputs.output) ? outputs.output : undefined; + const candidate = output ?? (isObject(outputs.message) ? outputs.message : undefined); + if (!candidate) return undefined; + return isObject(candidate.kwargs) ? candidate.kwargs : candidate; +} + +function matchPendingCall( + pendingCalls: PendingCall[], + id: string | undefined, + name: string | undefined, +): PendingCall | undefined { + let match = id + ? pendingCalls.find((call) => !call.consumed && call.id === id) + : undefined; + match ??= name + ? pendingCalls.find((call) => !call.consumed && call.name === name) + : undefined; + const remaining = pendingCalls.filter((call) => !call.consumed); + match ??= remaining.length === 1 ? remaining[0] : undefined; + if (match) match.consumed = true; + return match; +} + +function mergeItems(state: DecodeState, items: ConversationItem[]): void { + if (items.length === 0) return; + let overlap = Math.min(state.history.length, items.length); + while (overlap > 0) { + let matches = true; + const historyStart = state.history.length - overlap; + for (let index = 0; index < overlap; index += 1) { + if (state.history[historyStart + index] !== items[index]?.key) { + matches = false; + break; + } + } + if (matches) break; + overlap -= 1; + } + + const repeatedSnapshot = + overlap === 0 && + items.some((item) => item.stable === true && state.history.includes(item.key)); + + for (let index = overlap; index < items.length; index += 1) { + const item = items[index]; + if (!item) continue; + if (repeatedSnapshot && state.history.includes(item.key)) continue; + state.history.push(item.key); + if (!item.event) continue; + state.events.push(item.event); + if (item.event.type === "tool_call") { + state.pendingCalls.push({ + ...(item.event.id ? { id: item.event.id } : {}), + ...(item.event.name ? { name: item.event.name } : {}), + consumed: false, + }); + } else if (item.event.type === "tool_result" && item.event.callId) { + matchPendingCall(state.pendingCalls, item.event.callId, undefined); + } + } +} + +function canonicalRole(value: string | undefined, constructorClass?: string): string { + if (constructorClass === "HumanMessage" || constructorClass === "ChatMessage") { + return "user"; + } + if (constructorClass === "AIMessage") return "assistant"; + if (constructorClass === "ToolMessage" || constructorClass === "FunctionMessage") { + return "tool"; + } + if (constructorClass === "SystemMessage") return "system"; + if (value === "human") return "user"; + if (value === "ai") return "assistant"; + if (value === "function") return "tool"; + return value ?? "assistant"; +} + +function constructorName(value: Record): string | undefined { + if (!Array.isArray(value.id)) return undefined; + const last = value.id[value.id.length - 1]; + return typeof last === "string" ? last : undefined; +} + +function contentText(value: unknown): string { + if (typeof value === "string") return value; + if (value === null || value === undefined) return ""; + if (Array.isArray(value)) return blocksText(value); + if (isObject(value) && typeof value.value === "string") return value.value; + return ""; +} + +function reasoningText(value: Record): string { + const direct = firstString(value.thinking, value.text, value.content); + if (direct) return direct; + if (Array.isArray(value.summary)) { + return value.summary + .map((item) => (isObject(item) ? contentText(item.text ?? item.content) : "")) + .filter(Boolean) + .join("\n"); + } + return ""; +} + +function resultText(value: unknown): string { + if (typeof value === "string") return value; + if (value === undefined || value === null) return ""; + if (Array.isArray(value)) return blocksText(value) || jsonString(value); + if (isObject(value)) { + if (isObject(value.kwargs)) return resultText(value.kwargs.content); + if (typeof value.content === "string") return value.content; + } + return jsonString(value); +} + +function firstString(...values: unknown[]): string | undefined { + return values.find((value): value is string => typeof value === "string" && value.length > 0); +} + +function firstDefined(...values: unknown[]): unknown { + return values.find((value) => value !== undefined && value !== null); +} + +function firstNonEmpty(...values: unknown[]): unknown { + return values.find( + (value) => + value !== undefined && + value !== null && + (!Array.isArray(value) || value.length > 0), + ); +} diff --git a/src/index.ts b/src/index.ts index 015ba93..411dade 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ import { claudeCodeAdapter } from "./adapters/claude-code.js"; import { codexAdapter } from "./adapters/codex.js"; +import { langSmithAdapter } from "./adapters/langsmith.js"; import { lettaAdapter } from "./adapters/letta.js"; import { openHandsAdapter } from "./adapters/openhands.js"; import { resolveBounds } from "./bounds.js"; @@ -15,6 +16,7 @@ import { NormalizationError } from "./types.js"; const ADAPTERS: Record = { "claude-code": claudeCodeAdapter, codex: codexAdapter, + langsmith: langSmithAdapter, letta: lettaAdapter, openhands: openHandsAdapter, }; diff --git a/src/types.ts b/src/types.ts index 07159b1..74fb824 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,4 +1,9 @@ -export type TrajectorySource = "claude-code" | "codex" | "letta" | "openhands"; +export type TrajectorySource = + | "claude-code" + | "codex" + | "langsmith" + | "letta" + | "openhands"; export interface ToolArgumentBounds { /** Maximum Unicode code points in the serialized arguments object. */ diff --git a/test/normalize.test.ts b/test/normalize.test.ts index cccefab..652153e 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -14,6 +14,8 @@ const fixtures = [ { source: "claude-code", name: "claude-code/cleanup" }, { source: "codex", name: "codex/tool-calls" }, { source: "codex", name: "codex/cleanup" }, + { source: "langsmith", name: "langsmith/tool-call" }, + { source: "langsmith", name: "langsmith/cleanup" }, { source: "letta", name: "letta/tool-call" }, { source: "letta", name: "letta/cleanup" }, { source: "openhands", name: "openhands/tool-calls" }, @@ -65,7 +67,7 @@ describe("public API", () => { test("rejects an unknown source", () => { expect(() => normalizeTranscript({ - source: "langsmith" as TrajectorySource, + source: "not-a-source" as TrajectorySource, transcript: "{}", }), ).toThrow( @@ -101,6 +103,208 @@ describe("public API", () => { ).toThrow(expect.objectContaining({ code: "invalid_input" })); }); + test("rejects an invalid LangSmith document shape", () => { + expect(() => + normalizeTranscript({ + source: "langsmith", + transcript: '{"runs":{}}', + }), + ).toThrow(expect.objectContaining({ code: "invalid_input" })); + }); + + test("accepts a single LangSmith run object", () => { + const transcript = JSON.stringify({ + id: "run-1", + run_type: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { + choices: [{ message: { role: "assistant", content: "hi" } }], + }, + }); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + }); + + test("flattens nested LangSmith runs envelopes", () => { + const transcript = JSON.stringify({ + runs: [ + { + id: "root", + run_type: "chain", + child_runs: [ + { + id: "llm", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { role: "assistant", content: "hi" }, + }, + ], + }, + ], + }); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + }); + + test("decodes Anthropic reasoning and tool blocks from LangSmith runs", () => { + const firstAssistant = { + id: "msg-1", + role: "assistant", + content: [ + { type: "thinking", thinking: "I should check." }, + { type: "text", text: "Checking." }, + { + type: "tool_use", + id: "toolu-1", + name: "weather", + input: { city: "Paris" }, + }, + ], + }; + const transcript = JSON.stringify([ + { + id: "llm-1", + run_type: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { messages: [{ role: "user", content: "weather?" }] }, + outputs: { message: firstAssistant }, + }, + { + id: "tool-1", + run_type: "tool", + name: "weather", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: { city: "Paris" }, + outputs: { output: { condition: "sunny" } }, + }, + { + id: "llm-2", + run_type: "llm", + start_time: "2026-07-10T00:00:04Z", + end_time: "2026-07-10T00:00:05Z", + inputs: { + messages: [ + { role: "user", content: "weather?" }, + firstAssistant, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu-1", content: "sunny" }, + ], + }, + ], + }, + outputs: { message: { role: "assistant", content: "It is sunny." } }, + }, + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "reasoning", + "assistant", + "assistant", + "tool", + "assistant", + ]); + expect(result.records.filter((record) => record.role === "tool")).toHaveLength(1); + }); + + test("decodes OpenAI Responses items from LangSmith runs", () => { + const transcript = JSON.stringify([ + { + id: "response-1", + run_type: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { + instructions: "Be helpful.", + input: [{ type: "message", role: "user", content: "weather?" }], + }, + outputs: { + output: [ + { + type: "reasoning", + id: "reasoning-1", + summary: [{ type: "summary_text", text: "Check weather." }], + }, + { + type: "function_call", + call_id: "call-1", + name: "weather", + arguments: '{"city":"Paris"}', + }, + ], + }, + }, + { + id: "tool-1", + run_type: "tool", + name: "weather", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: { call_id: "call-1" }, + outputs: { output: "sunny" }, + }, + { + id: "response-2", + run_type: "llm", + start_time: "2026-07-10T00:00:04Z", + end_time: "2026-07-10T00:00:05Z", + inputs: { + input: [ + { type: "message", role: "user", content: "weather?" }, + { + type: "function_call", + call_id: "call-1", + name: "weather", + arguments: '{"city":"Paris"}', + }, + { type: "function_call_output", call_id: "call-1", output: "sunny" }, + ], + }, + outputs: { + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "It is sunny." }], + }, + ], + }, + }, + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "reasoning", + "assistant", + "tool", + "assistant", + ]); + }); + test("rejects a non-flat Letta document shape", () => { expect(() => normalizeTranscript({ From 9da78de6bf5842e188f1b61f0bd3557e6cca07f7 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:21:42 -0700 Subject: [PATCH 02/10] Handle live LangSmith trace variants --- PARITY.md | 17 +++++++++++-- README.md | 8 +++++- src/adapters/langsmith.ts | 4 +++ test/normalize.test.ts | 51 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 3 deletions(-) diff --git a/PARITY.md b/PARITY.md index 3fa8ec9..cda24c2 100644 --- a/PARITY.md +++ b/PARITY.md @@ -59,5 +59,18 @@ The LangSmith adapter is covered with synthetic runs matching the published LangSmith run and Messages-view formats. Fixtures exercise LangChain constructor messages, Vercel AI SDK content blocks, repeated history snapshots, tool-result matching by ID and tool name, run ordering, metadata, malformed -JSONL cleanup, and missing-timestamp repair. No private LangSmith corpus or -reference implementation was provided for differential testing. +JSONL cleanup, and missing-timestamp repair. + +Read-only validation was also run against two user-provided LangSmith projects +without retaining raw traces or normalized content in the repository. The +sample comprised one two-trace thread with 17 total runs (13 chain, two LLM, +and two tool runs) and one standalone LLM run. This surfaced and fixed two +native variants not present in the synthetic fixtures: string-valued +`outputs.output` completions and a tool call repeated in both content blocks +and the message-level `tool_calls` field. + +After those repairs, the combined thread normalized to 16 records with native +tool linkage preserved; its only diagnostic was the expected configured +tool-result truncation. The standalone LLM run normalized to three records +without diagnostics. No reference implementation was provided for differential +testing. diff --git a/README.md b/README.md index 86a5bae..ce26aa4 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,8 @@ ignores system and approval-control records. OpenHands inputs are serialized exports; when a native store uses individual event files, assembling the event array remains the caller's responsibility. -LangSmith inputs contain the runs for one trace. The output of +LangSmith inputs contain the runs for one trace or one chronological thread. +The output of `langsmith trace export --project --full` can be passed directly as one transcript per exported JSONL file. Runs are ordered by `dotted_order` and `start_time`; nested `child_runs` are flattened. The adapter @@ -128,6 +129,11 @@ deduplicated, while tool runs are linked to the earlier model tool call by call ID and then by tool name when an integration omits the ID. Fetching or exporting runs from LangSmith remains the caller's responsibility. +For a multi-turn thread, combine the runs from each member trace into one input +container before normalization. This preserves history and tool linkage that +cross trace boundaries; normalizing each trace separately can correctly report +an initial tool result as orphaned when its call occurred in the prior trace. + ## Normalized records A trajectory is an ordered array containing: diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index 84f646e..20d3106 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -253,6 +253,9 @@ function outputMessages(value: unknown): unknown[] { if (Array.isArray(value.messages)) return value.messages; if (isObject(value.message)) return [value.message]; if (Array.isArray(value.output)) return value.output; + if (typeof value.output === "string") { + return [{ role: "assistant", content: value.output }]; + } if (isObject(value.output)) { if (Array.isArray(value.output.messages)) return value.output.messages; if (isObject(value.output.update) && Array.isArray(value.output.update.messages)) { @@ -581,6 +584,7 @@ function mergeItems(state: DecodeState, items: ConversationItem[]): void { for (let index = overlap; index < items.length; index += 1) { const item = items[index]; if (!item) continue; + if (item.stable === true && state.history.includes(item.key)) continue; if (repeatedSnapshot && state.history.includes(item.key)) continue; state.history.push(item.key); if (!item.event) continue; diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 652153e..9b4eb68 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -133,6 +133,57 @@ describe("public API", () => { ]); }); + test("accepts string output from a LangSmith LLM run", () => { + const transcript = JSON.stringify({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { output: "hi" }, + }); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records).toContainEqual( + expect.objectContaining({ role: "assistant", content: "hi" }), + ); + }); + + test("deduplicates a tool call repeated in native message fields", () => { + const call = { + id: "call-1", + name: "weather", + args: { city: "Paris" }, + }; + const transcript = JSON.stringify({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "weather?" }] }, + outputs: { + role: "assistant", + content: [ + { + type: "tool_use", + id: call.id, + name: call.name, + input: call.args, + }, + ], + tool_calls: [call], + }, + }); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect( + result.records.filter( + (record) => record.role === "assistant" && record.content === null, + ), + ).toHaveLength(1); + expect(result.diagnostics).not.toContainEqual( + expect.objectContaining({ code: "duplicate_tool_call_id" }), + ); + }); + test("flattens nested LangSmith runs envelopes", () => { const transcript = JSON.stringify({ runs: [ From 34a3c37e98202426848a5dadd6536b38f1397b22 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:22:01 -0700 Subject: [PATCH 03/10] Update Python CLI bundle --- .../src/trajectory/_vendor/trajectory-cli.mjs | 517 ++++++++++++++++++ 1 file changed, 517 insertions(+) diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index 44bf343..51c02f8 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -382,6 +382,522 @@ class NormalizationError extends Error { } } +// src/adapters/langsmith.ts +var langSmithAdapter = { + source: "langsmith", + decode(transcript) { + const diagnostics = []; + const runs = parseRuns(transcript, diagnostics).sort(compareRuns); + const state = { + events: [], + history: [], + pendingCalls: [] + }; + let model; + let cwd; + let gitBranch; + let createdAt; + for (const { run } of runs) { + const metadata = runMetadata(run); + model ??= firstString(metadata.ls_model_name, metadata.model); + cwd ??= firstString(metadata.cwd, metadata.working_directory); + gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); + const start = parseTimestamp(run.start_time); + const end = parseTimestamp(run.end_time) ?? start; + createdAt ??= start; + if (run.run_type === "llm") { + const runModel = firstString(metadata.ls_model_name, metadata.model, model); + mergeItems(state, decodeMessages(inputMessages(run.inputs), start, runModel)); + mergeItems(state, decodeMessages(outputMessages(run.outputs), end, runModel, "assistant")); + continue; + } + if (run.run_type === "tool") { + const result = decodeToolRun(run, end, state.pendingCalls); + if (result) + mergeItems(state, [result]); + } + } + return { + events: state.events, + context: { + source: "langsmith", + ...cwd ? { cwd } : {}, + ...gitBranch ? { gitBranch } : {}, + ...model ? { model } : {}, + ...createdAt ? { createdAt } : {} + }, + diagnostics + }; + } +}; +function parseRuns(transcript, diagnostics) { + let parsed; + try { + parsed = JSON.parse(transcript); + } catch { + return parseRunJsonLines(transcript, diagnostics); + } + const roots = runContainer(parsed); + if (!roots) + throw invalidTranscript(); + const flattened = []; + for (const root of roots) + flattenRun(root, flattened); + if (flattened.length === 0) + throw invalidTranscript(); + return flattened.map((run, index) => ({ run, index })); +} +function parseRunJsonLines(transcript, diagnostics) { + const flattened = []; + for (const [index, raw] of transcript.split(` +`).entries()) { + if (!raw.trim()) + continue; + let value; + try { + value = JSON.parse(raw); + } catch { + diagnostics.push({ + code: "invalid_json_line", + message: `Skipped invalid JSON on line ${index + 1}.`, + inputLine: index + 1 + }); + continue; + } + const roots = runContainer(value); + if (!roots) { + diagnostics.push({ + code: "non_object_json_line", + message: `Skipped a non-run JSON value on line ${index + 1}.`, + inputLine: index + 1 + }); + continue; + } + for (const root of roots) + flattenRun(root, flattened); + } + if (flattened.length === 0) + throw invalidTranscript(); + return flattened.map((run, index) => ({ run, index })); +} +function runContainer(value) { + if (Array.isArray(value)) { + return value.every(isObject) ? value : undefined; + } + if (!isObject(value)) + return; + if (Array.isArray(value.runs)) { + return value.runs.every(isObject) ? value.runs : undefined; + } + if (typeof value.run_type === "string") + return [value]; + return; +} +function flattenRun(run, output) { + output.push(run); + if (!Array.isArray(run.child_runs)) + return; + for (const child of run.child_runs) { + if (isObject(child)) + flattenRun(child, output); + } +} +function invalidTranscript() { + return new NormalizationError("invalid_input", "LangSmith transcript must be a run object, a JSON run array, an object with a runs array, or JSONL containing runs."); +} +function compareRuns(left, right) { + const leftOrder = left.run.dotted_order; + const rightOrder = right.run.dotted_order; + if (typeof leftOrder === "string" && typeof rightOrder === "string") { + const order = leftOrder.localeCompare(rightOrder); + if (order !== 0) + return order; + } + const leftTime = parseTimestamp(left.run.start_time)?.getTime(); + const rightTime = parseTimestamp(right.run.start_time)?.getTime(); + if (leftTime !== undefined && rightTime !== undefined && leftTime !== rightTime) { + return leftTime - rightTime; + } + return left.index - right.index; +} +function runMetadata(run) { + const topLevel = isObject(run.metadata) ? run.metadata : {}; + const custom = isObject(run.custom_metadata) ? run.custom_metadata : {}; + const extra = isObject(run.extra) ? run.extra : {}; + const nested = isObject(extra.metadata) ? extra.metadata : {}; + return { ...topLevel, ...custom, ...nested }; +} +function inputMessages(value) { + if (!isObject(value)) + return []; + const messages = []; + if (value.system !== undefined) { + messages.push({ role: "system", content: value.system }); + } + if (value.instructions !== undefined) { + messages.push({ role: "system", content: value.instructions }); + } + const candidate = firstNonEmpty(value.messages, value.input, value.prompt); + if (candidate !== undefined) { + messages.push(...messageArray(candidate, "user")); + } else if (Array.isArray(value.prompts)) { + for (const prompt of value.prompts) { + if (typeof prompt === "string") + messages.push({ role: "user", content: prompt }); + } + } + return messages; +} +function outputMessages(value) { + if (!isObject(value)) { + return typeof value === "string" ? [{ role: "assistant", content: value }] : []; + } + if (Array.isArray(value.generations)) { + const firstBatch = Array.isArray(value.generations[0]) ? value.generations[0] : value.generations; + const messages = []; + for (const generation of firstBatch) { + if (!isObject(generation)) + continue; + if (generation.message !== undefined) + messages.push(generation.message); + else if (typeof generation.text === "string") { + messages.push({ role: "assistant", content: generation.text }); + } + } + if (messages.length > 0) + return messages; + } + if (Array.isArray(value.choices)) { + const messages = value.choices.map((choice) => isObject(choice) ? choice.message : undefined).filter((message) => message !== undefined); + if (messages.length > 0) + return messages; + } + if (Array.isArray(value.messages)) + return value.messages; + if (isObject(value.message)) + return [value.message]; + if (Array.isArray(value.output)) + return value.output; + if (typeof value.output === "string") { + return [{ role: "assistant", content: value.output }]; + } + if (isObject(value.output)) { + if (Array.isArray(value.output.messages)) + return value.output.messages; + if (isObject(value.output.update) && Array.isArray(value.output.update.messages)) { + return value.output.update.messages; + } + } + if (value.role !== undefined || value.type === "message") + return [value]; + if (typeof value.content === "string" || Array.isArray(value.content)) { + return [{ role: "assistant", content: value.content }]; + } + return []; +} +function messageArray(value, fallbackRole) { + if (typeof value === "string") + return [{ role: fallbackRole, content: value }]; + if (isObject(value)) + return [value]; + if (!Array.isArray(value)) + return []; + if (value.length === 1 && Array.isArray(value[0])) + return value[0]; + return value; +} +function decodeMessages(messages, timestamp, model, fallbackRole) { + const items = []; + for (const message of messages) { + items.push(...decodeMessage(message, timestamp, model, fallbackRole)); + } + return items; +} +function decodeMessage(value, timestamp, model, fallbackRole) { + if (typeof value === "string") { + return [messageItem(fallbackRole ?? "assistant", value, undefined, 0, timestamp, model)]; + } + if (!isObject(value)) + return []; + const constructorClass = constructorName(value); + const body = isObject(value.kwargs) ? value.kwargs : value; + const stableId = firstString(body.id, value.id); + const role = canonicalRole(firstString(body.role, body.type, fallbackRole), constructorClass); + const type = firstString(body.type, value.type); + if (type === "function_call") { + return [toolCallItem(body, stableId, 0, timestamp, model)]; + } + if (type === "function_call_output") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + if (type === "reasoning") { + const text = reasoningText(body); + return text ? [reasoningItem(text, stableId, 0, timestamp, model)] : []; + } + if (role === "tool") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + const items = []; + const content = body.content; + if (Array.isArray(content)) { + for (let index = 0;index < content.length; index += 1) { + const block = content[index]; + if (typeof block === "string") { + items.push(messageItem(role, block, stableId, index, timestamp, model)); + } else if (isObject(block)) { + items.push(...decodeContentBlock(block, role, stableId, index, timestamp, model)); + } + } + } else { + const text = contentText(content); + if (text) + items.push(messageItem(role, text, stableId, 0, timestamp, model)); + } + if (role === "assistant" && Array.isArray(body.tool_calls)) { + for (let index = 0;index < body.tool_calls.length; index += 1) { + const call = body.tool_calls[index]; + if (isObject(call)) { + items.push(toolCallItem(call, stableId, index + items.length, timestamp, model)); + } + } + } + return items; +} +function decodeContentBlock(block, role, stableId, index, timestamp, model) { + const type = block.type; + if (type === "tool_use" || type === "tool-call" || type === "function_call") { + return [toolCallItem(block, stableId, index, timestamp, model)]; + } + if (type === "tool_result" || type === "tool-result" || type === "function_call_output") { + return [toolResultItem(block, stableId, index, timestamp)]; + } + if (type === "thinking" || type === "reasoning" || type === "redacted_thinking") { + const text2 = reasoningText(block); + return text2 ? [reasoningItem(text2, stableId, index, timestamp, model)] : []; + } + if (type === "image" || type === "image_url") { + return [messageItem(role, "[image]", stableId, index, timestamp, model)]; + } + const text = contentText(block.text ?? block.content); + return text ? [messageItem(role, text, stableId, index, timestamp, model)] : []; +} +function messageItem(role, content, stableId, index, timestamp, model) { + const canonical = role === "assistant" ? "assistant" : role === "user" ? "user" : role; + const key = stableId ? `message:${stableId}:${index}` : `message:${canonical}:${jsonString(content)}`; + if (canonical !== "user" && canonical !== "assistant") + return { key }; + return { + key, + ...stableId ? { stable: true } : {}, + event: { + type: "message", + role: canonical, + content, + ...timestamp ? { timestamp } : {}, + ...model ? { model } : {} + } + }; +} +function reasoningItem(content, stableId, index, timestamp, model) { + return { + key: stableId ? `reasoning:${stableId}:${index}` : `reasoning:${jsonString(content)}`, + ...stableId ? { stable: true } : {}, + event: { + type: "reasoning", + content, + ...timestamp ? { timestamp } : {}, + ...model ? { model } : {} + } + }; +} +function toolCallItem(value, stableId, index, timestamp, model) { + const fn = isObject(value.function) ? value.function : {}; + const id = firstString(value.id, value.call_id, value.toolCallId); + const name = firstString(value.name, value.toolName, fn.name); + const rawArgs = firstDefined(value.arguments, value.args, value.input, fn.arguments); + const args = typeof rawArgs === "string" ? rawArgs : jsonString(rawArgs); + return { + key: id ? `tool-call:${id}` : `tool-call:${stableId ?? ""}:${index}:${name ?? ""}:${args}`, + ...id || stableId ? { stable: true } : {}, + event: { + type: "tool_call", + args, + ...id ? { id } : {}, + ...name ? { name } : {}, + ...timestamp ? { timestamp } : {}, + ...model ? { model } : {} + } + }; +} +function toolResultItem(value, stableId, index, timestamp) { + const id = firstString(value.tool_call_id, value.tool_use_id, value.call_id, value.toolCallId); + const raw = firstDefined(value.output, value.result, value.content); + const content = resultText(raw); + return { + key: id ? `tool-result:${id}` : `tool-result:${stableId ?? ""}:${index}:${content}`, + ...id || stableId ? { stable: true } : {}, + event: { + type: "tool_result", + content, + ...id ? { callId: id } : {}, + ...timestamp ? { timestamp } : {} + } + }; +} +function decodeToolRun(run, timestamp, pendingCalls) { + const inputs = isObject(run.inputs) ? run.inputs : {}; + const outputs = isObject(run.outputs) ? run.outputs : {}; + const embedded = embeddedToolMessage(outputs); + const explicitId = firstString(inputs.toolCallId, inputs.tool_call_id, inputs.call_id, outputs.toolCallId, outputs.tool_call_id, outputs.call_id, embedded?.tool_call_id, embedded?.tool_use_id, embedded?.call_id); + const name = firstString(inputs.toolName, inputs.tool_name, run.name); + const call = matchPendingCall(pendingCalls, explicitId, name); + const callId = explicitId ?? call?.id; + const rawResult = firstDefined(embedded?.content, embedded?.output, embedded?.result, outputs.result, outputs.output, outputs.content, run.error); + if (rawResult === undefined) + return; + let content = resultText(rawResult); + if (run.error && rawResult === run.error && !/^error/i.test(content)) { + content = `Error: ${content}`; + } + return { + key: callId ? `tool-result:${callId}` : `tool-run:${firstString(run.id, run.run_id) ?? "unknown"}`, + stable: true, + event: { + type: "tool_result", + content, + ...callId ? { callId } : {}, + ...timestamp ? { timestamp } : {} + } + }; +} +function embeddedToolMessage(outputs) { + const output = isObject(outputs.output) ? outputs.output : undefined; + const candidate = output ?? (isObject(outputs.message) ? outputs.message : undefined); + if (!candidate) + return; + return isObject(candidate.kwargs) ? candidate.kwargs : candidate; +} +function matchPendingCall(pendingCalls, id, name) { + let match = id ? pendingCalls.find((call) => !call.consumed && call.id === id) : undefined; + match ??= name ? pendingCalls.find((call) => !call.consumed && call.name === name) : undefined; + const remaining = pendingCalls.filter((call) => !call.consumed); + match ??= remaining.length === 1 ? remaining[0] : undefined; + if (match) + match.consumed = true; + return match; +} +function mergeItems(state, items) { + if (items.length === 0) + return; + let overlap = Math.min(state.history.length, items.length); + while (overlap > 0) { + let matches = true; + const historyStart = state.history.length - overlap; + for (let index = 0;index < overlap; index += 1) { + if (state.history[historyStart + index] !== items[index]?.key) { + matches = false; + break; + } + } + if (matches) + break; + overlap -= 1; + } + const repeatedSnapshot = overlap === 0 && items.some((item) => item.stable === true && state.history.includes(item.key)); + for (let index = overlap;index < items.length; index += 1) { + const item = items[index]; + if (!item) + continue; + if (item.stable === true && state.history.includes(item.key)) + continue; + if (repeatedSnapshot && state.history.includes(item.key)) + continue; + state.history.push(item.key); + if (!item.event) + continue; + state.events.push(item.event); + if (item.event.type === "tool_call") { + state.pendingCalls.push({ + ...item.event.id ? { id: item.event.id } : {}, + ...item.event.name ? { name: item.event.name } : {}, + consumed: false + }); + } else if (item.event.type === "tool_result" && item.event.callId) { + matchPendingCall(state.pendingCalls, item.event.callId, undefined); + } + } +} +function canonicalRole(value, constructorClass) { + if (constructorClass === "HumanMessage" || constructorClass === "ChatMessage") { + return "user"; + } + if (constructorClass === "AIMessage") + return "assistant"; + if (constructorClass === "ToolMessage" || constructorClass === "FunctionMessage") { + return "tool"; + } + if (constructorClass === "SystemMessage") + return "system"; + if (value === "human") + return "user"; + if (value === "ai") + return "assistant"; + if (value === "function") + return "tool"; + return value ?? "assistant"; +} +function constructorName(value) { + if (!Array.isArray(value.id)) + return; + const last = value.id[value.id.length - 1]; + return typeof last === "string" ? last : undefined; +} +function contentText(value) { + if (typeof value === "string") + return value; + if (value === null || value === undefined) + return ""; + if (Array.isArray(value)) + return blocksText(value); + if (isObject(value) && typeof value.value === "string") + return value.value; + return ""; +} +function reasoningText(value) { + const direct = firstString(value.thinking, value.text, value.content); + if (direct) + return direct; + if (Array.isArray(value.summary)) { + return value.summary.map((item) => isObject(item) ? contentText(item.text ?? item.content) : "").filter(Boolean).join(` +`); + } + return ""; +} +function resultText(value) { + if (typeof value === "string") + return value; + if (value === undefined || value === null) + return ""; + if (Array.isArray(value)) + return blocksText(value) || jsonString(value); + if (isObject(value)) { + if (isObject(value.kwargs)) + return resultText(value.kwargs.content); + if (typeof value.content === "string") + return value.content; + } + return jsonString(value); +} +function firstString(...values) { + return values.find((value) => typeof value === "string" && value.length > 0); +} +function firstDefined(...values) { + return values.find((value) => value !== undefined && value !== null); +} +function firstNonEmpty(...values) { + return values.find((value) => value !== undefined && value !== null && (!Array.isArray(value) || value.length > 0)); +} + // src/adapters/letta.ts var lettaAdapter = { source: "letta", @@ -1299,6 +1815,7 @@ function sliceCodePoints(text, start, end) { var ADAPTERS = { "claude-code": claudeCodeAdapter, codex: codexAdapter, + langsmith: langSmithAdapter, letta: lettaAdapter, openhands: openHandsAdapter }; From 9bc1fa7a283c7c689edf8d9af1df1925dd6a7769 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:22:33 -0700 Subject: [PATCH 04/10] Cover LangSmith in Python wrapper tests --- python/tests/test_wrapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/tests/test_wrapper.py b/python/tests/test_wrapper.py index c7776a3..c05a048 100644 --- a/python/tests/test_wrapper.py +++ b/python/tests/test_wrapper.py @@ -17,6 +17,8 @@ ("claude-code", "claude-code/cleanup", "input.jsonl"), ("codex", "codex/tool-calls", "input.jsonl"), ("codex", "codex/cleanup", "input.jsonl"), + ("langsmith", "langsmith/tool-call", "input.jsonl"), + ("langsmith", "langsmith/cleanup", "input.jsonl"), ("letta", "letta/tool-call", "input.json"), ("letta", "letta/cleanup", "input.json"), ("openhands", "openhands/tool-calls", "input.json"), @@ -58,7 +60,7 @@ def test_normalization_errors_include_code_and_batch_index(self) -> None: "source": "codex", "transcript": fixture_text("codex/cleanup", "input.jsonl"), } - invalid = {"source": "langsmith", "transcript": "{}"} + invalid = {"source": "not-a-source", "transcript": "{}"} with self.assertRaises(NormalizationError) as raised: normalize_many([valid, invalid]) From 24be177b0eee75e6a7a1b91c83eb724cd5f3a25b Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:26:06 -0700 Subject: [PATCH 05/10] Decode Anthropic SSE trace outputs --- PARITY.md | 6 +- README.md | 3 +- .../src/trajectory/_vendor/trajectory-cli.mjs | 84 ++++++++++++++- src/adapters/langsmith.ts | 102 +++++++++++++++++- test/normalize.test.ts | 41 +++++++ 5 files changed, 228 insertions(+), 8 deletions(-) diff --git a/PARITY.md b/PARITY.md index cda24c2..0160a55 100644 --- a/PARITY.md +++ b/PARITY.md @@ -65,9 +65,9 @@ Read-only validation was also run against two user-provided LangSmith projects without retaining raw traces or normalized content in the repository. The sample comprised one two-trace thread with 17 total runs (13 chain, two LLM, and two tool runs) and one standalone LLM run. This surfaced and fixed two -native variants not present in the synthetic fixtures: string-valued -`outputs.output` completions and a tool call repeated in both content blocks -and the message-level `tool_calls` field. +native variants not present in the original synthetic fixtures: an Anthropic +SSE event stream stored in string-valued `outputs.output`, and a tool call +repeated in both content blocks and the message-level `tool_calls` field. After those repairs, the combined thread normalized to 16 records with native tool linkage preserved; its only diagnostic was the expected configured diff --git a/README.md b/README.md index ce26aa4..a9b24c7 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,8 @@ directly as one transcript per exported JSONL file. Runs are ordered by `dotted_order` and `start_time`; nested `child_runs` are flattened. The adapter decodes the LangChain/LangGraph, OpenAI Chat Completions and Responses, Anthropic Messages, and Vercel AI SDK message envelopes documented by -LangSmith. Repeated message-history snapshots from successive LLM runs are +LangSmith. String-valued Anthropic SSE outputs are reconstructed from their +text, thinking, and tool-input deltas. Repeated message-history snapshots are deduplicated, while tool runs are linked to the earlier model tool call by call ID and then by tool name when an integration omits the ID. Fetching or exporting runs from LangSmith remains the caller's responsibility. diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index 51c02f8..d71a7d5 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -550,7 +550,10 @@ function inputMessages(value) { } function outputMessages(value) { if (!isObject(value)) { - return typeof value === "string" ? [{ role: "assistant", content: value }] : []; + if (typeof value !== "string") + return []; + const streamed = anthropicStreamMessages(value); + return streamed.length > 0 ? streamed : [{ role: "assistant", content: value }]; } if (Array.isArray(value.generations)) { const firstBatch = Array.isArray(value.generations[0]) ? value.generations[0] : value.generations; @@ -579,7 +582,8 @@ function outputMessages(value) { if (Array.isArray(value.output)) return value.output; if (typeof value.output === "string") { - return [{ role: "assistant", content: value.output }]; + const streamed = anthropicStreamMessages(value.output); + return streamed.length > 0 ? streamed : [{ role: "assistant", content: value.output }]; } if (isObject(value.output)) { if (Array.isArray(value.output.messages)) @@ -595,6 +599,82 @@ function outputMessages(value) { } return []; } +function anthropicStreamMessages(output) { + if (!output.includes("event:") || !output.includes(` +data:`)) + return []; + const blocks = new Map; + for (const line of output.split(` +`)) { + if (!line.startsWith("data:")) + continue; + const raw = line.slice("data:".length).trim(); + if (!raw || raw === "[DONE]") + continue; + let event; + try { + event = JSON.parse(raw); + } catch { + continue; + } + if (!isObject(event) || typeof event.index !== "number") + continue; + if (event.type === "content_block_start" && isObject(event.content_block)) { + const content2 = event.content_block; + if (typeof content2.type !== "string") + continue; + blocks.set(event.index, { + type: content2.type, + ...typeof content2.text === "string" ? { text: content2.text } : {}, + ...typeof content2.thinking === "string" ? { thinking: content2.thinking } : {}, + ...typeof content2.id === "string" ? { id: content2.id } : {}, + ...typeof content2.name === "string" ? { name: content2.name } : {}, + ...content2.input !== undefined ? { input: content2.input } : {}, + partialJson: "" + }); + continue; + } + if (event.type !== "content_block_delta" || !isObject(event.delta)) { + continue; + } + const delta = event.delta; + const block = blocks.get(event.index); + if (!block) + continue; + if (delta.type === "text_delta" && typeof delta.text === "string") { + block.text = (block.text ?? "") + delta.text; + } else if (delta.type === "thinking_delta" && typeof delta.thinking === "string") { + block.thinking = (block.thinking ?? "") + delta.thinking; + } else if (delta.type === "input_json_delta" && typeof delta.partial_json === "string") { + block.partialJson += delta.partial_json; + } + } + const content = []; + for (const [, block] of [...blocks].sort(([left], [right]) => left - right)) { + if (block.type === "text" && block.text) { + content.push({ type: "text", text: block.text }); + } else if (block.type === "thinking" && block.thinking) { + content.push({ type: "thinking", thinking: block.thinking }); + } else if (block.type === "tool_use") { + content.push({ + type: "tool_use", + ...block.id ? { id: block.id } : {}, + ...block.name ? { name: block.name } : {}, + input: streamedToolInput(block) + }); + } + } + return content.length > 0 ? [{ role: "assistant", content }] : []; +} +function streamedToolInput(block) { + if (!block.partialJson) + return block.input ?? {}; + try { + return JSON.parse(block.partialJson); + } catch { + return { _raw: block.partialJson }; + } +} function messageArray(value, fallbackRole) { if (typeof value === "string") return [{ role: fallbackRole, content: value }]; diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index 20d3106..a86a863 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -29,6 +29,16 @@ interface PendingCall { consumed: boolean; } +interface AnthropicStreamBlock { + type: string; + text?: string; + thinking?: string; + id?: string; + name?: string; + input?: unknown; + partialJson: string; +} + interface DecodeState { events: DecodedEvent[]; history: string[]; @@ -225,7 +235,11 @@ function inputMessages(value: unknown): unknown[] { function outputMessages(value: unknown): unknown[] { if (!isObject(value)) { - return typeof value === "string" ? [{ role: "assistant", content: value }] : []; + if (typeof value !== "string") return []; + const streamed = anthropicStreamMessages(value); + return streamed.length > 0 + ? streamed + : [{ role: "assistant", content: value }]; } if (Array.isArray(value.generations)) { @@ -254,7 +268,10 @@ function outputMessages(value: unknown): unknown[] { if (isObject(value.message)) return [value.message]; if (Array.isArray(value.output)) return value.output; if (typeof value.output === "string") { - return [{ role: "assistant", content: value.output }]; + const streamed = anthropicStreamMessages(value.output); + return streamed.length > 0 + ? streamed + : [{ role: "assistant", content: value.output }]; } if (isObject(value.output)) { if (Array.isArray(value.output.messages)) return value.output.messages; @@ -269,6 +286,87 @@ function outputMessages(value: unknown): unknown[] { return []; } +function anthropicStreamMessages(output: string): unknown[] { + if (!output.includes("event:") || !output.includes("\ndata:")) return []; + const blocks = new Map(); + + for (const line of output.split("\n")) { + if (!line.startsWith("data:")) continue; + const raw = line.slice("data:".length).trim(); + if (!raw || raw === "[DONE]") continue; + let event: unknown; + try { + event = JSON.parse(raw); + } catch { + continue; + } + if (!isObject(event) || typeof event.index !== "number") continue; + + if (event.type === "content_block_start" && isObject(event.content_block)) { + const content = event.content_block; + if (typeof content.type !== "string") continue; + blocks.set(event.index, { + type: content.type, + ...(typeof content.text === "string" ? { text: content.text } : {}), + ...(typeof content.thinking === "string" + ? { thinking: content.thinking } + : {}), + ...(typeof content.id === "string" ? { id: content.id } : {}), + ...(typeof content.name === "string" ? { name: content.name } : {}), + ...(content.input !== undefined ? { input: content.input } : {}), + partialJson: "", + }); + continue; + } + + if (event.type !== "content_block_delta" || !isObject(event.delta)) { + continue; + } + const delta = event.delta; + const block = blocks.get(event.index); + if (!block) continue; + if (delta.type === "text_delta" && typeof delta.text === "string") { + block.text = (block.text ?? "") + delta.text; + } else if ( + delta.type === "thinking_delta" && + typeof delta.thinking === "string" + ) { + block.thinking = (block.thinking ?? "") + delta.thinking; + } else if ( + delta.type === "input_json_delta" && + typeof delta.partial_json === "string" + ) { + block.partialJson += delta.partial_json; + } + } + + const content: Record[] = []; + for (const [, block] of [...blocks].sort(([left], [right]) => left - right)) { + if (block.type === "text" && block.text) { + content.push({ type: "text", text: block.text }); + } else if (block.type === "thinking" && block.thinking) { + content.push({ type: "thinking", thinking: block.thinking }); + } else if (block.type === "tool_use") { + content.push({ + type: "tool_use", + ...(block.id ? { id: block.id } : {}), + ...(block.name ? { name: block.name } : {}), + input: streamedToolInput(block), + }); + } + } + return content.length > 0 ? [{ role: "assistant", content }] : []; +} + +function streamedToolInput(block: AnthropicStreamBlock): unknown { + if (!block.partialJson) return block.input ?? {}; + try { + return JSON.parse(block.partialJson); + } catch { + return { _raw: block.partialJson }; + } +} + function messageArray(value: unknown, fallbackRole: string): unknown[] { if (typeof value === "string") return [{ role: fallbackRole, content: value }]; if (isObject(value)) return [value]; diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 9b4eb68..265a8b1 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -148,6 +148,47 @@ describe("public API", () => { ); }); + test("reconstructs Anthropic SSE stored as string output", () => { + const output = [ + "event: message_start", + 'data: {"type":"message_start","message":{"role":"assistant","content":[]}}', + "", + "event: content_block_start", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}', + "", + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Check first."}}', + "", + "event: content_block_start", + 'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}', + "", + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Hello "}}', + "", + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"world."}}', + "", + "event: message_stop", + 'data: {"type":"message_stop"}', + "", + ].join("\n"); + const transcript = JSON.stringify({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { output }, + }); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records).toContainEqual( + expect.objectContaining({ role: "reasoning", content: "Check first." }), + ); + expect(result.records).toContainEqual( + expect.objectContaining({ role: "assistant", content: "Hello world." }), + ); + }); + test("deduplicates a tool call repeated in native message fields", () => { const call = { id: "call-1", From 83369c1126fbc391ac92cf41ccf7da9dd7091292 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:36:40 -0700 Subject: [PATCH 06/10] Document canonical LangSmith run input --- README.md | 40 ++++++++++++++++++++++++++------------- src/adapters/langsmith.ts | 3 +++ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a9b24c7..24f7252 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ and is empty when the transcript required no recoverable cleanup. | --- | --- | --- | | `claude-code` | Native Claude Code JSONL | `claude-code` | | `codex` | Native Codex rollout JSONL | `codex` | -| `langsmith` | LangSmith run object, run array, `{ "runs": [...] }`, or run JSONL | `langsmith` | +| `langsmith` | Canonical LangSmith `Run` records as JSON array or `{ "runs": [...] }` | `langsmith` | | `letta` | Native Letta transcript JSON | `letta` | | `openhands` | JSON event array or an events-API `{ "items": [...] }` envelope | `openhands` | @@ -117,18 +117,32 @@ ignores system and approval-control records. OpenHands inputs are serialized exports; when a native store uses individual event files, assembling the event array remains the caller's responsibility. -LangSmith inputs contain the runs for one trace or one chronological thread. -The output of -`langsmith trace export --project --full` can be passed -directly as one transcript per exported JSONL file. Runs are ordered by -`dotted_order` and `start_time`; nested `child_runs` are flattened. The adapter -decodes the LangChain/LangGraph, OpenAI Chat Completions and Responses, -Anthropic Messages, and Vercel AI SDK message envelopes documented by -LangSmith. String-valued Anthropic SSE outputs are reconstructed from their -text, thinking, and tool-input deltas. Repeated message-history snapshots are -deduplicated, while tool runs are linked to the earlier model tool call by call -ID and then by tool name when an integration omits the ID. Fetching or exporting -runs from LangSmith remains the caller's responsibility. +The canonical LangSmith input is the public `Run` span format returned by the +SDK or `/runs/query`: a JSON array of runs, or the API's `{ "runs": [...] }` +envelope, for one trace or chronological thread. A trace is represented by a +root run and child runs linked through `trace_id`, `parent_run_id`, and +`dotted_order`; LangSmith does not define a separate flattened-conversation +schema. Runs are ordered by `dotted_order` and `start_time`, and SDK trees with +nested `child_runs` are flattened. + +For compatibility, the adapter also accepts a single `Run`, the JSONL written +by `langsmith trace export --project --full`, and the +CLI's `run_id` / `custom_metadata` field aliases. These are alternate +serializations of Run data, not a trajectory-specific LangSmith format. + +Run `inputs` and `outputs` remain integration-specific. The adapter follows the +LangSmith Messages-view formats for LangChain/LangGraph, OpenAI Chat +Completions and Responses, Anthropic Messages, and the Vercel AI SDK. Repeated +message-history snapshots are deduplicated, while tool runs are linked to the +earlier model tool call by call ID and then by tool name when an integration +omits the ID. Fetching or exporting runs from LangSmith remains the caller's +responsibility. + +LangSmith's Anthropic wrapper aggregates stream events before storing outputs, +but that reducer is not exported as a public SDK utility. As a compatibility +fallback for traces recorded outside the wrapper, string-valued Anthropic SSE +outputs are reconstructed locally from their text, thinking, and tool-input +deltas. For a multi-turn thread, combine the runs from each member trace into one input container before normalization. This preserves history and tool linkage that diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index a86a863..6c4dbdd 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -287,6 +287,9 @@ function outputMessages(value: unknown): unknown[] { } function anthropicStreamMessages(output: string): unknown[] { + // LangSmith's wrapAnthropic performs equivalent aggregation before ingest, + // but its messageAggregator is private. This handles traces produced by + // custom HTTP instrumentation that stored the raw Anthropic SSE response. if (!output.includes("event:") || !output.includes("\ndata:")) return []; const blocks = new Map(); From bd8412fc18448c52e2631298084c6042ba500b8e Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:42:02 -0700 Subject: [PATCH 07/10] Narrow LangSmith input to canonical runs --- PARITY.md | 6 +- README.md | 22 ++- fixtures/langsmith/cleanup/expected.json | 1 - fixtures/langsmith/cleanup/input.json | 1 + fixtures/langsmith/cleanup/input.jsonl | 4 - fixtures/langsmith/tool-call/input.json | 1 + fixtures/langsmith/tool-call/input.jsonl | 3 - .../src/trajectory/_vendor/trajectory-cli.mjs | 82 ++--------- python/tests/test_wrapper.py | 4 +- src/adapters/langsmith.ts | 98 ++++---------- test/normalize.test.ts | 127 +++++++++--------- 11 files changed, 111 insertions(+), 238 deletions(-) create mode 100644 fixtures/langsmith/cleanup/input.json delete mode 100644 fixtures/langsmith/cleanup/input.jsonl create mode 100644 fixtures/langsmith/tool-call/input.json delete mode 100644 fixtures/langsmith/tool-call/input.jsonl diff --git a/PARITY.md b/PARITY.md index 0160a55..ca9eccf 100644 --- a/PARITY.md +++ b/PARITY.md @@ -56,10 +56,10 @@ records in the compatibility fixtures. ## LangSmith adapter The LangSmith adapter is covered with synthetic runs matching the published -LangSmith run and Messages-view formats. Fixtures exercise LangChain +LangSmith Run and Messages-view formats. Fixtures exercise LangChain constructor messages, Vercel AI SDK content blocks, repeated history snapshots, -tool-result matching by ID and tool name, run ordering, metadata, malformed -JSONL cleanup, and missing-timestamp repair. +tool-result matching by ID and tool name, run ordering, metadata, and +missing-timestamp repair. Read-only validation was also run against two user-provided LangSmith projects without retaining raw traces or normalized content in the repository. The diff --git a/README.md b/README.md index 24f7252..423c3f9 100644 --- a/README.md +++ b/README.md @@ -117,18 +117,13 @@ ignores system and approval-control records. OpenHands inputs are serialized exports; when a native store uses individual event files, assembling the event array remains the caller's responsibility. -The canonical LangSmith input is the public `Run` span format returned by the +The LangSmith input is the canonical public `Run` span format returned by the SDK or `/runs/query`: a JSON array of runs, or the API's `{ "runs": [...] }` -envelope, for one trace or chronological thread. A trace is represented by a -root run and child runs linked through `trace_id`, `parent_run_id`, and +envelope, for one trace or chronological thread. Each run must include `id`, +`trace_id`, `name`, `run_type`, and `inputs`. A trace is represented by a root +run and child runs linked through `trace_id`, `parent_run_id`, and `dotted_order`; LangSmith does not define a separate flattened-conversation -schema. Runs are ordered by `dotted_order` and `start_time`, and SDK trees with -nested `child_runs` are flattened. - -For compatibility, the adapter also accepts a single `Run`, the JSONL written -by `langsmith trace export --project --full`, and the -CLI's `run_id` / `custom_metadata` field aliases. These are alternate -serializations of Run data, not a trajectory-specific LangSmith format. +schema. Runs are ordered by `dotted_order` and `start_time`. Run `inputs` and `outputs` remain integration-specific. The adapter follows the LangSmith Messages-view formats for LangChain/LangGraph, OpenAI Chat @@ -139,10 +134,9 @@ omits the ID. Fetching or exporting runs from LangSmith remains the caller's responsibility. LangSmith's Anthropic wrapper aggregates stream events before storing outputs, -but that reducer is not exported as a public SDK utility. As a compatibility -fallback for traces recorded outside the wrapper, string-valued Anthropic SSE -outputs are reconstructed locally from their text, thinking, and tool-input -deltas. +but that reducer is not exported as a public SDK utility. When canonical Run +data contains a raw Anthropic SSE string in `outputs.output`, the adapter +reconstructs its text, thinking, and tool-input deltas locally. For a multi-turn thread, combine the runs from each member trace into one input container before normalization. This preserves history and tool linkage that diff --git a/fixtures/langsmith/cleanup/expected.json b/fixtures/langsmith/cleanup/expected.json index 843ae7f..0086626 100644 --- a/fixtures/langsmith/cleanup/expected.json +++ b/fixtures/langsmith/cleanup/expected.json @@ -8,7 +8,6 @@ { "role": "assistant", "content": "It is sunny in Paris.", "timestamp": "2026-07-10T13:00:04.000Z" } ], "diagnostics": [ - { "code": "invalid_json_line", "message": "Skipped invalid JSON on line 1.", "inputLine": 1 }, { "code": "timestamps_interpolated", "message": "Interpolated timestamps for 1 normalized records.", "count": 1 } ] } diff --git a/fixtures/langsmith/cleanup/input.json b/fixtures/langsmith/cleanup/input.json new file mode 100644 index 0000000..bbbfcf4 --- /dev/null +++ b/fixtures/langsmith/cleanup/input.json @@ -0,0 +1 @@ +[{"trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","start_time":"2026-07-10T13:00:00Z","end_time":"2026-07-10T13:00:01Z","inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]},"id":"vercel-1","extra":{"metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4","cwd":"/workspace","git_branch":"feature/weather"}}},{"trace_id":"trace-2","parent_run_id":"vercel-1","run_type":"tool","name":"weather","start_time":"2026-07-10T13:00:02Z","end_time":"2026-07-10T13:00:03Z","inputs":{"args":{"city":"Paris"}},"outputs":{"result":{"condition":"sunny"}},"id":"vercel-tool"},{"trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]},{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]},{"role":"tool","toolCallId":"call-vercel","content":{"condition":"sunny"}}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"It is sunny in Paris."}]},"id":"vercel-2","extra":{"metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4"}}}] diff --git a/fixtures/langsmith/cleanup/input.jsonl b/fixtures/langsmith/cleanup/input.jsonl deleted file mode 100644 index 2bc39f9..0000000 --- a/fixtures/langsmith/cleanup/input.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -not-json -{"run_id":"vercel-1","trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","start_time":"2026-07-10T13:00:00Z","end_time":"2026-07-10T13:00:01Z","custom_metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4","cwd":"/workspace","git_branch":"feature/weather"},"inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]}} -{"run_id":"vercel-tool","trace_id":"trace-2","parent_run_id":"vercel-1","run_type":"tool","name":"weather","start_time":"2026-07-10T13:00:02Z","end_time":"2026-07-10T13:00:03Z","inputs":{"args":{"city":"Paris"}},"outputs":{"result":{"condition":"sunny"}}} -{"run_id":"vercel-2","trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","custom_metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4"},"inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]},{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]},{"role":"tool","toolCallId":"call-vercel","content":{"condition":"sunny"}}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"It is sunny in Paris."}]}} diff --git a/fixtures/langsmith/tool-call/input.json b/fixtures/langsmith/tool-call/input.json new file mode 100644 index 0000000..83524da --- /dev/null +++ b/fixtures/langsmith/tool-call/input.json @@ -0,0 +1 @@ +[{"id":"llm-2","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:04Z","end_time":"2026-07-10T12:00:05Z","dotted_order":"20260710T120004000000Zllm-2","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"It is sunny and 22C in Paris.","id":"ai-2"}}}]]}},{"id":"llm-1","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:00Z","end_time":"2026-07-10T12:00:01Z","dotted_order":"20260710T120000000000Zllm-1","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}}}]]}},{"id":"tool-run-1","trace_id":"trace-1","parent_run_id":"llm-1","run_type":"tool","name":"get_weather","start_time":"2026-07-10T12:00:02Z","end_time":"2026-07-10T12:00:03Z","dotted_order":"20260710T120002000000Ztool-run-1","inputs":{"city":"Paris"},"outputs":{"output":{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}}}] diff --git a/fixtures/langsmith/tool-call/input.jsonl b/fixtures/langsmith/tool-call/input.jsonl deleted file mode 100644 index d5f0281..0000000 --- a/fixtures/langsmith/tool-call/input.jsonl +++ /dev/null @@ -1,3 +0,0 @@ -{"id":"llm-2","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:04Z","end_time":"2026-07-10T12:00:05Z","dotted_order":"20260710T120004000000Zllm-2","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"It is sunny and 22C in Paris.","id":"ai-2"}}}]]}} -{"id":"llm-1","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:00Z","end_time":"2026-07-10T12:00:01Z","dotted_order":"20260710T120000000000Zllm-1","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}}}]]}} -{"id":"tool-run-1","trace_id":"trace-1","parent_run_id":"llm-1","run_type":"tool","name":"get_weather","start_time":"2026-07-10T12:00:02Z","end_time":"2026-07-10T12:00:03Z","dotted_order":"20260710T120002000000Ztool-run-1","inputs":{"city":"Paris"},"outputs":{"output":{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}}} diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index d71a7d5..87e458a 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -386,8 +386,7 @@ class NormalizationError extends Error { var langSmithAdapter = { source: "langsmith", decode(transcript) { - const diagnostics = []; - const runs = parseRuns(transcript, diagnostics).sort(compareRuns); + const runs = parseRuns(transcript).sort(compareRuns); const state = { events: [], history: [], @@ -426,84 +425,28 @@ var langSmithAdapter = { ...model ? { model } : {}, ...createdAt ? { createdAt } : {} }, - diagnostics + diagnostics: [] }; } }; -function parseRuns(transcript, diagnostics) { +function parseRuns(transcript) { let parsed; try { parsed = JSON.parse(transcript); } catch { - return parseRunJsonLines(transcript, diagnostics); - } - const roots = runContainer(parsed); - if (!roots) throw invalidTranscript(); - const flattened = []; - for (const root of roots) - flattenRun(root, flattened); - if (flattened.length === 0) - throw invalidTranscript(); - return flattened.map((run, index) => ({ run, index })); -} -function parseRunJsonLines(transcript, diagnostics) { - const flattened = []; - for (const [index, raw] of transcript.split(` -`).entries()) { - if (!raw.trim()) - continue; - let value; - try { - value = JSON.parse(raw); - } catch { - diagnostics.push({ - code: "invalid_json_line", - message: `Skipped invalid JSON on line ${index + 1}.`, - inputLine: index + 1 - }); - continue; - } - const roots = runContainer(value); - if (!roots) { - diagnostics.push({ - code: "non_object_json_line", - message: `Skipped a non-run JSON value on line ${index + 1}.`, - inputLine: index + 1 - }); - continue; - } - for (const root of roots) - flattenRun(root, flattened); } - if (flattened.length === 0) + const runs = Array.isArray(parsed) ? parsed : isObject(parsed) && Array.isArray(parsed.runs) ? parsed.runs : undefined; + if (!runs || runs.length === 0 || !runs.every(isCanonicalRun)) { throw invalidTranscript(); - return flattened.map((run, index) => ({ run, index })); -} -function runContainer(value) { - if (Array.isArray(value)) { - return value.every(isObject) ? value : undefined; } - if (!isObject(value)) - return; - if (Array.isArray(value.runs)) { - return value.runs.every(isObject) ? value.runs : undefined; - } - if (typeof value.run_type === "string") - return [value]; - return; + return runs.map((run, index) => ({ run, index })); } -function flattenRun(run, output) { - output.push(run); - if (!Array.isArray(run.child_runs)) - return; - for (const child of run.child_runs) { - if (isObject(child)) - flattenRun(child, output); - } +function isCanonicalRun(value) { + return isObject(value) && typeof value.id === "string" && typeof value.trace_id === "string" && typeof value.name === "string" && typeof value.run_type === "string" && isObject(value.inputs); } function invalidTranscript() { - return new NormalizationError("invalid_input", "LangSmith transcript must be a run object, a JSON run array, an object with a runs array, or JSONL containing runs."); + return new NormalizationError("invalid_input", "LangSmith transcript must be a non-empty JSON array of canonical Run records or an object with a runs array."); } function compareRuns(left, right) { const leftOrder = left.run.dotted_order; @@ -521,11 +464,8 @@ function compareRuns(left, right) { return left.index - right.index; } function runMetadata(run) { - const topLevel = isObject(run.metadata) ? run.metadata : {}; - const custom = isObject(run.custom_metadata) ? run.custom_metadata : {}; const extra = isObject(run.extra) ? run.extra : {}; - const nested = isObject(extra.metadata) ? extra.metadata : {}; - return { ...topLevel, ...custom, ...nested }; + return isObject(extra.metadata) ? extra.metadata : {}; } function inputMessages(value) { if (!isObject(value)) @@ -840,7 +780,7 @@ function decodeToolRun(run, timestamp, pendingCalls) { content = `Error: ${content}`; } return { - key: callId ? `tool-result:${callId}` : `tool-run:${firstString(run.id, run.run_id) ?? "unknown"}`, + key: callId ? `tool-result:${callId}` : `tool-run:${firstString(run.id) ?? "unknown"}`, stable: true, event: { type: "tool_result", diff --git a/python/tests/test_wrapper.py b/python/tests/test_wrapper.py index c05a048..c46abf0 100644 --- a/python/tests/test_wrapper.py +++ b/python/tests/test_wrapper.py @@ -17,8 +17,8 @@ ("claude-code", "claude-code/cleanup", "input.jsonl"), ("codex", "codex/tool-calls", "input.jsonl"), ("codex", "codex/cleanup", "input.jsonl"), - ("langsmith", "langsmith/tool-call", "input.jsonl"), - ("langsmith", "langsmith/cleanup", "input.jsonl"), + ("langsmith", "langsmith/tool-call", "input.json"), + ("langsmith", "langsmith/cleanup", "input.json"), ("letta", "letta/tool-call", "input.json"), ("letta", "letta/cleanup", "input.json"), ("openhands", "openhands/tool-calls", "input.json"), diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index 6c4dbdd..4cabc54 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -3,7 +3,6 @@ import type { DecodedSession, SourceAdapter, } from "../internal.js"; -import type { Diagnostic } from "../types.js"; import { NormalizationError } from "../types.js"; import { blocksText, @@ -49,8 +48,7 @@ export const langSmithAdapter: SourceAdapter = { source: "langsmith", decode(transcript: string): DecodedSession { - const diagnostics: Diagnostic[] = []; - const runs = parseRuns(transcript, diagnostics).sort(compareRuns); + const runs = parseRuns(transcript).sort(compareRuns); const state: DecodeState = { events: [], history: [], @@ -102,90 +100,45 @@ export const langSmithAdapter: SourceAdapter = { ...(model ? { model } : {}), ...(createdAt ? { createdAt } : {}), }, - diagnostics, + diagnostics: [], }; }, }; -function parseRuns( - transcript: string, - diagnostics: Diagnostic[], -): OrderedRun[] { +function parseRuns(transcript: string): OrderedRun[] { let parsed: unknown; try { parsed = JSON.parse(transcript); } catch { - return parseRunJsonLines(transcript, diagnostics); + throw invalidTranscript(); } - const roots = runContainer(parsed); - if (!roots) throw invalidTranscript(); - const flattened: Record[] = []; - for (const root of roots) flattenRun(root, flattened); - if (flattened.length === 0) throw invalidTranscript(); - return flattened.map((run, index) => ({ run, index })); -} - -function parseRunJsonLines( - transcript: string, - diagnostics: Diagnostic[], -): OrderedRun[] { - const flattened: Record[] = []; - for (const [index, raw] of transcript.split("\n").entries()) { - if (!raw.trim()) continue; - let value: unknown; - try { - value = JSON.parse(raw); - } catch { - diagnostics.push({ - code: "invalid_json_line", - message: `Skipped invalid JSON on line ${index + 1}.`, - inputLine: index + 1, - }); - continue; - } - const roots = runContainer(value); - if (!roots) { - diagnostics.push({ - code: "non_object_json_line", - message: `Skipped a non-run JSON value on line ${index + 1}.`, - inputLine: index + 1, - }); - continue; - } - for (const root of roots) flattenRun(root, flattened); - } - if (flattened.length === 0) throw invalidTranscript(); - return flattened.map((run, index) => ({ run, index })); -} - -function runContainer(value: unknown): Record[] | undefined { - if (Array.isArray(value)) { - return value.every(isObject) ? value : undefined; + const runs = Array.isArray(parsed) + ? parsed + : isObject(parsed) && Array.isArray(parsed.runs) + ? parsed.runs + : undefined; + if (!runs || runs.length === 0 || !runs.every(isCanonicalRun)) { + throw invalidTranscript(); } - if (!isObject(value)) return undefined; - if (Array.isArray(value.runs)) { - return value.runs.every(isObject) ? value.runs : undefined; - } - if (typeof value.run_type === "string") return [value]; - return undefined; + return runs.map((run, index) => ({ run, index })); } -function flattenRun( - run: Record, - output: Record[], -): void { - output.push(run); - if (!Array.isArray(run.child_runs)) return; - for (const child of run.child_runs) { - if (isObject(child)) flattenRun(child, output); - } +function isCanonicalRun(value: unknown): value is Record { + return ( + isObject(value) && + typeof value.id === "string" && + typeof value.trace_id === "string" && + typeof value.name === "string" && + typeof value.run_type === "string" && + isObject(value.inputs) + ); } function invalidTranscript(): NormalizationError { return new NormalizationError( "invalid_input", - "LangSmith transcript must be a run object, a JSON run array, an object with a runs array, or JSONL containing runs.", + "LangSmith transcript must be a non-empty JSON array of canonical Run records or an object with a runs array.", ); } @@ -205,11 +158,8 @@ function compareRuns(left: OrderedRun, right: OrderedRun): number { } function runMetadata(run: Record): Record { - const topLevel = isObject(run.metadata) ? run.metadata : {}; - const custom = isObject(run.custom_metadata) ? run.custom_metadata : {}; const extra = isObject(run.extra) ? run.extra : {}; - const nested = isObject(extra.metadata) ? extra.metadata : {}; - return { ...topLevel, ...custom, ...nested }; + return isObject(extra.metadata) ? extra.metadata : {}; } function inputMessages(value: unknown): unknown[] { @@ -625,7 +575,7 @@ function decodeToolRun( return { key: callId ? `tool-result:${callId}` - : `tool-run:${firstString(run.id, run.run_id) ?? "unknown"}`, + : `tool-run:${firstString(run.id) ?? "unknown"}`, stable: true, event: { type: "tool_result", diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 265a8b1..179e255 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -35,7 +35,9 @@ describe("golden fixtures", () => { test(fixture.name, () => { const input = fixtureText( fixture.name, - fixture.source === "openhands" || fixture.source === "letta" + fixture.source === "openhands" || + fixture.source === "letta" || + fixture.source === "langsmith" ? "input.json" : "input.jsonl", ); @@ -112,34 +114,15 @@ describe("public API", () => { ).toThrow(expect.objectContaining({ code: "invalid_input" })); }); - test("accepts a single LangSmith run object", () => { - const transcript = JSON.stringify({ - id: "run-1", - run_type: "llm", - start_time: "2026-07-10T00:00:00Z", - end_time: "2026-07-10T00:00:01Z", - inputs: { messages: [{ role: "user", content: "hello" }] }, - outputs: { - choices: [{ message: { role: "assistant", content: "hi" } }], - }, - }); - - const result = normalizeTranscript({ source: "langsmith", transcript }); - - expect(result.records.map((record) => record.role)).toEqual([ - "meta", - "user", - "assistant", - ]); - }); - test("accepts string output from a LangSmith LLM run", () => { - const transcript = JSON.stringify({ - id: "run-1", - run_type: "llm", - inputs: { messages: [{ role: "user", content: "hello" }] }, - outputs: { output: "hi" }, - }); + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { output: "hi" }, + }), + ]); const result = normalizeTranscript({ source: "langsmith", transcript }); @@ -172,12 +155,14 @@ describe("public API", () => { 'data: {"type":"message_stop"}', "", ].join("\n"); - const transcript = JSON.stringify({ - id: "run-1", - run_type: "llm", - inputs: { messages: [{ role: "user", content: "hello" }] }, - outputs: { output }, - }); + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { output }, + }), + ]); const result = normalizeTranscript({ source: "langsmith", transcript }); @@ -195,23 +180,25 @@ describe("public API", () => { name: "weather", args: { city: "Paris" }, }; - const transcript = JSON.stringify({ - id: "run-1", - run_type: "llm", - inputs: { messages: [{ role: "user", content: "weather?" }] }, - outputs: { - role: "assistant", - content: [ - { - type: "tool_use", - id: call.id, - name: call.name, - input: call.args, - }, - ], - tool_calls: [call], - }, - }); + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "weather?" }] }, + outputs: { + role: "assistant", + content: [ + { + type: "tool_use", + id: call.id, + name: call.name, + input: call.args, + }, + ], + tool_calls: [call], + }, + }), + ]); const result = normalizeTranscript({ source: "langsmith", transcript }); @@ -225,21 +212,15 @@ describe("public API", () => { ); }); - test("flattens nested LangSmith runs envelopes", () => { + test("accepts a canonical LangSmith runs envelope", () => { const transcript = JSON.stringify({ runs: [ - { - id: "root", - run_type: "chain", - child_runs: [ - { - id: "llm", - run_type: "llm", - inputs: { messages: [{ role: "user", content: "hello" }] }, - outputs: { role: "assistant", content: "hi" }, - }, - ], - }, + canonicalLangSmithRun({ + id: "llm", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { role: "assistant", content: "hi" }, + }), ], }); @@ -304,7 +285,7 @@ describe("public API", () => { }, outputs: { message: { role: "assistant", content: "It is sunny." } }, }, - ]); + ].map(canonicalLangSmithRun)); const result = normalizeTranscript({ source: "langsmith", transcript }); @@ -383,7 +364,7 @@ describe("public API", () => { ], }, }, - ]); + ].map(canonicalLangSmithRun)); const result = normalizeTranscript({ source: "langsmith", transcript }); @@ -673,6 +654,20 @@ function fixtureText(name: string, file: string): string { return readFileSync(fileURLToPath(url), "utf8"); } +function canonicalLangSmithRun( + run: Record, +): Record { + const id = typeof run.id === "string" ? run.id : "run"; + return { + id, + trace_id: "trace-test", + name: id, + run_type: "llm", + inputs: {}, + ...run, + }; +} + function codexMessages(user: string, assistant: string): string { return [codexMessage("user", user), codexMessage("assistant", assistant)].join("\n"); } From c9b30bd47b06878dbd9e5a96ebf91bb69c93e06e Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 17:49:55 -0700 Subject: [PATCH 08/10] Deduplicate repeated LangSmith messages --- .../src/trajectory/_vendor/trajectory-cli.mjs | 5 ++ src/adapters/langsmith.ts | 14 +++++ test/normalize.test.ts | 53 +++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index 87e458a..dd8b13f 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -835,6 +835,8 @@ function mergeItems(state, items) { state.history.push(item.key); if (!item.event) continue; + if (isDuplicateAdjacentMessage(state.events.at(-1), item.event)) + continue; state.events.push(item.event); if (item.event.type === "tool_call") { state.pendingCalls.push({ @@ -847,6 +849,9 @@ function mergeItems(state, items) { } } } +function isDuplicateAdjacentMessage(previous, current) { + return previous?.type === "message" && current.type === "message" && previous.role === current.role && previous.content === current.content && previous.timestamp?.getTime() === current.timestamp?.getTime(); +} function canonicalRole(value, constructorClass) { if (constructorClass === "HumanMessage" || constructorClass === "ChatMessage") { return "user"; diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index 4cabc54..7e7749e 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -639,6 +639,7 @@ function mergeItems(state: DecodeState, items: ConversationItem[]): void { if (repeatedSnapshot && state.history.includes(item.key)) continue; state.history.push(item.key); if (!item.event) continue; + if (isDuplicateAdjacentMessage(state.events.at(-1), item.event)) continue; state.events.push(item.event); if (item.event.type === "tool_call") { state.pendingCalls.push({ @@ -652,6 +653,19 @@ function mergeItems(state: DecodeState, items: ConversationItem[]): void { } } +function isDuplicateAdjacentMessage( + previous: DecodedEvent | undefined, + current: DecodedEvent, +): boolean { + return ( + previous?.type === "message" && + current.type === "message" && + previous.role === current.role && + previous.content === current.content && + previous.timestamp?.getTime() === current.timestamp?.getTime() + ); +} + function canonicalRole(value: string | undefined, constructorClass?: string): string { if (constructorClass === "HumanMessage" || constructorClass === "ChatMessage") { return "user"; diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 179e255..8d03ae0 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -233,6 +233,59 @@ describe("public API", () => { ]); }); + test("deduplicates identical adjacent LangSmith messages with different IDs", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "llm", + start_time: "2026-07-10T00:00:00Z", + inputs: { + messages: [ + { id: "message-1", role: "user", content: "hello" }, + { id: "message-2", role: "user", content: "hello" }, + ], + }, + outputs: { role: "assistant", content: "hi" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + }); + + test("preserves identical LangSmith messages from different turns", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "llm-1", + start_time: "2026-07-10T00:00:00Z", + inputs: { + messages: [{ id: "message-1", role: "user", content: "retry" }], + }, + outputs: { role: "assistant", content: "first" }, + }), + canonicalLangSmithRun({ + id: "llm-2", + start_time: "2026-07-10T00:00:02Z", + inputs: { + messages: [{ id: "message-2", role: "user", content: "retry" }], + }, + outputs: { role: "assistant", content: "second" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect( + result.records.filter( + (record) => record.role === "user" && record.content === "retry", + ), + ).toHaveLength(2); + }); + test("decodes Anthropic reasoning and tool blocks from LangSmith runs", () => { const firstAssistant = { id: "msg-1", From b0c09dc0d6b471d3e7c9d0a04663791c05556940 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Fri, 10 Jul 2026 18:02:38 -0700 Subject: [PATCH 09/10] Align LangSmith normalization with canonical examples --- PARITY.md | 26 +++++-- README.md | 5 +- .../official-anthropic/expected.json | 11 +++ .../langsmith/official-anthropic/input.json | 73 ++++++++++++++++++ .../official-openai-responses/expected.json | 10 +++ .../official-openai-responses/input.json | 65 ++++++++++++++++ .../src/trajectory/_vendor/trajectory-cli.mjs | 34 ++++++--- python/tests/test_wrapper.py | 2 + src/adapters/langsmith.ts | 55 +++++++++++--- test/normalize.test.ts | 75 +++++++++++++++++++ 10 files changed, 327 insertions(+), 29 deletions(-) create mode 100644 fixtures/langsmith/official-anthropic/expected.json create mode 100644 fixtures/langsmith/official-anthropic/input.json create mode 100644 fixtures/langsmith/official-openai-responses/expected.json create mode 100644 fixtures/langsmith/official-openai-responses/input.json diff --git a/PARITY.md b/PARITY.md index ca9eccf..624108c 100644 --- a/PARITY.md +++ b/PARITY.md @@ -55,11 +55,14 @@ records in the compatibility fixtures. ## LangSmith adapter -The LangSmith adapter is covered with synthetic runs matching the published -LangSmith Run and Messages-view formats. Fixtures exercise LangChain -constructor messages, Vercel AI SDK content blocks, repeated history snapshots, -tool-result matching by ID and tool name, run ordering, metadata, and -missing-timestamp repair. +The LangSmith adapter is covered with synthetic edge cases and canonical Run +trees adapted from LangSmith's published Messages-view examples. The official +OpenAI Responses and Anthropic examples are stored with root spans, UUIDs, +timestamps, `extra.metadata`, and hierarchical `dotted_order` values so their +normalized fixtures require no repairs or diagnostics. Fixtures also exercise +LangChain constructor messages, Vercel AI SDK content blocks, repeated history +snapshots, excluded internal runs, bare tool outputs, tool-result matching by ID +and tool name, run ordering, metadata, and missing-timestamp repair. Read-only validation was also run against two user-provided LangSmith projects without retaining raw traces or normalized content in the repository. The @@ -69,8 +72,15 @@ native variants not present in the original synthetic fixtures: an Anthropic SSE event stream stored in string-valued `outputs.output`, and a tool call repeated in both content blocks and the message-level `tool_calls` field. -After those repairs, the combined thread normalized to 16 records with native +After those repairs, the combined thread normalized to 15 records with native tool linkage preserved; its only diagnostic was the expected configured tool-result truncation. The standalone LLM run normalized to three records -without diagnostics. No reference implementation was provided for differential -testing. +without diagnostics. + +The official examples were taken from LangSmith's +[Messages-view trace format reference](https://docs.langchain.com/langsmith/messages-view-trace-format). +All five published integration traces were also exercised directly: LangChain, +OpenAI Chat Completions, OpenAI Responses, Vercel AI SDK, and Anthropic +Messages. This differential check caught Responses item-ID versus call-ID +linkage, mixed stable/semantic message deduplication, bare tool-output objects, +and `ls_message_view_exclude` handling. diff --git a/README.md b/README.md index 423c3f9..8507660 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,9 @@ LangSmith Messages-view formats for LangChain/LangGraph, OpenAI Chat Completions and Responses, Anthropic Messages, and the Vercel AI SDK. Repeated message-history snapshots are deduplicated, while tool runs are linked to the earlier model tool call by call ID and then by tool name when an integration -omits the ID. Fetching or exporting runs from LangSmith remains the caller's -responsibility. +omits the ID. Runs carrying the `ls_message_view_exclude` metadata key are +ignored, matching LangSmith's Messages-view behavior. Fetching or exporting +runs from LangSmith remains the caller's responsibility. LangSmith's Anthropic wrapper aggregates stream events before storing outputs, but that reducer is not exported as a public SDK utility. When canonical Run diff --git a/fixtures/langsmith/official-anthropic/expected.json b/fixtures/langsmith/official-anthropic/expected.json new file mode 100644 index 0000000..342587b --- /dev/null +++ b/fixtures/langsmith/official-anthropic/expected.json @@ -0,0 +1,11 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "model": "claude-opus-4-7" }, + { "role": "user", "content": "what is the weather in paris?", "timestamp": "2026-07-10T12:00:01.000Z" }, + { "role": "assistant", "content": "Let me check.", "timestamp": "2026-07-10T12:00:02.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "toolu_01", "name": "get_weather", "args": "{\"city\":\"Paris\"}" }], "timestamp": "2026-07-10T12:00:02.000Z" }, + { "role": "tool", "tool_call_id": "toolu_01", "content": "{\"temperature\":22,\"condition\":\"Sunny\"}", "timestamp": "2026-07-10T12:00:04.000Z" }, + { "role": "assistant", "content": "It's sunny and 22°C in Paris.", "timestamp": "2026-07-10T12:00:06.000Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/official-anthropic/input.json b/fixtures/langsmith/official-anthropic/input.json new file mode 100644 index 0000000..4f7a91c --- /dev/null +++ b/fixtures/langsmith/official-anthropic/input.json @@ -0,0 +1,73 @@ +[ + { + "id": "00000000-0000-4000-8000-000000000004", + "trace_id": "00000000-0000-4000-8000-000000000004", + "name": "Weather Agent", + "run_type": "chain", + "start_time": "2026-07-10T12:00:00Z", + "end_time": "2026-07-10T12:00:07Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004", + "inputs": { "input": "what is the weather in paris?" }, + "outputs": { "output": "It's sunny and 22°C in Paris." } + }, + { + "id": "00000000-0000-4000-8000-000000000001", + "trace_id": "00000000-0000-4000-8000-000000000004", + "parent_run_id": "00000000-0000-4000-8000-000000000004", + "name": "ChatAnthropic", + "run_type": "llm", + "start_time": "2026-07-10T12:00:01Z", + "end_time": "2026-07-10T12:00:02Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004.20260710T120001000000Z00000000-0000-4000-8000-000000000001", + "extra": { "metadata": { "ls_provider": "anthropic", "ls_model_name": "claude-opus-4-7", "ls_message_format": "anthropic" } }, + "inputs": { + "system": "You are a helpful assistant.", + "messages": [{ "role": "user", "content": "what is the weather in paris?" }] + }, + "outputs": { + "message": { + "id": "msg_01", + "role": "assistant", + "type": "message", + "content": [ + { "type": "text", "text": "Let me check." }, + { "type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": { "city": "Paris" } } + ] + } + } + }, + { + "id": "00000000-0000-4000-8000-000000000002", + "trace_id": "00000000-0000-4000-8000-000000000004", + "parent_run_id": "00000000-0000-4000-8000-000000000004", + "name": "get_weather", + "run_type": "tool", + "start_time": "2026-07-10T12:00:03Z", + "end_time": "2026-07-10T12:00:04Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004.20260710T120003000000Z00000000-0000-4000-8000-000000000002", + "inputs": { "city": "Paris" }, + "outputs": { "output": { "temperature": 22, "condition": "Sunny" } } + }, + { + "id": "00000000-0000-4000-8000-000000000003", + "trace_id": "00000000-0000-4000-8000-000000000004", + "parent_run_id": "00000000-0000-4000-8000-000000000004", + "name": "ChatAnthropic", + "run_type": "llm", + "start_time": "2026-07-10T12:00:05Z", + "end_time": "2026-07-10T12:00:06Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004.20260710T120005000000Z00000000-0000-4000-8000-000000000003", + "extra": { "metadata": { "ls_provider": "anthropic", "ls_model_name": "claude-opus-4-7", "ls_message_format": "anthropic" } }, + "inputs": { + "system": "You are a helpful assistant.", + "messages": [ + { "role": "user", "content": "what is the weather in paris?" }, + { "role": "assistant", "content": [{ "type": "text", "text": "Let me check." }, { "type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": { "city": "Paris" } }] }, + { "role": "user", "content": [{ "type": "tool_result", "tool_use_id": "toolu_01", "content": "Sunny, 22C" }] } + ] + }, + "outputs": { + "message": { "id": "msg_02", "role": "assistant", "type": "message", "content": [{ "type": "text", "text": "It's sunny and 22°C in Paris." }] } + } + } +] diff --git a/fixtures/langsmith/official-openai-responses/expected.json b/fixtures/langsmith/official-openai-responses/expected.json new file mode 100644 index 0000000..a229195 --- /dev/null +++ b/fixtures/langsmith/official-openai-responses/expected.json @@ -0,0 +1,10 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "model": "gpt-4.1" }, + { "role": "user", "content": "what time is it in san francisco?", "timestamp": "2026-07-10T12:00:01.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "call_LVsl", "name": "get_time", "args": "{\"timezone\":\"America/Los_Angeles\"}" }], "timestamp": "2026-07-10T12:00:02.000Z" }, + { "role": "tool", "tool_call_id": "call_LVsl", "content": "12:00 PM (America/Los_Angeles)", "timestamp": "2026-07-10T12:00:04.000Z" }, + { "role": "assistant", "content": "It is currently 12:00 PM in San Francisco.", "timestamp": "2026-07-10T12:00:06.000Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/official-openai-responses/input.json b/fixtures/langsmith/official-openai-responses/input.json new file mode 100644 index 0000000..b548b62 --- /dev/null +++ b/fixtures/langsmith/official-openai-responses/input.json @@ -0,0 +1,65 @@ +[ + { + "id": "00000000-0000-4000-8000-000000000010", + "trace_id": "00000000-0000-4000-8000-000000000010", + "name": "Helpful Assistant", + "run_type": "chain", + "start_time": "2026-07-10T12:00:00Z", + "end_time": "2026-07-10T12:00:07Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010", + "inputs": { "input": "what time is it in san francisco?" }, + "outputs": { "output": "It is currently 12:00 PM in San Francisco." } + }, + { + "id": "00000000-0000-4000-8000-000000000011", + "trace_id": "00000000-0000-4000-8000-000000000010", + "parent_run_id": "00000000-0000-4000-8000-000000000010", + "name": "Helpful Assistant Response", + "run_type": "llm", + "start_time": "2026-07-10T12:00:01Z", + "end_time": "2026-07-10T12:00:02Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010.20260710T120001000000Z00000000-0000-4000-8000-000000000011", + "extra": { "metadata": { "ls_integration": "openai-agents-sdk", "ls_model_name": "gpt-4.1" } }, + "inputs": { + "instructions": "You are a helpful assistant.", + "input": [{ "role": "user", "content": "what time is it in san francisco?" }] + }, + "outputs": { + "output": [{ "type": "function_call", "call_id": "call_LVsl", "name": "get_time", "arguments": "{\"timezone\":\"America/Los_Angeles\"}", "id": "fc_0ed8" }] + } + }, + { + "id": "00000000-0000-4000-8000-000000000012", + "trace_id": "00000000-0000-4000-8000-000000000010", + "parent_run_id": "00000000-0000-4000-8000-000000000010", + "name": "get_time", + "run_type": "tool", + "start_time": "2026-07-10T12:00:03Z", + "end_time": "2026-07-10T12:00:04Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010.20260710T120003000000Z00000000-0000-4000-8000-000000000012", + "inputs": { "timezone": "America/Los_Angeles" }, + "outputs": { "output": "12:00 PM (America/Los_Angeles)", "call_id": "call_LVsl" } + }, + { + "id": "00000000-0000-4000-8000-000000000013", + "trace_id": "00000000-0000-4000-8000-000000000010", + "parent_run_id": "00000000-0000-4000-8000-000000000010", + "name": "Helpful Assistant Response", + "run_type": "llm", + "start_time": "2026-07-10T12:00:05Z", + "end_time": "2026-07-10T12:00:06Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010.20260710T120005000000Z00000000-0000-4000-8000-000000000013", + "extra": { "metadata": { "ls_integration": "openai-agents-sdk", "ls_model_name": "gpt-4.1" } }, + "inputs": { + "instructions": "You are a helpful assistant.", + "input": [ + { "role": "user", "content": "what time is it in san francisco?" }, + { "type": "function_call", "call_id": "call_LVsl", "name": "get_time", "arguments": "{\"timezone\":\"America/Los_Angeles\"}", "id": "fc_0ed8" }, + { "type": "function_call_output", "call_id": "call_LVsl", "output": "12:00 PM (America/Los_Angeles)" } + ] + }, + "outputs": { + "output": [{ "type": "message", "role": "assistant", "status": "completed", "content": [{ "type": "output_text", "text": "It is currently 12:00 PM in San Francisco.", "annotations": [] }] }] + } + } +] diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index dd8b13f..c7e565d 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -398,6 +398,8 @@ var langSmithAdapter = { let createdAt; for (const { run } of runs) { const metadata = runMetadata(run); + if (Object.hasOwn(metadata, "ls_message_view_exclude")) + continue; model ??= firstString(metadata.ls_model_name, metadata.model); cwd ??= firstString(metadata.cwd, metadata.working_directory); gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); @@ -703,11 +705,14 @@ function decodeContentBlock(block, role, stableId, index, timestamp, model) { } function messageItem(role, content, stableId, index, timestamp, model) { const canonical = role === "assistant" ? "assistant" : role === "user" ? "user" : role; - const key = stableId ? `message:${stableId}:${index}` : `message:${canonical}:${jsonString(content)}`; - if (canonical !== "user" && canonical !== "assistant") - return { key }; + const semanticKey = `message:${canonical}:${jsonString(content)}`; + const key = stableId ? `message:${stableId}:${index}` : semanticKey; + if (canonical !== "user" && canonical !== "assistant") { + return { key, semanticKey }; + } return { key, + semanticKey, ...stableId ? { stable: true } : {}, event: { type: "message", @@ -732,7 +737,7 @@ function reasoningItem(content, stableId, index, timestamp, model) { } function toolCallItem(value, stableId, index, timestamp, model) { const fn = isObject(value.function) ? value.function : {}; - const id = firstString(value.id, value.call_id, value.toolCallId); + const id = value.type === "function_call" ? firstString(value.call_id, value.id, value.toolCallId) : firstString(value.id, value.call_id, value.toolCallId); const name = firstString(value.name, value.toolName, fn.name); const rawArgs = firstDefined(value.arguments, value.args, value.input, fn.arguments); const args = typeof rawArgs === "string" ? rawArgs : jsonString(rawArgs); @@ -772,7 +777,7 @@ function decodeToolRun(run, timestamp, pendingCalls) { const name = firstString(inputs.toolName, inputs.tool_name, run.name); const call = matchPendingCall(pendingCalls, explicitId, name); const callId = explicitId ?? call?.id; - const rawResult = firstDefined(embedded?.content, embedded?.output, embedded?.result, outputs.result, outputs.output, outputs.content, run.error); + const rawResult = firstDefined(embedded?.content, embedded?.output, embedded?.result, outputs.result, outputs.output, outputs.content, run.error, Object.keys(outputs).length > 0 ? outputs : undefined); if (rawResult === undefined) return; let content = resultText(rawResult); @@ -814,7 +819,7 @@ function mergeItems(state, items) { let matches = true; const historyStart = state.history.length - overlap; for (let index = 0;index < overlap; index += 1) { - if (state.history[historyStart + index] !== items[index]?.key) { + if (!conversationItemsMatch(state.history[historyStart + index], items[index])) { matches = false; break; } @@ -823,16 +828,18 @@ function mergeItems(state, items) { break; overlap -= 1; } - const repeatedSnapshot = overlap === 0 && items.some((item) => item.stable === true && state.history.includes(item.key)); + const repeatedSnapshot = overlap === 0 && items.some((item) => item.stable === true && state.history.some((historyItem) => historyItem.key === item.key)); for (let index = overlap;index < items.length; index += 1) { const item = items[index]; if (!item) continue; - if (item.stable === true && state.history.includes(item.key)) + if (item.stable === true && state.history.some((historyItem) => historyItem.key === item.key)) { continue; - if (repeatedSnapshot && state.history.includes(item.key)) + } + if (repeatedSnapshot && state.history.some((historyItem) => conversationItemsMatch(historyItem, item))) { continue; - state.history.push(item.key); + } + state.history.push(item); if (!item.event) continue; if (isDuplicateAdjacentMessage(state.events.at(-1), item.event)) @@ -849,6 +856,13 @@ function mergeItems(state, items) { } } } +function conversationItemsMatch(previous, current) { + if (!previous || !current) + return false; + if (previous.key === current.key) + return true; + return previous.semanticKey !== undefined && previous.semanticKey === current.semanticKey && (previous.stable !== true || current.stable !== true); +} function isDuplicateAdjacentMessage(previous, current) { return previous?.type === "message" && current.type === "message" && previous.role === current.role && previous.content === current.content && previous.timestamp?.getTime() === current.timestamp?.getTime(); } diff --git a/python/tests/test_wrapper.py b/python/tests/test_wrapper.py index c46abf0..9174e7d 100644 --- a/python/tests/test_wrapper.py +++ b/python/tests/test_wrapper.py @@ -19,6 +19,8 @@ ("codex", "codex/cleanup", "input.jsonl"), ("langsmith", "langsmith/tool-call", "input.json"), ("langsmith", "langsmith/cleanup", "input.json"), + ("langsmith", "langsmith/official-openai-responses", "input.json"), + ("langsmith", "langsmith/official-anthropic", "input.json"), ("letta", "letta/tool-call", "input.json"), ("letta", "letta/cleanup", "input.json"), ("openhands", "openhands/tool-calls", "input.json"), diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index 7e7749e..da5624f 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -18,6 +18,7 @@ interface OrderedRun { interface ConversationItem { key: string; + semanticKey?: string; stable?: boolean; event?: DecodedEvent; } @@ -40,7 +41,7 @@ interface AnthropicStreamBlock { interface DecodeState { events: DecodedEvent[]; - history: string[]; + history: ConversationItem[]; pendingCalls: PendingCall[]; } @@ -61,6 +62,7 @@ export const langSmithAdapter: SourceAdapter = { for (const { run } of runs) { const metadata = runMetadata(run); + if (Object.hasOwn(metadata, "ls_message_view_exclude")) continue; model ??= firstString(metadata.ls_model_name, metadata.model); cwd ??= firstString(metadata.cwd, metadata.working_directory); gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); @@ -442,12 +444,16 @@ function messageItem( model?: string, ): ConversationItem { const canonical = role === "assistant" ? "assistant" : role === "user" ? "user" : role; + const semanticKey = `message:${canonical}:${jsonString(content)}`; const key = stableId ? `message:${stableId}:${index}` - : `message:${canonical}:${jsonString(content)}`; - if (canonical !== "user" && canonical !== "assistant") return { key }; + : semanticKey; + if (canonical !== "user" && canonical !== "assistant") { + return { key, semanticKey }; + } return { key, + semanticKey, ...(stableId ? { stable: true } : {}), event: { type: "message", @@ -488,7 +494,10 @@ function toolCallItem( model?: string, ): ConversationItem { const fn = isObject(value.function) ? value.function : {}; - const id = firstString(value.id, value.call_id, value.toolCallId); + const id = + value.type === "function_call" + ? firstString(value.call_id, value.id, value.toolCallId) + : firstString(value.id, value.call_id, value.toolCallId); const name = firstString(value.name, value.toolName, fn.name); const rawArgs = firstDefined(value.arguments, value.args, value.input, fn.arguments); const args = typeof rawArgs === "string" ? rawArgs : jsonString(rawArgs); @@ -566,6 +575,7 @@ function decodeToolRun( outputs.output, outputs.content, run.error, + Object.keys(outputs).length > 0 ? outputs : undefined, ); if (rawResult === undefined) return undefined; let content = resultText(rawResult); @@ -619,7 +629,7 @@ function mergeItems(state: DecodeState, items: ConversationItem[]): void { let matches = true; const historyStart = state.history.length - overlap; for (let index = 0; index < overlap; index += 1) { - if (state.history[historyStart + index] !== items[index]?.key) { + if (!conversationItemsMatch(state.history[historyStart + index], items[index])) { matches = false; break; } @@ -630,14 +640,28 @@ function mergeItems(state: DecodeState, items: ConversationItem[]): void { const repeatedSnapshot = overlap === 0 && - items.some((item) => item.stable === true && state.history.includes(item.key)); + items.some( + (item) => + item.stable === true && + state.history.some((historyItem) => historyItem.key === item.key), + ); for (let index = overlap; index < items.length; index += 1) { const item = items[index]; if (!item) continue; - if (item.stable === true && state.history.includes(item.key)) continue; - if (repeatedSnapshot && state.history.includes(item.key)) continue; - state.history.push(item.key); + if ( + item.stable === true && + state.history.some((historyItem) => historyItem.key === item.key) + ) { + continue; + } + if ( + repeatedSnapshot && + state.history.some((historyItem) => conversationItemsMatch(historyItem, item)) + ) { + continue; + } + state.history.push(item); if (!item.event) continue; if (isDuplicateAdjacentMessage(state.events.at(-1), item.event)) continue; state.events.push(item.event); @@ -653,6 +677,19 @@ function mergeItems(state: DecodeState, items: ConversationItem[]): void { } } +function conversationItemsMatch( + previous: ConversationItem | undefined, + current: ConversationItem | undefined, +): boolean { + if (!previous || !current) return false; + if (previous.key === current.key) return true; + return ( + previous.semanticKey !== undefined && + previous.semanticKey === current.semanticKey && + (previous.stable !== true || current.stable !== true) + ); +} + function isDuplicateAdjacentMessage( previous: DecodedEvent | undefined, current: DecodedEvent, diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 8d03ae0..c0e61f0 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -16,6 +16,8 @@ const fixtures = [ { source: "codex", name: "codex/cleanup" }, { source: "langsmith", name: "langsmith/tool-call" }, { source: "langsmith", name: "langsmith/cleanup" }, + { source: "langsmith", name: "langsmith/official-openai-responses" }, + { source: "langsmith", name: "langsmith/official-anthropic" }, { source: "letta", name: "letta/tool-call" }, { source: "letta", name: "letta/cleanup" }, { source: "openhands", name: "openhands/tool-calls" }, @@ -286,6 +288,79 @@ describe("public API", () => { ).toHaveLength(2); }); + test("serializes a bare LangSmith tool output object", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { input: [{ role: "user", content: "weather?" }] }, + outputs: { + output: [ + { + type: "function_call", + id: "fc-item", + call_id: "call-1", + name: "weather", + arguments: "{}", + }, + ], + }, + }), + canonicalLangSmithRun({ + id: "tool", + name: "weather", + run_type: "tool", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: {}, + outputs: { call_id: "call-1", temperature: 22 }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records).toContainEqual( + expect.objectContaining({ + role: "tool", + tool_call_id: "call-1", + content: '{"call_id":"call-1","temperature":22}', + }), + ); + expect(result.diagnostics).toEqual([]); + }); + + test("excludes LangSmith runs marked for exclusion from Messages view", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "internal", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + extra: { metadata: { ls_message_view_exclude: false } }, + inputs: { messages: [{ role: "user", content: "Classify this" }] }, + outputs: { role: "assistant", content: "billing" }, + }), + canonicalLangSmithRun({ + id: "visible", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: { messages: [{ role: "user", content: "Help me" }] }, + outputs: { role: "assistant", content: "Sure" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + expect(result.records).not.toContainEqual( + expect.objectContaining({ content: "Classify this" }), + ); + }); + test("decodes Anthropic reasoning and tool blocks from LangSmith runs", () => { const firstAssistant = { id: "msg-1", From 99bf6068907d2873d5f62b017135bcf040c0e4c1 Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Sat, 11 Jul 2026 16:37:39 -0700 Subject: [PATCH 10/10] Handle Deep Agents LangSmith aggregates --- PARITY.md | 7 + README.md | 7 + .../langsmith/deepagents-code/expected.json | 21 +++ fixtures/langsmith/deepagents-code/input.json | 82 ++++++++++ fixtures/langsmith/deepagents/expected.json | 49 ++++++ fixtures/langsmith/deepagents/input.json | 144 ++++++++++++++++++ .../src/trajectory/_vendor/trajectory-cli.mjs | 50 +++++- python/tests/test_wrapper.py | 2 + src/adapters/langsmith.ts | 90 ++++++++++- test/normalize.test.ts | 41 +++++ 10 files changed, 489 insertions(+), 4 deletions(-) create mode 100644 fixtures/langsmith/deepagents-code/expected.json create mode 100644 fixtures/langsmith/deepagents-code/input.json create mode 100644 fixtures/langsmith/deepagents/expected.json create mode 100644 fixtures/langsmith/deepagents/input.json diff --git a/PARITY.md b/PARITY.md index 33a932e..f42ff26 100644 --- a/PARITY.md +++ b/PARITY.md @@ -85,6 +85,13 @@ Messages. This differential check caught Responses item-ID versus call-ID linkage, mixed stable/semantic message deduplication, bare tool-output objects, and `ls_message_view_exclude` handling. +Deep Agents and Deep Agents Code have dedicated canonical Run fixtures. For +roots marked `ls_integration: deepagents` or `deepagents-code`, the adapter +normalizes the aggregate `outputs.messages` LangGraph state and ignores +redundant child spans. The fixtures cover reasoning, duplicated native tool-call +representations, tool results, final responses, model extraction, and prove +that Claude Code and Codex integration markers remain on the generic path. + ## Deep Agents SDK checkpoints The `deepagents` fixture is generated by Python diff --git a/README.md b/README.md index 94afa7c..c8855b0 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,13 @@ omits the ID. Runs carrying the `ls_message_view_exclude` metadata key are ignored, matching LangSmith's Messages-view behavior. Fetching or exporting runs from LangSmith remains the caller's responsibility. +Traces whose root metadata sets `ls_integration` to `deepagents` or +`deepagents-code` use the LangGraph aggregate stored in the root run's +`outputs.messages`. Their redundant child LLM and tool spans are ignored, which +prevents accumulated message history from appearing more than once. No other +integration receives this aggregate-root special case; all other traces use the +generic LangSmith run decoder described above. + LangSmith's Anthropic wrapper aggregates stream events before storing outputs, but that reducer is not exported as a public SDK utility. When canonical Run data contains a raw Anthropic SSE string in `outputs.output`, the adapter diff --git a/fixtures/langsmith/deepagents-code/expected.json b/fixtures/langsmith/deepagents-code/expected.json new file mode 100644 index 0000000..5434720 --- /dev/null +++ b/fixtures/langsmith/deepagents-code/expected.json @@ -0,0 +1,21 @@ +{ + "records": [ + { + "role": "meta", + "source": "langsmith", + "cwd": "/workspace/deepagents-code", + "model": "gpt-5.5" + }, + { + "role": "user", + "content": "Reply briefly.", + "timestamp": "2026-07-11T13:00:00.000Z" + }, + { + "role": "assistant", + "content": "Done.", + "timestamp": "2026-07-11T13:00:02.000Z" + } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/deepagents-code/input.json b/fixtures/langsmith/deepagents-code/input.json new file mode 100644 index 0000000..a979cab --- /dev/null +++ b/fixtures/langsmith/deepagents-code/input.json @@ -0,0 +1,82 @@ +[ + { + "id": "trace-deepagents-code-1", + "trace_id": "trace-deepagents-code-1", + "name": "LangGraph", + "run_type": "chain", + "start_time": "2026-07-11T13:00:00.000Z", + "end_time": "2026-07-11T13:00:02.000Z", + "dotted_order": "20260711T130000000000Ztrace-deepagents-code-1", + "inputs": { + "messages": [ + { + "role": "user", + "content": "Reply briefly." + } + ] + }, + "outputs": { + "messages": [ + { + "id": "human-code-1", + "type": "human", + "content": "Reply briefly." + }, + { + "id": "ai-code-1", + "type": "ai", + "content": "Done.", + "response_metadata": { + "model_name": "gpt-5.5" + } + } + ] + }, + "extra": { + "metadata": { + "ls_agent_kind": "coding_agent", + "ls_integration": "deepagents-code", + "ls_trace_schema_version": "coding-agent-v1", + "cwd": "/workspace/deepagents-code" + } + } + }, + { + "id": "child-code-llm-1", + "trace_id": "trace-deepagents-code-1", + "parent_run_id": "trace-deepagents-code-1", + "name": "ChatOpenAI", + "run_type": "llm", + "start_time": "2026-07-11T13:00:00.500Z", + "end_time": "2026-07-11T13:00:01.500Z", + "dotted_order": "20260711T130000000000Ztrace-deepagents-code-1.20260711T130000500000Zchild-code-llm-1", + "inputs": { + "messages": [ + [ + { + "role": "user", + "content": "Duplicated child input" + } + ] + ] + }, + "outputs": { + "generations": [ + [ + { + "message": { + "role": "assistant", + "content": "Duplicated child output" + } + } + ] + ] + }, + "extra": { + "metadata": { + "ls_integration": "deepagents-code", + "ls_model_name": "gpt-5.5" + } + } + } +] diff --git a/fixtures/langsmith/deepagents/expected.json b/fixtures/langsmith/deepagents/expected.json new file mode 100644 index 0000000..440ff18 --- /dev/null +++ b/fixtures/langsmith/deepagents/expected.json @@ -0,0 +1,49 @@ +{ + "records": [ + { + "role": "meta", + "source": "langsmith", + "cwd": "/workspace/deepagents", + "model": "claude-sonnet-4-6" + }, + { + "role": "user", + "content": "What is the weather in Paris?", + "timestamp": "2026-07-11T12:00:00.000Z" + }, + { + "role": "reasoning", + "content": "I should check the weather tool.", + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "assistant", + "content": "I’ll check.", + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call-weather-1", + "name": "get_weather", + "args": "{\"city\":\"Paris\"}" + } + ], + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "tool", + "tool_call_id": "call-weather-1", + "content": "Sunny, 22 C", + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "assistant", + "content": "It is sunny and 22 C in Paris.", + "timestamp": "2026-07-11T12:00:05.000Z" + } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/deepagents/input.json b/fixtures/langsmith/deepagents/input.json new file mode 100644 index 0000000..8051f20 --- /dev/null +++ b/fixtures/langsmith/deepagents/input.json @@ -0,0 +1,144 @@ +[ + { + "id": "trace-deepagents-1", + "trace_id": "trace-deepagents-1", + "name": "LangGraph", + "run_type": "chain", + "start_time": "2026-07-11T12:00:00.000Z", + "end_time": "2026-07-11T12:00:05.000Z", + "dotted_order": "20260711T120000000000Ztrace-deepagents-1", + "inputs": { + "messages": [ + { + "role": "user", + "content": "What is the weather in Paris?" + } + ] + }, + "outputs": { + "messages": [ + { + "id": "human-1", + "type": "human", + "content": "What is the weather in Paris?" + }, + { + "id": "ai-1", + "type": "ai", + "content": [ + { + "type": "reasoning", + "reasoning": "I should check the weather tool." + }, + { + "type": "text", + "text": "I’ll check." + }, + { + "type": "tool_use", + "id": "call-weather-1", + "name": "get_weather", + "input": { + "city": "Paris" + } + } + ], + "tool_calls": [ + { + "id": "call-weather-1", + "name": "get_weather", + "args": { + "city": "Paris" + }, + "type": "tool_call" + } + ], + "response_metadata": { + "model_name": "claude-sonnet-4-6" + } + }, + { + "id": "tool-1", + "type": "tool", + "name": "get_weather", + "tool_call_id": "call-weather-1", + "content": "Sunny, 22 C" + }, + { + "id": "ai-2", + "type": "ai", + "content": "It is sunny and 22 C in Paris.", + "tool_calls": [], + "response_metadata": { + "model_name": "claude-sonnet-4-6" + } + } + ] + }, + "extra": { + "metadata": { + "ls_integration": "deepagents", + "cwd": "/workspace/deepagents" + } + } + }, + { + "id": "child-llm-1", + "trace_id": "trace-deepagents-1", + "parent_run_id": "trace-deepagents-1", + "name": "ChatAnthropic", + "run_type": "llm", + "start_time": "2026-07-11T12:00:01.000Z", + "end_time": "2026-07-11T12:00:02.000Z", + "dotted_order": "20260711T120000000000Ztrace-deepagents-1.20260711T120001000000Zchild-llm-1", + "inputs": { + "messages": [ + [ + { + "role": "user", + "content": "This accumulated child history must be ignored." + } + ] + ] + }, + "outputs": { + "generations": [ + [ + { + "message": { + "role": "assistant", + "content": "This child output must be ignored." + } + } + ] + ] + }, + "extra": { + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_model_name": "claude-sonnet-4-6" + } + } + }, + { + "id": "child-tool-1", + "trace_id": "trace-deepagents-1", + "parent_run_id": "trace-deepagents-1", + "name": "get_weather", + "run_type": "tool", + "start_time": "2026-07-11T12:00:02.000Z", + "end_time": "2026-07-11T12:00:03.000Z", + "dotted_order": "20260711T120000000000Ztrace-deepagents-1.20260711T120002000000Zchild-tool-1", + "inputs": { + "city": "Paris" + }, + "outputs": { + "output": "This child tool output must be ignored." + }, + "extra": { + "metadata": { + "ls_integration": "deepagents" + } + } + } +] diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index 866ca36..1ce621e 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -398,6 +398,7 @@ var langSmithAdapter = { source: "langsmith", decode(transcript) { const runs = parseRuns(transcript).sort(compareRuns); + const deepAgentsRoots = findDeepAgentsRoots(runs); const state = { events: [], history: [], @@ -408,15 +409,27 @@ var langSmithAdapter = { let gitBranch; let createdAt; for (const { run } of runs) { + const traceId = firstString(run.trace_id); + const deepAgentsRoot = traceId ? deepAgentsRoots.get(traceId) : undefined; + if (deepAgentsRoot && run !== deepAgentsRoot) + continue; const metadata = runMetadata(run); if (Object.hasOwn(metadata, "ls_message_view_exclude")) continue; - model ??= firstString(metadata.ls_model_name, metadata.model); + const aggregateOutputs = deepAgentsRoot ? outputMessages(run.outputs) : []; + const aggregateModel = deepAgentsRoot ? modelFromMessages(aggregateOutputs) : undefined; + model ??= firstString(metadata.ls_model_name, metadata.model, aggregateModel); cwd ??= firstString(metadata.cwd, metadata.working_directory); gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); const start = parseTimestamp(run.start_time); const end = parseTimestamp(run.end_time) ?? start; createdAt ??= start; + if (deepAgentsRoot) { + const runModel = firstString(metadata.ls_model_name, metadata.model, aggregateModel, model); + mergeItems(state, decodeMessages(inputMessages(run.inputs), start, runModel)); + mergeItems(state, decodeMessages(aggregateOutputs, end, runModel, "assistant")); + continue; + } if (run.run_type === "llm") { const runModel = firstString(metadata.ls_model_name, metadata.model, model); mergeItems(state, decodeMessages(inputMessages(run.inputs), start, runModel)); @@ -442,6 +455,26 @@ var langSmithAdapter = { }; } }; +function findDeepAgentsRoots(runs) { + const roots = new Map; + for (const { run } of runs) { + const traceId = firstString(run.trace_id); + if (!traceId || roots.has(traceId) || !isRootRun(run)) + continue; + if (!isDeepAgentsIntegration(runMetadata(run))) + continue; + if (outputMessages(run.outputs).length === 0) + continue; + roots.set(traceId, run); + } + return roots; +} +function isRootRun(run) { + return run.id === run.trace_id || run.parent_run_id == null; +} +function isDeepAgentsIntegration(metadata) { + return metadata.ls_integration === "deepagents" || metadata.ls_integration === "deepagents-code"; +} function parseRuns(transcript) { let parsed; try { @@ -639,6 +672,19 @@ function messageArray(value, fallbackRole) { return value[0]; return value; } +function modelFromMessages(messages) { + for (const message of messages) { + if (!isObject(message)) + continue; + const body = isObject(message.kwargs) ? message.kwargs : message; + const responseMetadata = isObject(body.response_metadata) ? body.response_metadata : {}; + const additional = isObject(body.additional_kwargs) ? body.additional_kwargs : {}; + const model = firstString(body.model, body.model_name, responseMetadata.model_name, responseMetadata.model, additional.model_name, additional.model); + if (model) + return model; + } + return; +} function decodeMessages(messages, timestamp, model, fallbackRole) { const items = []; for (const message of messages) { @@ -914,7 +960,7 @@ function contentText(value) { return ""; } function reasoningText(value) { - const direct = firstString(value.thinking, value.text, value.content); + const direct = firstString(value.reasoning, value.thinking, value.text, value.content); if (direct) return direct; if (Array.isArray(value.summary)) { diff --git a/python/tests/test_wrapper.py b/python/tests/test_wrapper.py index 9a4863f..2bea6a6 100644 --- a/python/tests/test_wrapper.py +++ b/python/tests/test_wrapper.py @@ -29,6 +29,8 @@ ("langsmith", "langsmith/cleanup", "input.json"), ("langsmith", "langsmith/official-openai-responses", "input.json"), ("langsmith", "langsmith/official-anthropic", "input.json"), + ("langsmith", "langsmith/deepagents", "input.json"), + ("langsmith", "langsmith/deepagents-code", "input.json"), ("letta", "letta/tool-call", "input.json"), ("letta", "letta/cleanup", "input.json"), ("letta", "letta/local-v3", "input.jsonl"), diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts index da5624f..44ce49c 100644 --- a/src/adapters/langsmith.ts +++ b/src/adapters/langsmith.ts @@ -50,6 +50,7 @@ export const langSmithAdapter: SourceAdapter = { decode(transcript: string): DecodedSession { const runs = parseRuns(transcript).sort(compareRuns); + const deepAgentsRoots = findDeepAgentsRoots(runs); const state: DecodeState = { events: [], history: [], @@ -61,15 +62,47 @@ export const langSmithAdapter: SourceAdapter = { let createdAt: Date | undefined; for (const { run } of runs) { + const traceId = firstString(run.trace_id); + const deepAgentsRoot = traceId ? deepAgentsRoots.get(traceId) : undefined; + if (deepAgentsRoot && run !== deepAgentsRoot) continue; + const metadata = runMetadata(run); if (Object.hasOwn(metadata, "ls_message_view_exclude")) continue; - model ??= firstString(metadata.ls_model_name, metadata.model); + const aggregateOutputs = deepAgentsRoot + ? outputMessages(run.outputs) + : []; + const aggregateModel = deepAgentsRoot + ? modelFromMessages(aggregateOutputs) + : undefined; + model ??= firstString( + metadata.ls_model_name, + metadata.model, + aggregateModel, + ); cwd ??= firstString(metadata.cwd, metadata.working_directory); gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); const start = parseTimestamp(run.start_time); const end = parseTimestamp(run.end_time) ?? start; createdAt ??= start; + if (deepAgentsRoot) { + const runModel = firstString( + metadata.ls_model_name, + metadata.model, + aggregateModel, + model, + ); + mergeItems( + state, + decodeMessages(inputMessages(run.inputs), start, runModel), + ); + mergeItems( + state, + decodeMessages(aggregateOutputs, end, runModel, "assistant"), + ); + continue; + } + if (run.run_type === "llm") { const runModel = firstString( metadata.ls_model_name, @@ -107,6 +140,31 @@ export const langSmithAdapter: SourceAdapter = { }, }; +function findDeepAgentsRoots( + runs: OrderedRun[], +): Map> { + const roots = new Map>(); + for (const { run } of runs) { + const traceId = firstString(run.trace_id); + if (!traceId || roots.has(traceId) || !isRootRun(run)) continue; + if (!isDeepAgentsIntegration(runMetadata(run))) continue; + if (outputMessages(run.outputs).length === 0) continue; + roots.set(traceId, run); + } + return roots; +} + +function isRootRun(run: Record): boolean { + return run.id === run.trace_id || run.parent_run_id == null; +} + +function isDeepAgentsIntegration( + metadata: Record, +): boolean { + return metadata.ls_integration === "deepagents" || + metadata.ls_integration === "deepagents-code"; +} + function parseRuns(transcript: string): OrderedRun[] { let parsed: unknown; try { @@ -330,6 +388,29 @@ function messageArray(value: unknown, fallbackRole: string): unknown[] { return value; } +function modelFromMessages(messages: unknown[]): string | undefined { + for (const message of messages) { + if (!isObject(message)) continue; + const body = isObject(message.kwargs) ? message.kwargs : message; + const responseMetadata = isObject(body.response_metadata) + ? body.response_metadata + : {}; + const additional = isObject(body.additional_kwargs) + ? body.additional_kwargs + : {}; + const model = firstString( + body.model, + body.model_name, + responseMetadata.model_name, + responseMetadata.model, + additional.model_name, + additional.model, + ); + if (model) return model; + } + return undefined; +} + function decodeMessages( messages: unknown[], timestamp?: Date, @@ -733,7 +814,12 @@ function contentText(value: unknown): string { } function reasoningText(value: Record): string { - const direct = firstString(value.thinking, value.text, value.content); + const direct = firstString( + value.reasoning, + value.thinking, + value.text, + value.content, + ); if (direct) return direct; if (Array.isArray(value.summary)) { return value.summary diff --git a/test/normalize.test.ts b/test/normalize.test.ts index c88c953..90a09c4 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -18,6 +18,8 @@ const fixtures = [ { source: "langsmith", name: "langsmith/cleanup" }, { source: "langsmith", name: "langsmith/official-openai-responses" }, { source: "langsmith", name: "langsmith/official-anthropic" }, + { source: "langsmith", name: "langsmith/deepagents" }, + { source: "langsmith", name: "langsmith/deepagents-code" }, { source: "letta", name: "letta/tool-call" }, { source: "letta", name: "letta/cleanup" }, { source: "letta", name: "letta/local-v3" }, @@ -135,6 +137,45 @@ describe("public API", () => { ); }); + for (const integration of ["claude-code", "openai-codex"] as const) { + test(`keeps ${integration} LangSmith traces on the generic path`, () => { + const root = canonicalLangSmithRun({ + id: "trace-test", + trace_id: "trace-test", + run_type: "chain", + inputs: { messages: [{ role: "user", content: "root input" }] }, + outputs: { + messages: [ + { role: "user", content: "root input" }, + { role: "assistant", content: "root aggregate" }, + ], + }, + extra: { metadata: { ls_integration: integration } }, + }); + const child = canonicalLangSmithRun({ + id: "child-llm", + trace_id: "trace-test", + parent_run_id: "trace-test", + inputs: { messages: [{ role: "user", content: "generic input" }] }, + outputs: { message: { role: "assistant", content: "generic output" } }, + }); + + const result = normalizeTranscript({ + source: "langsmith", + transcript: JSON.stringify([root, child]), + }); + const contents = result.records.flatMap((record) => + "content" in record && typeof record.content === "string" + ? [record.content] + : [], + ); + + expect(contents).toContain("generic input"); + expect(contents).toContain("generic output"); + expect(contents).not.toContain("root aggregate"); + }); + } + test("reconstructs Anthropic SSE stored as string output", () => { const output = [ "event: message_start",