diff --git a/PARITY.md b/PARITY.md index 6d79149..f42ff26 100644 --- a/PARITY.md +++ b/PARITY.md @@ -53,6 +53,45 @@ shapes were checked against the `dream-pipeline` OpenHands source. Both the array and `{items: [...]}` input forms produced exact production-equivalent records in the compatibility fixtures. +## LangSmith adapter + +The LangSmith adapter is covered with synthetic edge cases and canonical Run +trees adapted from LangSmith's published Messages-view examples. The official +OpenAI Responses and Anthropic examples are stored with root spans, UUIDs, +timestamps, `extra.metadata`, and hierarchical `dotted_order` values so their +normalized fixtures require no repairs or diagnostics. Fixtures also exercise +LangChain constructor messages, Vercel AI SDK content blocks, repeated history +snapshots, excluded internal runs, bare tool outputs, tool-result matching by ID +and tool name, run ordering, metadata, and missing-timestamp repair. + +Read-only validation was also run against two user-provided LangSmith projects +without retaining raw traces or normalized content in the repository. The +sample comprised one two-trace thread with 17 total runs (13 chain, two LLM, +and two tool runs) and one standalone LLM run. This surfaced and fixed two +native variants not present in the original synthetic fixtures: an Anthropic +SSE event stream stored in string-valued `outputs.output`, and a tool call +repeated in both content blocks and the message-level `tool_calls` field. + +After those repairs, the combined thread normalized to 15 records with native +tool linkage preserved; its only diagnostic was the expected configured +tool-result truncation. The standalone LLM run normalized to three records +without diagnostics. + +The official examples were taken from LangSmith's +[Messages-view trace format reference](https://docs.langchain.com/langsmith/messages-view-trace-format). +All five published integration traces were also exercised directly: LangChain, +OpenAI Chat Completions, OpenAI Responses, Vercel AI SDK, and Anthropic +Messages. This differential check caught Responses item-ID versus call-ID +linkage, mixed stable/semantic message deduplication, bare tool-output objects, +and `ls_message_view_exclude` handling. + +Deep Agents and Deep Agents Code have dedicated canonical Run fixtures. For +roots marked `ls_integration: deepagents` or `deepagents-code`, the adapter +normalizes the aggregate `outputs.messages` LangGraph state and ignores +redundant child spans. The fixtures cover reasoning, duplicated native tool-call +representations, tool results, final responses, model extraction, and prove +that Claude Code and Codex integration markers remain on the generic path. + ## Deep Agents SDK checkpoints The `deepagents` fixture is generated by Python diff --git a/README.md b/README.md index 1f3a53b..c8855b0 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ and is empty when the transcript required no recoverable cleanup. | --- | --- | --- | | `claude-code` | Native Claude Code JSONL | `claude-code` | | `codex` | Native Codex rollout JSONL | `codex` | +| `langsmith` | Canonical LangSmith `Run` records as JSON array or `{ "runs": [...] }` | `langsmith` | | `letta` | Cloud/API message array or local conversation JSONL (legacy and v3) | `letta` | | `openhands` | JSON event array or an events-API `{ "items": [...] }` envelope | `openhands` | | `deepagents` | User-supplied Python LangGraph `SqliteSaver` database plus `threadId` | `deepagents` | @@ -130,6 +131,40 @@ not a supported native input. OpenHands inputs are serialized exports; when a native store uses individual event files, assembling the event array remains the caller's responsibility. +The LangSmith input is the canonical public `Run` span format returned by the +SDK or `/runs/query`: a JSON array of runs, or the API's `{ "runs": [...] }` +envelope, for one trace or chronological thread. Each run must include `id`, +`trace_id`, `name`, `run_type`, and `inputs`. A trace is represented by a root +run and child runs linked through `trace_id`, `parent_run_id`, and +`dotted_order`; LangSmith does not define a separate flattened-conversation +schema. Runs are ordered by `dotted_order` and `start_time`. + +Run `inputs` and `outputs` remain integration-specific. The adapter follows the +LangSmith Messages-view formats for LangChain/LangGraph, OpenAI Chat +Completions and Responses, Anthropic Messages, and the Vercel AI SDK. Repeated +message-history snapshots are deduplicated, while tool runs are linked to the +earlier model tool call by call ID and then by tool name when an integration +omits the ID. Runs carrying the `ls_message_view_exclude` metadata key are +ignored, matching LangSmith's Messages-view behavior. Fetching or exporting +runs from LangSmith remains the caller's responsibility. + +Traces whose root metadata sets `ls_integration` to `deepagents` or +`deepagents-code` use the LangGraph aggregate stored in the root run's +`outputs.messages`. Their redundant child LLM and tool spans are ignored, which +prevents accumulated message history from appearing more than once. No other +integration receives this aggregate-root special case; all other traces use the +generic LangSmith run decoder described above. + +LangSmith's Anthropic wrapper aggregates stream events before storing outputs, +but that reducer is not exported as a public SDK utility. When canonical Run +data contains a raw Anthropic SSE string in `outputs.output`, the adapter +reconstructs its text, thinking, and tool-input deltas locally. + +For a multi-turn thread, combine the runs from each member trace into one input +container before normalization. This preserves history and tool linkage that +cross trace boundaries; normalizing each trace separately can correctly report +an initial tool result as orphaned when its call occurred in the prior trace. + ## Normalized records A trajectory is an ordered array containing: diff --git a/fixtures/langsmith/cleanup/expected.json b/fixtures/langsmith/cleanup/expected.json new file mode 100644 index 0000000..0086626 --- /dev/null +++ b/fixtures/langsmith/cleanup/expected.json @@ -0,0 +1,13 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "cwd": "/workspace", "git_branch": "feature/weather", "model": "claude-sonnet-4" }, + { "role": "user", "content": "Check Paris weather.", "timestamp": "2026-07-10T13:00:00.000Z" }, + { "role": "assistant", "content": "I'll check.", "timestamp": "2026-07-10T13:00:01.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "call-vercel", "name": "weather", "args": "{\"city\":\"Paris\"}" }], "timestamp": "2026-07-10T13:00:01.000Z" }, + { "role": "tool", "tool_call_id": "call-vercel", "content": "{\"condition\":\"sunny\"}", "timestamp": "2026-07-10T13:00:03.000Z" }, + { "role": "assistant", "content": "It is sunny in Paris.", "timestamp": "2026-07-10T13:00:04.000Z" } + ], + "diagnostics": [ + { "code": "timestamps_interpolated", "message": "Interpolated timestamps for 1 normalized records.", "count": 1 } + ] +} diff --git a/fixtures/langsmith/cleanup/input.json b/fixtures/langsmith/cleanup/input.json new file mode 100644 index 0000000..bbbfcf4 --- /dev/null +++ b/fixtures/langsmith/cleanup/input.json @@ -0,0 +1 @@ +[{"trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","start_time":"2026-07-10T13:00:00Z","end_time":"2026-07-10T13:00:01Z","inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]},"id":"vercel-1","extra":{"metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4","cwd":"/workspace","git_branch":"feature/weather"}}},{"trace_id":"trace-2","parent_run_id":"vercel-1","run_type":"tool","name":"weather","start_time":"2026-07-10T13:00:02Z","end_time":"2026-07-10T13:00:03Z","inputs":{"args":{"city":"Paris"}},"outputs":{"result":{"condition":"sunny"}},"id":"vercel-tool"},{"trace_id":"trace-2","run_type":"llm","name":"ai.doGenerate","inputs":{"prompt":[{"role":"user","content":[{"type":"text","text":"Check Paris weather."}]},{"role":"assistant","content":[{"type":"text","text":"I'll check."},{"type":"tool-call","toolCallId":"call-vercel","toolName":"weather","input":{"city":"Paris"}}]},{"role":"tool","toolCallId":"call-vercel","content":{"condition":"sunny"}}]},"outputs":{"role":"assistant","content":[{"type":"text","text":"It is sunny in Paris."}]},"id":"vercel-2","extra":{"metadata":{"ls_integration":"vercel-ai-sdk","ls_model_name":"claude-sonnet-4"}}}] diff --git a/fixtures/langsmith/deepagents-code/expected.json b/fixtures/langsmith/deepagents-code/expected.json new file mode 100644 index 0000000..5434720 --- /dev/null +++ b/fixtures/langsmith/deepagents-code/expected.json @@ -0,0 +1,21 @@ +{ + "records": [ + { + "role": "meta", + "source": "langsmith", + "cwd": "/workspace/deepagents-code", + "model": "gpt-5.5" + }, + { + "role": "user", + "content": "Reply briefly.", + "timestamp": "2026-07-11T13:00:00.000Z" + }, + { + "role": "assistant", + "content": "Done.", + "timestamp": "2026-07-11T13:00:02.000Z" + } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/deepagents-code/input.json b/fixtures/langsmith/deepagents-code/input.json new file mode 100644 index 0000000..a979cab --- /dev/null +++ b/fixtures/langsmith/deepagents-code/input.json @@ -0,0 +1,82 @@ +[ + { + "id": "trace-deepagents-code-1", + "trace_id": "trace-deepagents-code-1", + "name": "LangGraph", + "run_type": "chain", + "start_time": "2026-07-11T13:00:00.000Z", + "end_time": "2026-07-11T13:00:02.000Z", + "dotted_order": "20260711T130000000000Ztrace-deepagents-code-1", + "inputs": { + "messages": [ + { + "role": "user", + "content": "Reply briefly." + } + ] + }, + "outputs": { + "messages": [ + { + "id": "human-code-1", + "type": "human", + "content": "Reply briefly." + }, + { + "id": "ai-code-1", + "type": "ai", + "content": "Done.", + "response_metadata": { + "model_name": "gpt-5.5" + } + } + ] + }, + "extra": { + "metadata": { + "ls_agent_kind": "coding_agent", + "ls_integration": "deepagents-code", + "ls_trace_schema_version": "coding-agent-v1", + "cwd": "/workspace/deepagents-code" + } + } + }, + { + "id": "child-code-llm-1", + "trace_id": "trace-deepagents-code-1", + "parent_run_id": "trace-deepagents-code-1", + "name": "ChatOpenAI", + "run_type": "llm", + "start_time": "2026-07-11T13:00:00.500Z", + "end_time": "2026-07-11T13:00:01.500Z", + "dotted_order": "20260711T130000000000Ztrace-deepagents-code-1.20260711T130000500000Zchild-code-llm-1", + "inputs": { + "messages": [ + [ + { + "role": "user", + "content": "Duplicated child input" + } + ] + ] + }, + "outputs": { + "generations": [ + [ + { + "message": { + "role": "assistant", + "content": "Duplicated child output" + } + } + ] + ] + }, + "extra": { + "metadata": { + "ls_integration": "deepagents-code", + "ls_model_name": "gpt-5.5" + } + } + } +] diff --git a/fixtures/langsmith/deepagents/expected.json b/fixtures/langsmith/deepagents/expected.json new file mode 100644 index 0000000..440ff18 --- /dev/null +++ b/fixtures/langsmith/deepagents/expected.json @@ -0,0 +1,49 @@ +{ + "records": [ + { + "role": "meta", + "source": "langsmith", + "cwd": "/workspace/deepagents", + "model": "claude-sonnet-4-6" + }, + { + "role": "user", + "content": "What is the weather in Paris?", + "timestamp": "2026-07-11T12:00:00.000Z" + }, + { + "role": "reasoning", + "content": "I should check the weather tool.", + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "assistant", + "content": "I’ll check.", + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call-weather-1", + "name": "get_weather", + "args": "{\"city\":\"Paris\"}" + } + ], + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "tool", + "tool_call_id": "call-weather-1", + "content": "Sunny, 22 C", + "timestamp": "2026-07-11T12:00:05.000Z" + }, + { + "role": "assistant", + "content": "It is sunny and 22 C in Paris.", + "timestamp": "2026-07-11T12:00:05.000Z" + } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/deepagents/input.json b/fixtures/langsmith/deepagents/input.json new file mode 100644 index 0000000..8051f20 --- /dev/null +++ b/fixtures/langsmith/deepagents/input.json @@ -0,0 +1,144 @@ +[ + { + "id": "trace-deepagents-1", + "trace_id": "trace-deepagents-1", + "name": "LangGraph", + "run_type": "chain", + "start_time": "2026-07-11T12:00:00.000Z", + "end_time": "2026-07-11T12:00:05.000Z", + "dotted_order": "20260711T120000000000Ztrace-deepagents-1", + "inputs": { + "messages": [ + { + "role": "user", + "content": "What is the weather in Paris?" + } + ] + }, + "outputs": { + "messages": [ + { + "id": "human-1", + "type": "human", + "content": "What is the weather in Paris?" + }, + { + "id": "ai-1", + "type": "ai", + "content": [ + { + "type": "reasoning", + "reasoning": "I should check the weather tool." + }, + { + "type": "text", + "text": "I’ll check." + }, + { + "type": "tool_use", + "id": "call-weather-1", + "name": "get_weather", + "input": { + "city": "Paris" + } + } + ], + "tool_calls": [ + { + "id": "call-weather-1", + "name": "get_weather", + "args": { + "city": "Paris" + }, + "type": "tool_call" + } + ], + "response_metadata": { + "model_name": "claude-sonnet-4-6" + } + }, + { + "id": "tool-1", + "type": "tool", + "name": "get_weather", + "tool_call_id": "call-weather-1", + "content": "Sunny, 22 C" + }, + { + "id": "ai-2", + "type": "ai", + "content": "It is sunny and 22 C in Paris.", + "tool_calls": [], + "response_metadata": { + "model_name": "claude-sonnet-4-6" + } + } + ] + }, + "extra": { + "metadata": { + "ls_integration": "deepagents", + "cwd": "/workspace/deepagents" + } + } + }, + { + "id": "child-llm-1", + "trace_id": "trace-deepagents-1", + "parent_run_id": "trace-deepagents-1", + "name": "ChatAnthropic", + "run_type": "llm", + "start_time": "2026-07-11T12:00:01.000Z", + "end_time": "2026-07-11T12:00:02.000Z", + "dotted_order": "20260711T120000000000Ztrace-deepagents-1.20260711T120001000000Zchild-llm-1", + "inputs": { + "messages": [ + [ + { + "role": "user", + "content": "This accumulated child history must be ignored." + } + ] + ] + }, + "outputs": { + "generations": [ + [ + { + "message": { + "role": "assistant", + "content": "This child output must be ignored." + } + } + ] + ] + }, + "extra": { + "metadata": { + "ls_integration": "langchain_chat_model", + "ls_model_name": "claude-sonnet-4-6" + } + } + }, + { + "id": "child-tool-1", + "trace_id": "trace-deepagents-1", + "parent_run_id": "trace-deepagents-1", + "name": "get_weather", + "run_type": "tool", + "start_time": "2026-07-11T12:00:02.000Z", + "end_time": "2026-07-11T12:00:03.000Z", + "dotted_order": "20260711T120000000000Ztrace-deepagents-1.20260711T120002000000Zchild-tool-1", + "inputs": { + "city": "Paris" + }, + "outputs": { + "output": "This child tool output must be ignored." + }, + "extra": { + "metadata": { + "ls_integration": "deepagents" + } + } + } +] diff --git a/fixtures/langsmith/official-anthropic/expected.json b/fixtures/langsmith/official-anthropic/expected.json new file mode 100644 index 0000000..342587b --- /dev/null +++ b/fixtures/langsmith/official-anthropic/expected.json @@ -0,0 +1,11 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "model": "claude-opus-4-7" }, + { "role": "user", "content": "what is the weather in paris?", "timestamp": "2026-07-10T12:00:01.000Z" }, + { "role": "assistant", "content": "Let me check.", "timestamp": "2026-07-10T12:00:02.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "toolu_01", "name": "get_weather", "args": "{\"city\":\"Paris\"}" }], "timestamp": "2026-07-10T12:00:02.000Z" }, + { "role": "tool", "tool_call_id": "toolu_01", "content": "{\"temperature\":22,\"condition\":\"Sunny\"}", "timestamp": "2026-07-10T12:00:04.000Z" }, + { "role": "assistant", "content": "It's sunny and 22°C in Paris.", "timestamp": "2026-07-10T12:00:06.000Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/official-anthropic/input.json b/fixtures/langsmith/official-anthropic/input.json new file mode 100644 index 0000000..4f7a91c --- /dev/null +++ b/fixtures/langsmith/official-anthropic/input.json @@ -0,0 +1,73 @@ +[ + { + "id": "00000000-0000-4000-8000-000000000004", + "trace_id": "00000000-0000-4000-8000-000000000004", + "name": "Weather Agent", + "run_type": "chain", + "start_time": "2026-07-10T12:00:00Z", + "end_time": "2026-07-10T12:00:07Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004", + "inputs": { "input": "what is the weather in paris?" }, + "outputs": { "output": "It's sunny and 22°C in Paris." } + }, + { + "id": "00000000-0000-4000-8000-000000000001", + "trace_id": "00000000-0000-4000-8000-000000000004", + "parent_run_id": "00000000-0000-4000-8000-000000000004", + "name": "ChatAnthropic", + "run_type": "llm", + "start_time": "2026-07-10T12:00:01Z", + "end_time": "2026-07-10T12:00:02Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004.20260710T120001000000Z00000000-0000-4000-8000-000000000001", + "extra": { "metadata": { "ls_provider": "anthropic", "ls_model_name": "claude-opus-4-7", "ls_message_format": "anthropic" } }, + "inputs": { + "system": "You are a helpful assistant.", + "messages": [{ "role": "user", "content": "what is the weather in paris?" }] + }, + "outputs": { + "message": { + "id": "msg_01", + "role": "assistant", + "type": "message", + "content": [ + { "type": "text", "text": "Let me check." }, + { "type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": { "city": "Paris" } } + ] + } + } + }, + { + "id": "00000000-0000-4000-8000-000000000002", + "trace_id": "00000000-0000-4000-8000-000000000004", + "parent_run_id": "00000000-0000-4000-8000-000000000004", + "name": "get_weather", + "run_type": "tool", + "start_time": "2026-07-10T12:00:03Z", + "end_time": "2026-07-10T12:00:04Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004.20260710T120003000000Z00000000-0000-4000-8000-000000000002", + "inputs": { "city": "Paris" }, + "outputs": { "output": { "temperature": 22, "condition": "Sunny" } } + }, + { + "id": "00000000-0000-4000-8000-000000000003", + "trace_id": "00000000-0000-4000-8000-000000000004", + "parent_run_id": "00000000-0000-4000-8000-000000000004", + "name": "ChatAnthropic", + "run_type": "llm", + "start_time": "2026-07-10T12:00:05Z", + "end_time": "2026-07-10T12:00:06Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000004.20260710T120005000000Z00000000-0000-4000-8000-000000000003", + "extra": { "metadata": { "ls_provider": "anthropic", "ls_model_name": "claude-opus-4-7", "ls_message_format": "anthropic" } }, + "inputs": { + "system": "You are a helpful assistant.", + "messages": [ + { "role": "user", "content": "what is the weather in paris?" }, + { "role": "assistant", "content": [{ "type": "text", "text": "Let me check." }, { "type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": { "city": "Paris" } }] }, + { "role": "user", "content": [{ "type": "tool_result", "tool_use_id": "toolu_01", "content": "Sunny, 22C" }] } + ] + }, + "outputs": { + "message": { "id": "msg_02", "role": "assistant", "type": "message", "content": [{ "type": "text", "text": "It's sunny and 22°C in Paris." }] } + } + } +] diff --git a/fixtures/langsmith/official-openai-responses/expected.json b/fixtures/langsmith/official-openai-responses/expected.json new file mode 100644 index 0000000..a229195 --- /dev/null +++ b/fixtures/langsmith/official-openai-responses/expected.json @@ -0,0 +1,10 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "model": "gpt-4.1" }, + { "role": "user", "content": "what time is it in san francisco?", "timestamp": "2026-07-10T12:00:01.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "call_LVsl", "name": "get_time", "args": "{\"timezone\":\"America/Los_Angeles\"}" }], "timestamp": "2026-07-10T12:00:02.000Z" }, + { "role": "tool", "tool_call_id": "call_LVsl", "content": "12:00 PM (America/Los_Angeles)", "timestamp": "2026-07-10T12:00:04.000Z" }, + { "role": "assistant", "content": "It is currently 12:00 PM in San Francisco.", "timestamp": "2026-07-10T12:00:06.000Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/official-openai-responses/input.json b/fixtures/langsmith/official-openai-responses/input.json new file mode 100644 index 0000000..b548b62 --- /dev/null +++ b/fixtures/langsmith/official-openai-responses/input.json @@ -0,0 +1,65 @@ +[ + { + "id": "00000000-0000-4000-8000-000000000010", + "trace_id": "00000000-0000-4000-8000-000000000010", + "name": "Helpful Assistant", + "run_type": "chain", + "start_time": "2026-07-10T12:00:00Z", + "end_time": "2026-07-10T12:00:07Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010", + "inputs": { "input": "what time is it in san francisco?" }, + "outputs": { "output": "It is currently 12:00 PM in San Francisco." } + }, + { + "id": "00000000-0000-4000-8000-000000000011", + "trace_id": "00000000-0000-4000-8000-000000000010", + "parent_run_id": "00000000-0000-4000-8000-000000000010", + "name": "Helpful Assistant Response", + "run_type": "llm", + "start_time": "2026-07-10T12:00:01Z", + "end_time": "2026-07-10T12:00:02Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010.20260710T120001000000Z00000000-0000-4000-8000-000000000011", + "extra": { "metadata": { "ls_integration": "openai-agents-sdk", "ls_model_name": "gpt-4.1" } }, + "inputs": { + "instructions": "You are a helpful assistant.", + "input": [{ "role": "user", "content": "what time is it in san francisco?" }] + }, + "outputs": { + "output": [{ "type": "function_call", "call_id": "call_LVsl", "name": "get_time", "arguments": "{\"timezone\":\"America/Los_Angeles\"}", "id": "fc_0ed8" }] + } + }, + { + "id": "00000000-0000-4000-8000-000000000012", + "trace_id": "00000000-0000-4000-8000-000000000010", + "parent_run_id": "00000000-0000-4000-8000-000000000010", + "name": "get_time", + "run_type": "tool", + "start_time": "2026-07-10T12:00:03Z", + "end_time": "2026-07-10T12:00:04Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010.20260710T120003000000Z00000000-0000-4000-8000-000000000012", + "inputs": { "timezone": "America/Los_Angeles" }, + "outputs": { "output": "12:00 PM (America/Los_Angeles)", "call_id": "call_LVsl" } + }, + { + "id": "00000000-0000-4000-8000-000000000013", + "trace_id": "00000000-0000-4000-8000-000000000010", + "parent_run_id": "00000000-0000-4000-8000-000000000010", + "name": "Helpful Assistant Response", + "run_type": "llm", + "start_time": "2026-07-10T12:00:05Z", + "end_time": "2026-07-10T12:00:06Z", + "dotted_order": "20260710T120000000000Z00000000-0000-4000-8000-000000000010.20260710T120005000000Z00000000-0000-4000-8000-000000000013", + "extra": { "metadata": { "ls_integration": "openai-agents-sdk", "ls_model_name": "gpt-4.1" } }, + "inputs": { + "instructions": "You are a helpful assistant.", + "input": [ + { "role": "user", "content": "what time is it in san francisco?" }, + { "type": "function_call", "call_id": "call_LVsl", "name": "get_time", "arguments": "{\"timezone\":\"America/Los_Angeles\"}", "id": "fc_0ed8" }, + { "type": "function_call_output", "call_id": "call_LVsl", "output": "12:00 PM (America/Los_Angeles)" } + ] + }, + "outputs": { + "output": [{ "type": "message", "role": "assistant", "status": "completed", "content": [{ "type": "output_text", "text": "It is currently 12:00 PM in San Francisco.", "annotations": [] }] }] + } + } +] diff --git a/fixtures/langsmith/tool-call/expected.json b/fixtures/langsmith/tool-call/expected.json new file mode 100644 index 0000000..b50ad68 --- /dev/null +++ b/fixtures/langsmith/tool-call/expected.json @@ -0,0 +1,10 @@ +{ + "records": [ + { "role": "meta", "source": "langsmith", "model": "gpt-4o" }, + { "role": "user", "content": "What is the weather?", "timestamp": "2026-07-10T12:00:00.000Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "call-weather", "name": "get_weather", "args": "{\"city\":\"Paris\"}" }], "timestamp": "2026-07-10T12:00:01.000Z" }, + { "role": "tool", "tool_call_id": "call-weather", "content": "Sunny, 22C", "timestamp": "2026-07-10T12:00:03.000Z" }, + { "role": "assistant", "content": "It is sunny and 22C in Paris.", "timestamp": "2026-07-10T12:00:05.000Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/langsmith/tool-call/input.json b/fixtures/langsmith/tool-call/input.json new file mode 100644 index 0000000..83524da --- /dev/null +++ b/fixtures/langsmith/tool-call/input.json @@ -0,0 +1 @@ +[{"id":"llm-2","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:04Z","end_time":"2026-07-10T12:00:05Z","dotted_order":"20260710T120004000000Zllm-2","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"It is sunny and 22C in Paris.","id":"ai-2"}}}]]}},{"id":"llm-1","trace_id":"trace-1","run_type":"llm","name":"ChatOpenAI","start_time":"2026-07-10T12:00:00Z","end_time":"2026-07-10T12:00:01Z","dotted_order":"20260710T120000000000Zllm-1","extra":{"metadata":{"ls_integration":"langchain_chat_model","ls_model_name":"gpt-4o"}},"inputs":{"messages":[[{"lc":1,"type":"constructor","id":["langchain","schema","messages","SystemMessage"],"kwargs":{"content":"Be helpful.","id":"sys-1"}},{"lc":1,"type":"constructor","id":["langchain","schema","messages","HumanMessage"],"kwargs":{"content":"What is the weather?","id":"user-1"}}]]},"outputs":{"generations":[[{"message":{"lc":1,"type":"constructor","id":["langchain","schema","messages","AIMessage"],"kwargs":{"content":"","id":"ai-1","tool_calls":[{"name":"get_weather","args":{"city":"Paris"},"id":"call-weather","type":"tool_call"}]}}}]]}},{"id":"tool-run-1","trace_id":"trace-1","parent_run_id":"llm-1","run_type":"tool","name":"get_weather","start_time":"2026-07-10T12:00:02Z","end_time":"2026-07-10T12:00:03Z","dotted_order":"20260710T120002000000Ztool-run-1","inputs":{"city":"Paris"},"outputs":{"output":{"lc":1,"type":"constructor","id":["langchain","schema","messages","ToolMessage"],"kwargs":{"content":"Sunny, 22C","tool_call_id":"call-weather","id":"tool-1"}}}}] diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index db70ec6..1ce621e 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -393,6 +393,607 @@ class NormalizationError extends Error { } } +// src/adapters/langsmith.ts +var langSmithAdapter = { + source: "langsmith", + decode(transcript) { + const runs = parseRuns(transcript).sort(compareRuns); + const deepAgentsRoots = findDeepAgentsRoots(runs); + const state = { + events: [], + history: [], + pendingCalls: [] + }; + let model; + let cwd; + let gitBranch; + let createdAt; + for (const { run } of runs) { + const traceId = firstString(run.trace_id); + const deepAgentsRoot = traceId ? deepAgentsRoots.get(traceId) : undefined; + if (deepAgentsRoot && run !== deepAgentsRoot) + continue; + const metadata = runMetadata(run); + if (Object.hasOwn(metadata, "ls_message_view_exclude")) + continue; + const aggregateOutputs = deepAgentsRoot ? outputMessages(run.outputs) : []; + const aggregateModel = deepAgentsRoot ? modelFromMessages(aggregateOutputs) : undefined; + model ??= firstString(metadata.ls_model_name, metadata.model, aggregateModel); + cwd ??= firstString(metadata.cwd, metadata.working_directory); + gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); + const start = parseTimestamp(run.start_time); + const end = parseTimestamp(run.end_time) ?? start; + createdAt ??= start; + if (deepAgentsRoot) { + const runModel = firstString(metadata.ls_model_name, metadata.model, aggregateModel, model); + mergeItems(state, decodeMessages(inputMessages(run.inputs), start, runModel)); + mergeItems(state, decodeMessages(aggregateOutputs, end, runModel, "assistant")); + continue; + } + if (run.run_type === "llm") { + const runModel = firstString(metadata.ls_model_name, metadata.model, model); + mergeItems(state, decodeMessages(inputMessages(run.inputs), start, runModel)); + mergeItems(state, decodeMessages(outputMessages(run.outputs), end, runModel, "assistant")); + continue; + } + if (run.run_type === "tool") { + const result = decodeToolRun(run, end, state.pendingCalls); + if (result) + mergeItems(state, [result]); + } + } + return { + events: state.events, + context: { + source: "langsmith", + ...cwd ? { cwd } : {}, + ...gitBranch ? { gitBranch } : {}, + ...model ? { model } : {}, + ...createdAt ? { createdAt } : {} + }, + diagnostics: [] + }; + } +}; +function findDeepAgentsRoots(runs) { + const roots = new Map; + for (const { run } of runs) { + const traceId = firstString(run.trace_id); + if (!traceId || roots.has(traceId) || !isRootRun(run)) + continue; + if (!isDeepAgentsIntegration(runMetadata(run))) + continue; + if (outputMessages(run.outputs).length === 0) + continue; + roots.set(traceId, run); + } + return roots; +} +function isRootRun(run) { + return run.id === run.trace_id || run.parent_run_id == null; +} +function isDeepAgentsIntegration(metadata) { + return metadata.ls_integration === "deepagents" || metadata.ls_integration === "deepagents-code"; +} +function parseRuns(transcript) { + let parsed; + try { + parsed = JSON.parse(transcript); + } catch { + throw invalidTranscript(); + } + const runs = Array.isArray(parsed) ? parsed : isObject(parsed) && Array.isArray(parsed.runs) ? parsed.runs : undefined; + if (!runs || runs.length === 0 || !runs.every(isCanonicalRun)) { + throw invalidTranscript(); + } + return runs.map((run, index) => ({ run, index })); +} +function isCanonicalRun(value) { + return isObject(value) && typeof value.id === "string" && typeof value.trace_id === "string" && typeof value.name === "string" && typeof value.run_type === "string" && isObject(value.inputs); +} +function invalidTranscript() { + return new NormalizationError("invalid_input", "LangSmith transcript must be a non-empty JSON array of canonical Run records or an object with a runs array."); +} +function compareRuns(left, right) { + const leftOrder = left.run.dotted_order; + const rightOrder = right.run.dotted_order; + if (typeof leftOrder === "string" && typeof rightOrder === "string") { + const order = leftOrder.localeCompare(rightOrder); + if (order !== 0) + return order; + } + const leftTime = parseTimestamp(left.run.start_time)?.getTime(); + const rightTime = parseTimestamp(right.run.start_time)?.getTime(); + if (leftTime !== undefined && rightTime !== undefined && leftTime !== rightTime) { + return leftTime - rightTime; + } + return left.index - right.index; +} +function runMetadata(run) { + const extra = isObject(run.extra) ? run.extra : {}; + return isObject(extra.metadata) ? extra.metadata : {}; +} +function inputMessages(value) { + if (!isObject(value)) + return []; + const messages = []; + if (value.system !== undefined) { + messages.push({ role: "system", content: value.system }); + } + if (value.instructions !== undefined) { + messages.push({ role: "system", content: value.instructions }); + } + const candidate = firstNonEmpty(value.messages, value.input, value.prompt); + if (candidate !== undefined) { + messages.push(...messageArray(candidate, "user")); + } else if (Array.isArray(value.prompts)) { + for (const prompt of value.prompts) { + if (typeof prompt === "string") + messages.push({ role: "user", content: prompt }); + } + } + return messages; +} +function outputMessages(value) { + if (!isObject(value)) { + if (typeof value !== "string") + return []; + const streamed = anthropicStreamMessages(value); + return streamed.length > 0 ? streamed : [{ role: "assistant", content: value }]; + } + if (Array.isArray(value.generations)) { + const firstBatch = Array.isArray(value.generations[0]) ? value.generations[0] : value.generations; + const messages = []; + for (const generation of firstBatch) { + if (!isObject(generation)) + continue; + if (generation.message !== undefined) + messages.push(generation.message); + else if (typeof generation.text === "string") { + messages.push({ role: "assistant", content: generation.text }); + } + } + if (messages.length > 0) + return messages; + } + if (Array.isArray(value.choices)) { + const messages = value.choices.map((choice) => isObject(choice) ? choice.message : undefined).filter((message) => message !== undefined); + if (messages.length > 0) + return messages; + } + if (Array.isArray(value.messages)) + return value.messages; + if (isObject(value.message)) + return [value.message]; + if (Array.isArray(value.output)) + return value.output; + if (typeof value.output === "string") { + const streamed = anthropicStreamMessages(value.output); + return streamed.length > 0 ? streamed : [{ role: "assistant", content: value.output }]; + } + if (isObject(value.output)) { + if (Array.isArray(value.output.messages)) + return value.output.messages; + if (isObject(value.output.update) && Array.isArray(value.output.update.messages)) { + return value.output.update.messages; + } + } + if (value.role !== undefined || value.type === "message") + return [value]; + if (typeof value.content === "string" || Array.isArray(value.content)) { + return [{ role: "assistant", content: value.content }]; + } + return []; +} +function anthropicStreamMessages(output) { + if (!output.includes("event:") || !output.includes(` +data:`)) + return []; + const blocks = new Map; + for (const line of output.split(` +`)) { + if (!line.startsWith("data:")) + continue; + const raw = line.slice("data:".length).trim(); + if (!raw || raw === "[DONE]") + continue; + let event; + try { + event = JSON.parse(raw); + } catch { + continue; + } + if (!isObject(event) || typeof event.index !== "number") + continue; + if (event.type === "content_block_start" && isObject(event.content_block)) { + const content2 = event.content_block; + if (typeof content2.type !== "string") + continue; + blocks.set(event.index, { + type: content2.type, + ...typeof content2.text === "string" ? { text: content2.text } : {}, + ...typeof content2.thinking === "string" ? { thinking: content2.thinking } : {}, + ...typeof content2.id === "string" ? { id: content2.id } : {}, + ...typeof content2.name === "string" ? { name: content2.name } : {}, + ...content2.input !== undefined ? { input: content2.input } : {}, + partialJson: "" + }); + continue; + } + if (event.type !== "content_block_delta" || !isObject(event.delta)) { + continue; + } + const delta = event.delta; + const block = blocks.get(event.index); + if (!block) + continue; + if (delta.type === "text_delta" && typeof delta.text === "string") { + block.text = (block.text ?? "") + delta.text; + } else if (delta.type === "thinking_delta" && typeof delta.thinking === "string") { + block.thinking = (block.thinking ?? "") + delta.thinking; + } else if (delta.type === "input_json_delta" && typeof delta.partial_json === "string") { + block.partialJson += delta.partial_json; + } + } + const content = []; + for (const [, block] of [...blocks].sort(([left], [right]) => left - right)) { + if (block.type === "text" && block.text) { + content.push({ type: "text", text: block.text }); + } else if (block.type === "thinking" && block.thinking) { + content.push({ type: "thinking", thinking: block.thinking }); + } else if (block.type === "tool_use") { + content.push({ + type: "tool_use", + ...block.id ? { id: block.id } : {}, + ...block.name ? { name: block.name } : {}, + input: streamedToolInput(block) + }); + } + } + return content.length > 0 ? [{ role: "assistant", content }] : []; +} +function streamedToolInput(block) { + if (!block.partialJson) + return block.input ?? {}; + try { + return JSON.parse(block.partialJson); + } catch { + return { _raw: block.partialJson }; + } +} +function messageArray(value, fallbackRole) { + if (typeof value === "string") + return [{ role: fallbackRole, content: value }]; + if (isObject(value)) + return [value]; + if (!Array.isArray(value)) + return []; + if (value.length === 1 && Array.isArray(value[0])) + return value[0]; + return value; +} +function modelFromMessages(messages) { + for (const message of messages) { + if (!isObject(message)) + continue; + const body = isObject(message.kwargs) ? message.kwargs : message; + const responseMetadata = isObject(body.response_metadata) ? body.response_metadata : {}; + const additional = isObject(body.additional_kwargs) ? body.additional_kwargs : {}; + const model = firstString(body.model, body.model_name, responseMetadata.model_name, responseMetadata.model, additional.model_name, additional.model); + if (model) + return model; + } + return; +} +function decodeMessages(messages, timestamp, model, fallbackRole) { + const items = []; + for (const message of messages) { + items.push(...decodeMessage(message, timestamp, model, fallbackRole)); + } + return items; +} +function decodeMessage(value, timestamp, model, fallbackRole) { + if (typeof value === "string") { + return [messageItem(fallbackRole ?? "assistant", value, undefined, 0, timestamp, model)]; + } + if (!isObject(value)) + return []; + const constructorClass = constructorName(value); + const body = isObject(value.kwargs) ? value.kwargs : value; + const stableId = firstString(body.id, value.id); + const role = canonicalRole(firstString(body.role, body.type, fallbackRole), constructorClass); + const type = firstString(body.type, value.type); + if (type === "function_call") { + return [toolCallItem(body, stableId, 0, timestamp, model)]; + } + if (type === "function_call_output") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + if (type === "reasoning") { + const text = reasoningText(body); + return text ? [reasoningItem(text, stableId, 0, timestamp, model)] : []; + } + if (role === "tool") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + const items = []; + const content = body.content; + if (Array.isArray(content)) { + for (let index = 0;index < content.length; index += 1) { + const block = content[index]; + if (typeof block === "string") { + items.push(messageItem(role, block, stableId, index, timestamp, model)); + } else if (isObject(block)) { + items.push(...decodeContentBlock(block, role, stableId, index, timestamp, model)); + } + } + } else { + const text = contentText(content); + if (text) + items.push(messageItem(role, text, stableId, 0, timestamp, model)); + } + if (role === "assistant" && Array.isArray(body.tool_calls)) { + for (let index = 0;index < body.tool_calls.length; index += 1) { + const call = body.tool_calls[index]; + if (isObject(call)) { + items.push(toolCallItem(call, stableId, index + items.length, timestamp, model)); + } + } + } + return items; +} +function decodeContentBlock(block, role, stableId, index, timestamp, model) { + const type = block.type; + if (type === "tool_use" || type === "tool-call" || type === "function_call") { + return [toolCallItem(block, stableId, index, timestamp, model)]; + } + if (type === "tool_result" || type === "tool-result" || type === "function_call_output") { + return [toolResultItem(block, stableId, index, timestamp)]; + } + if (type === "thinking" || type === "reasoning" || type === "redacted_thinking") { + const text2 = reasoningText(block); + return text2 ? [reasoningItem(text2, stableId, index, timestamp, model)] : []; + } + if (type === "image" || type === "image_url") { + return [messageItem(role, "[image]", stableId, index, timestamp, model)]; + } + const text = contentText(block.text ?? block.content); + return text ? [messageItem(role, text, stableId, index, timestamp, model)] : []; +} +function messageItem(role, content, stableId, index, timestamp, model) { + const canonical = role === "assistant" ? "assistant" : role === "user" ? "user" : role; + const semanticKey = `message:${canonical}:${jsonString(content)}`; + const key = stableId ? `message:${stableId}:${index}` : semanticKey; + if (canonical !== "user" && canonical !== "assistant") { + return { key, semanticKey }; + } + return { + key, + semanticKey, + ...stableId ? { stable: true } : {}, + event: { + type: "message", + role: canonical, + content, + ...timestamp ? { timestamp } : {}, + ...model ? { model } : {} + } + }; +} +function reasoningItem(content, stableId, index, timestamp, model) { + return { + key: stableId ? `reasoning:${stableId}:${index}` : `reasoning:${jsonString(content)}`, + ...stableId ? { stable: true } : {}, + event: { + type: "reasoning", + content, + ...timestamp ? { timestamp } : {}, + ...model ? { model } : {} + } + }; +} +function toolCallItem(value, stableId, index, timestamp, model) { + const fn = isObject(value.function) ? value.function : {}; + const id = value.type === "function_call" ? firstString(value.call_id, value.id, value.toolCallId) : firstString(value.id, value.call_id, value.toolCallId); + const name = firstString(value.name, value.toolName, fn.name); + const rawArgs = firstDefined(value.arguments, value.args, value.input, fn.arguments); + const args = typeof rawArgs === "string" ? rawArgs : jsonString(rawArgs); + return { + key: id ? `tool-call:${id}` : `tool-call:${stableId ?? ""}:${index}:${name ?? ""}:${args}`, + ...id || stableId ? { stable: true } : {}, + event: { + type: "tool_call", + args, + ...id ? { id } : {}, + ...name ? { name } : {}, + ...timestamp ? { timestamp } : {}, + ...model ? { model } : {} + } + }; +} +function toolResultItem(value, stableId, index, timestamp) { + const id = firstString(value.tool_call_id, value.tool_use_id, value.call_id, value.toolCallId); + const raw = firstDefined(value.output, value.result, value.content); + const content = resultText(raw); + return { + key: id ? `tool-result:${id}` : `tool-result:${stableId ?? ""}:${index}:${content}`, + ...id || stableId ? { stable: true } : {}, + event: { + type: "tool_result", + content, + ...id ? { callId: id } : {}, + ...timestamp ? { timestamp } : {} + } + }; +} +function decodeToolRun(run, timestamp, pendingCalls) { + const inputs = isObject(run.inputs) ? run.inputs : {}; + const outputs = isObject(run.outputs) ? run.outputs : {}; + const embedded = embeddedToolMessage(outputs); + const explicitId = firstString(inputs.toolCallId, inputs.tool_call_id, inputs.call_id, outputs.toolCallId, outputs.tool_call_id, outputs.call_id, embedded?.tool_call_id, embedded?.tool_use_id, embedded?.call_id); + const name = firstString(inputs.toolName, inputs.tool_name, run.name); + const call = matchPendingCall(pendingCalls, explicitId, name); + const callId = explicitId ?? call?.id; + const rawResult = firstDefined(embedded?.content, embedded?.output, embedded?.result, outputs.result, outputs.output, outputs.content, run.error, Object.keys(outputs).length > 0 ? outputs : undefined); + if (rawResult === undefined) + return; + let content = resultText(rawResult); + if (run.error && rawResult === run.error && !/^error/i.test(content)) { + content = `Error: ${content}`; + } + return { + key: callId ? `tool-result:${callId}` : `tool-run:${firstString(run.id) ?? "unknown"}`, + stable: true, + event: { + type: "tool_result", + content, + ...callId ? { callId } : {}, + ...timestamp ? { timestamp } : {} + } + }; +} +function embeddedToolMessage(outputs) { + const output = isObject(outputs.output) ? outputs.output : undefined; + const candidate = output ?? (isObject(outputs.message) ? outputs.message : undefined); + if (!candidate) + return; + return isObject(candidate.kwargs) ? candidate.kwargs : candidate; +} +function matchPendingCall(pendingCalls, id, name) { + let match = id ? pendingCalls.find((call) => !call.consumed && call.id === id) : undefined; + match ??= name ? pendingCalls.find((call) => !call.consumed && call.name === name) : undefined; + const remaining = pendingCalls.filter((call) => !call.consumed); + match ??= remaining.length === 1 ? remaining[0] : undefined; + if (match) + match.consumed = true; + return match; +} +function mergeItems(state, items) { + if (items.length === 0) + return; + let overlap = Math.min(state.history.length, items.length); + while (overlap > 0) { + let matches = true; + const historyStart = state.history.length - overlap; + for (let index = 0;index < overlap; index += 1) { + if (!conversationItemsMatch(state.history[historyStart + index], items[index])) { + matches = false; + break; + } + } + if (matches) + break; + overlap -= 1; + } + const repeatedSnapshot = overlap === 0 && items.some((item) => item.stable === true && state.history.some((historyItem) => historyItem.key === item.key)); + for (let index = overlap;index < items.length; index += 1) { + const item = items[index]; + if (!item) + continue; + if (item.stable === true && state.history.some((historyItem) => historyItem.key === item.key)) { + continue; + } + if (repeatedSnapshot && state.history.some((historyItem) => conversationItemsMatch(historyItem, item))) { + continue; + } + state.history.push(item); + if (!item.event) + continue; + if (isDuplicateAdjacentMessage(state.events.at(-1), item.event)) + continue; + state.events.push(item.event); + if (item.event.type === "tool_call") { + state.pendingCalls.push({ + ...item.event.id ? { id: item.event.id } : {}, + ...item.event.name ? { name: item.event.name } : {}, + consumed: false + }); + } else if (item.event.type === "tool_result" && item.event.callId) { + matchPendingCall(state.pendingCalls, item.event.callId, undefined); + } + } +} +function conversationItemsMatch(previous, current) { + if (!previous || !current) + return false; + if (previous.key === current.key) + return true; + return previous.semanticKey !== undefined && previous.semanticKey === current.semanticKey && (previous.stable !== true || current.stable !== true); +} +function isDuplicateAdjacentMessage(previous, current) { + return previous?.type === "message" && current.type === "message" && previous.role === current.role && previous.content === current.content && previous.timestamp?.getTime() === current.timestamp?.getTime(); +} +function canonicalRole(value, constructorClass) { + if (constructorClass === "HumanMessage" || constructorClass === "ChatMessage") { + return "user"; + } + if (constructorClass === "AIMessage") + return "assistant"; + if (constructorClass === "ToolMessage" || constructorClass === "FunctionMessage") { + return "tool"; + } + if (constructorClass === "SystemMessage") + return "system"; + if (value === "human") + return "user"; + if (value === "ai") + return "assistant"; + if (value === "function") + return "tool"; + return value ?? "assistant"; +} +function constructorName(value) { + if (!Array.isArray(value.id)) + return; + const last = value.id[value.id.length - 1]; + return typeof last === "string" ? last : undefined; +} +function contentText(value) { + if (typeof value === "string") + return value; + if (value === null || value === undefined) + return ""; + if (Array.isArray(value)) + return blocksText(value); + if (isObject(value) && typeof value.value === "string") + return value.value; + return ""; +} +function reasoningText(value) { + const direct = firstString(value.reasoning, value.thinking, value.text, value.content); + if (direct) + return direct; + if (Array.isArray(value.summary)) { + return value.summary.map((item) => isObject(item) ? contentText(item.text ?? item.content) : "").filter(Boolean).join(` +`); + } + return ""; +} +function resultText(value) { + if (typeof value === "string") + return value; + if (value === undefined || value === null) + return ""; + if (Array.isArray(value)) + return blocksText(value) || jsonString(value); + if (isObject(value)) { + if (isObject(value.kwargs)) + return resultText(value.kwargs.content); + if (typeof value.content === "string") + return value.content; + } + return jsonString(value); +} +function firstString(...values) { + return values.find((value) => typeof value === "string" && value.length > 0); +} +function firstDefined(...values) { + return values.find((value) => value !== undefined && value !== null); +} +function firstNonEmpty(...values) { + return values.find((value) => value !== undefined && value !== null && (!Array.isArray(value) || value.length > 0)); +} + // src/adapters/letta.ts var lettaAdapter = { source: "letta", @@ -1685,6 +2286,7 @@ function isToolCall(value) { var ADAPTERS = { "claude-code": claudeCodeAdapter, codex: codexAdapter, + langsmith: langSmithAdapter, letta: lettaAdapter, openhands: openHandsAdapter }; diff --git a/python/tests/test_wrapper.py b/python/tests/test_wrapper.py index fb50d26..2bea6a6 100644 --- a/python/tests/test_wrapper.py +++ b/python/tests/test_wrapper.py @@ -25,6 +25,12 @@ ("claude-code", "claude-code/cleanup", "input.jsonl"), ("codex", "codex/tool-calls", "input.jsonl"), ("codex", "codex/cleanup", "input.jsonl"), + ("langsmith", "langsmith/tool-call", "input.json"), + ("langsmith", "langsmith/cleanup", "input.json"), + ("langsmith", "langsmith/official-openai-responses", "input.json"), + ("langsmith", "langsmith/official-anthropic", "input.json"), + ("langsmith", "langsmith/deepagents", "input.json"), + ("langsmith", "langsmith/deepagents-code", "input.json"), ("letta", "letta/tool-call", "input.json"), ("letta", "letta/cleanup", "input.json"), ("letta", "letta/local-v3", "input.jsonl"), @@ -68,7 +74,7 @@ def test_normalization_errors_include_code_and_batch_index(self) -> None: "source": "codex", "transcript": fixture_text("codex/cleanup", "input.jsonl"), } - invalid = {"source": "langsmith", "transcript": "{}"} + invalid = {"source": "not-a-source", "transcript": "{}"} with self.assertRaises(NormalizationError) as raised: normalize_many([valid, invalid]) diff --git a/src/adapters/langsmith.ts b/src/adapters/langsmith.ts new file mode 100644 index 0000000..44ce49c --- /dev/null +++ b/src/adapters/langsmith.ts @@ -0,0 +1,859 @@ +import type { + DecodedEvent, + DecodedSession, + SourceAdapter, +} from "../internal.js"; +import { NormalizationError } from "../types.js"; +import { + blocksText, + isObject, + jsonString, + parseTimestamp, +} from "./shared.js"; + +interface OrderedRun { + run: Record; + index: number; +} + +interface ConversationItem { + key: string; + semanticKey?: string; + stable?: boolean; + event?: DecodedEvent; +} + +interface PendingCall { + id?: string; + name?: string; + consumed: boolean; +} + +interface AnthropicStreamBlock { + type: string; + text?: string; + thinking?: string; + id?: string; + name?: string; + input?: unknown; + partialJson: string; +} + +interface DecodeState { + events: DecodedEvent[]; + history: ConversationItem[]; + pendingCalls: PendingCall[]; +} + +export const langSmithAdapter: SourceAdapter = { + source: "langsmith", + + decode(transcript: string): DecodedSession { + const runs = parseRuns(transcript).sort(compareRuns); + const deepAgentsRoots = findDeepAgentsRoots(runs); + const state: DecodeState = { + events: [], + history: [], + pendingCalls: [], + }; + let model: string | undefined; + let cwd: string | undefined; + let gitBranch: string | undefined; + let createdAt: Date | undefined; + + for (const { run } of runs) { + const traceId = firstString(run.trace_id); + const deepAgentsRoot = traceId ? deepAgentsRoots.get(traceId) : undefined; + if (deepAgentsRoot && run !== deepAgentsRoot) continue; + + const metadata = runMetadata(run); + if (Object.hasOwn(metadata, "ls_message_view_exclude")) continue; + const aggregateOutputs = deepAgentsRoot + ? outputMessages(run.outputs) + : []; + const aggregateModel = deepAgentsRoot + ? modelFromMessages(aggregateOutputs) + : undefined; + model ??= firstString( + metadata.ls_model_name, + metadata.model, + aggregateModel, + ); + cwd ??= firstString(metadata.cwd, metadata.working_directory); + gitBranch ??= firstString(metadata.git_branch, metadata.gitBranch); + const start = parseTimestamp(run.start_time); + const end = parseTimestamp(run.end_time) ?? start; + createdAt ??= start; + + if (deepAgentsRoot) { + const runModel = firstString( + metadata.ls_model_name, + metadata.model, + aggregateModel, + model, + ); + mergeItems( + state, + decodeMessages(inputMessages(run.inputs), start, runModel), + ); + mergeItems( + state, + decodeMessages(aggregateOutputs, end, runModel, "assistant"), + ); + continue; + } + + if (run.run_type === "llm") { + const runModel = firstString( + metadata.ls_model_name, + metadata.model, + model, + ); + mergeItems( + state, + decodeMessages(inputMessages(run.inputs), start, runModel), + ); + mergeItems( + state, + decodeMessages(outputMessages(run.outputs), end, runModel, "assistant"), + ); + continue; + } + + if (run.run_type === "tool") { + const result = decodeToolRun(run, end, state.pendingCalls); + if (result) mergeItems(state, [result]); + } + } + + return { + events: state.events, + context: { + source: "langsmith", + ...(cwd ? { cwd } : {}), + ...(gitBranch ? { gitBranch } : {}), + ...(model ? { model } : {}), + ...(createdAt ? { createdAt } : {}), + }, + diagnostics: [], + }; + }, +}; + +function findDeepAgentsRoots( + runs: OrderedRun[], +): Map> { + const roots = new Map>(); + for (const { run } of runs) { + const traceId = firstString(run.trace_id); + if (!traceId || roots.has(traceId) || !isRootRun(run)) continue; + if (!isDeepAgentsIntegration(runMetadata(run))) continue; + if (outputMessages(run.outputs).length === 0) continue; + roots.set(traceId, run); + } + return roots; +} + +function isRootRun(run: Record): boolean { + return run.id === run.trace_id || run.parent_run_id == null; +} + +function isDeepAgentsIntegration( + metadata: Record, +): boolean { + return metadata.ls_integration === "deepagents" || + metadata.ls_integration === "deepagents-code"; +} + +function parseRuns(transcript: string): OrderedRun[] { + let parsed: unknown; + try { + parsed = JSON.parse(transcript); + } catch { + throw invalidTranscript(); + } + + const runs = Array.isArray(parsed) + ? parsed + : isObject(parsed) && Array.isArray(parsed.runs) + ? parsed.runs + : undefined; + if (!runs || runs.length === 0 || !runs.every(isCanonicalRun)) { + throw invalidTranscript(); + } + return runs.map((run, index) => ({ run, index })); +} + +function isCanonicalRun(value: unknown): value is Record { + return ( + isObject(value) && + typeof value.id === "string" && + typeof value.trace_id === "string" && + typeof value.name === "string" && + typeof value.run_type === "string" && + isObject(value.inputs) + ); +} + +function invalidTranscript(): NormalizationError { + return new NormalizationError( + "invalid_input", + "LangSmith transcript must be a non-empty JSON array of canonical Run records or an object with a runs array.", + ); +} + +function compareRuns(left: OrderedRun, right: OrderedRun): number { + const leftOrder = left.run.dotted_order; + const rightOrder = right.run.dotted_order; + if (typeof leftOrder === "string" && typeof rightOrder === "string") { + const order = leftOrder.localeCompare(rightOrder); + if (order !== 0) return order; + } + const leftTime = parseTimestamp(left.run.start_time)?.getTime(); + const rightTime = parseTimestamp(right.run.start_time)?.getTime(); + if (leftTime !== undefined && rightTime !== undefined && leftTime !== rightTime) { + return leftTime - rightTime; + } + return left.index - right.index; +} + +function runMetadata(run: Record): Record { + const extra = isObject(run.extra) ? run.extra : {}; + return isObject(extra.metadata) ? extra.metadata : {}; +} + +function inputMessages(value: unknown): unknown[] { + if (!isObject(value)) return []; + const messages: unknown[] = []; + if (value.system !== undefined) { + messages.push({ role: "system", content: value.system }); + } + if (value.instructions !== undefined) { + messages.push({ role: "system", content: value.instructions }); + } + + const candidate = firstNonEmpty(value.messages, value.input, value.prompt); + if (candidate !== undefined) { + messages.push(...messageArray(candidate, "user")); + } else if (Array.isArray(value.prompts)) { + for (const prompt of value.prompts) { + if (typeof prompt === "string") messages.push({ role: "user", content: prompt }); + } + } + return messages; +} + +function outputMessages(value: unknown): unknown[] { + if (!isObject(value)) { + if (typeof value !== "string") return []; + const streamed = anthropicStreamMessages(value); + return streamed.length > 0 + ? streamed + : [{ role: "assistant", content: value }]; + } + + if (Array.isArray(value.generations)) { + const firstBatch = Array.isArray(value.generations[0]) + ? value.generations[0] + : value.generations; + const messages: unknown[] = []; + for (const generation of firstBatch) { + if (!isObject(generation)) continue; + if (generation.message !== undefined) messages.push(generation.message); + else if (typeof generation.text === "string") { + messages.push({ role: "assistant", content: generation.text }); + } + } + if (messages.length > 0) return messages; + } + + if (Array.isArray(value.choices)) { + const messages = value.choices + .map((choice) => (isObject(choice) ? choice.message : undefined)) + .filter((message) => message !== undefined); + if (messages.length > 0) return messages; + } + + if (Array.isArray(value.messages)) return value.messages; + if (isObject(value.message)) return [value.message]; + if (Array.isArray(value.output)) return value.output; + if (typeof value.output === "string") { + const streamed = anthropicStreamMessages(value.output); + return streamed.length > 0 + ? streamed + : [{ role: "assistant", content: value.output }]; + } + if (isObject(value.output)) { + if (Array.isArray(value.output.messages)) return value.output.messages; + if (isObject(value.output.update) && Array.isArray(value.output.update.messages)) { + return value.output.update.messages; + } + } + if (value.role !== undefined || value.type === "message") return [value]; + if (typeof value.content === "string" || Array.isArray(value.content)) { + return [{ role: "assistant", content: value.content }]; + } + return []; +} + +function anthropicStreamMessages(output: string): unknown[] { + // LangSmith's wrapAnthropic performs equivalent aggregation before ingest, + // but its messageAggregator is private. This handles traces produced by + // custom HTTP instrumentation that stored the raw Anthropic SSE response. + if (!output.includes("event:") || !output.includes("\ndata:")) return []; + const blocks = new Map(); + + for (const line of output.split("\n")) { + if (!line.startsWith("data:")) continue; + const raw = line.slice("data:".length).trim(); + if (!raw || raw === "[DONE]") continue; + let event: unknown; + try { + event = JSON.parse(raw); + } catch { + continue; + } + if (!isObject(event) || typeof event.index !== "number") continue; + + if (event.type === "content_block_start" && isObject(event.content_block)) { + const content = event.content_block; + if (typeof content.type !== "string") continue; + blocks.set(event.index, { + type: content.type, + ...(typeof content.text === "string" ? { text: content.text } : {}), + ...(typeof content.thinking === "string" + ? { thinking: content.thinking } + : {}), + ...(typeof content.id === "string" ? { id: content.id } : {}), + ...(typeof content.name === "string" ? { name: content.name } : {}), + ...(content.input !== undefined ? { input: content.input } : {}), + partialJson: "", + }); + continue; + } + + if (event.type !== "content_block_delta" || !isObject(event.delta)) { + continue; + } + const delta = event.delta; + const block = blocks.get(event.index); + if (!block) continue; + if (delta.type === "text_delta" && typeof delta.text === "string") { + block.text = (block.text ?? "") + delta.text; + } else if ( + delta.type === "thinking_delta" && + typeof delta.thinking === "string" + ) { + block.thinking = (block.thinking ?? "") + delta.thinking; + } else if ( + delta.type === "input_json_delta" && + typeof delta.partial_json === "string" + ) { + block.partialJson += delta.partial_json; + } + } + + const content: Record[] = []; + for (const [, block] of [...blocks].sort(([left], [right]) => left - right)) { + if (block.type === "text" && block.text) { + content.push({ type: "text", text: block.text }); + } else if (block.type === "thinking" && block.thinking) { + content.push({ type: "thinking", thinking: block.thinking }); + } else if (block.type === "tool_use") { + content.push({ + type: "tool_use", + ...(block.id ? { id: block.id } : {}), + ...(block.name ? { name: block.name } : {}), + input: streamedToolInput(block), + }); + } + } + return content.length > 0 ? [{ role: "assistant", content }] : []; +} + +function streamedToolInput(block: AnthropicStreamBlock): unknown { + if (!block.partialJson) return block.input ?? {}; + try { + return JSON.parse(block.partialJson); + } catch { + return { _raw: block.partialJson }; + } +} + +function messageArray(value: unknown, fallbackRole: string): unknown[] { + if (typeof value === "string") return [{ role: fallbackRole, content: value }]; + if (isObject(value)) return [value]; + if (!Array.isArray(value)) return []; + if (value.length === 1 && Array.isArray(value[0])) return value[0]; + return value; +} + +function modelFromMessages(messages: unknown[]): string | undefined { + for (const message of messages) { + if (!isObject(message)) continue; + const body = isObject(message.kwargs) ? message.kwargs : message; + const responseMetadata = isObject(body.response_metadata) + ? body.response_metadata + : {}; + const additional = isObject(body.additional_kwargs) + ? body.additional_kwargs + : {}; + const model = firstString( + body.model, + body.model_name, + responseMetadata.model_name, + responseMetadata.model, + additional.model_name, + additional.model, + ); + if (model) return model; + } + return undefined; +} + +function decodeMessages( + messages: unknown[], + timestamp?: Date, + model?: string, + fallbackRole?: string, +): ConversationItem[] { + const items: ConversationItem[] = []; + for (const message of messages) { + items.push(...decodeMessage(message, timestamp, model, fallbackRole)); + } + return items; +} + +function decodeMessage( + value: unknown, + timestamp?: Date, + model?: string, + fallbackRole?: string, +): ConversationItem[] { + if (typeof value === "string") { + return [messageItem(fallbackRole ?? "assistant", value, undefined, 0, timestamp, model)]; + } + if (!isObject(value)) return []; + + const constructorClass = constructorName(value); + const body = isObject(value.kwargs) ? value.kwargs : value; + const stableId = firstString(body.id, value.id); + const role = canonicalRole( + firstString(body.role, body.type, fallbackRole), + constructorClass, + ); + const type = firstString(body.type, value.type); + + if (type === "function_call") { + return [toolCallItem(body, stableId, 0, timestamp, model)]; + } + if (type === "function_call_output") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + if (type === "reasoning") { + const text = reasoningText(body); + return text ? [reasoningItem(text, stableId, 0, timestamp, model)] : []; + } + + if (role === "tool") { + return [toolResultItem(body, stableId, 0, timestamp)]; + } + + const items: ConversationItem[] = []; + const content = body.content; + if (Array.isArray(content)) { + for (let index = 0; index < content.length; index += 1) { + const block = content[index]; + if (typeof block === "string") { + items.push(messageItem(role, block, stableId, index, timestamp, model)); + } else if (isObject(block)) { + items.push(...decodeContentBlock(block, role, stableId, index, timestamp, model)); + } + } + } else { + const text = contentText(content); + if (text) items.push(messageItem(role, text, stableId, 0, timestamp, model)); + } + + if (role === "assistant" && Array.isArray(body.tool_calls)) { + for (let index = 0; index < body.tool_calls.length; index += 1) { + const call = body.tool_calls[index]; + if (isObject(call)) { + items.push(toolCallItem(call, stableId, index + items.length, timestamp, model)); + } + } + } + return items; +} + +function decodeContentBlock( + block: Record, + role: string, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem[] { + const type = block.type; + if (type === "tool_use" || type === "tool-call" || type === "function_call") { + return [toolCallItem(block, stableId, index, timestamp, model)]; + } + if ( + type === "tool_result" || + type === "tool-result" || + type === "function_call_output" + ) { + return [toolResultItem(block, stableId, index, timestamp)]; + } + if (type === "thinking" || type === "reasoning" || type === "redacted_thinking") { + const text = reasoningText(block); + return text ? [reasoningItem(text, stableId, index, timestamp, model)] : []; + } + if (type === "image" || type === "image_url") { + return [messageItem(role, "[image]", stableId, index, timestamp, model)]; + } + const text = contentText(block.text ?? block.content); + return text ? [messageItem(role, text, stableId, index, timestamp, model)] : []; +} + +function messageItem( + role: string, + content: string, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem { + const canonical = role === "assistant" ? "assistant" : role === "user" ? "user" : role; + const semanticKey = `message:${canonical}:${jsonString(content)}`; + const key = stableId + ? `message:${stableId}:${index}` + : semanticKey; + if (canonical !== "user" && canonical !== "assistant") { + return { key, semanticKey }; + } + return { + key, + semanticKey, + ...(stableId ? { stable: true } : {}), + event: { + type: "message", + role: canonical, + content, + ...(timestamp ? { timestamp } : {}), + ...(model ? { model } : {}), + }, + }; +} + +function reasoningItem( + content: string, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem { + return { + key: stableId + ? `reasoning:${stableId}:${index}` + : `reasoning:${jsonString(content)}`, + ...(stableId ? { stable: true } : {}), + event: { + type: "reasoning", + content, + ...(timestamp ? { timestamp } : {}), + ...(model ? { model } : {}), + }, + }; +} + +function toolCallItem( + value: Record, + stableId: string | undefined, + index: number, + timestamp?: Date, + model?: string, +): ConversationItem { + const fn = isObject(value.function) ? value.function : {}; + const id = + value.type === "function_call" + ? firstString(value.call_id, value.id, value.toolCallId) + : firstString(value.id, value.call_id, value.toolCallId); + const name = firstString(value.name, value.toolName, fn.name); + const rawArgs = firstDefined(value.arguments, value.args, value.input, fn.arguments); + const args = typeof rawArgs === "string" ? rawArgs : jsonString(rawArgs); + return { + key: id + ? `tool-call:${id}` + : `tool-call:${stableId ?? ""}:${index}:${name ?? ""}:${args}`, + ...(id || stableId ? { stable: true } : {}), + event: { + type: "tool_call", + args, + ...(id ? { id } : {}), + ...(name ? { name } : {}), + ...(timestamp ? { timestamp } : {}), + ...(model ? { model } : {}), + }, + }; +} + +function toolResultItem( + value: Record, + stableId: string | undefined, + index: number, + timestamp?: Date, +): ConversationItem { + const id = firstString( + value.tool_call_id, + value.tool_use_id, + value.call_id, + value.toolCallId, + ); + const raw = firstDefined(value.output, value.result, value.content); + const content = resultText(raw); + return { + key: id + ? `tool-result:${id}` + : `tool-result:${stableId ?? ""}:${index}:${content}`, + ...(id || stableId ? { stable: true } : {}), + event: { + type: "tool_result", + content, + ...(id ? { callId: id } : {}), + ...(timestamp ? { timestamp } : {}), + }, + }; +} + +function decodeToolRun( + run: Record, + timestamp: Date | undefined, + pendingCalls: PendingCall[], +): ConversationItem | undefined { + const inputs = isObject(run.inputs) ? run.inputs : {}; + const outputs = isObject(run.outputs) ? run.outputs : {}; + const embedded = embeddedToolMessage(outputs); + const explicitId = firstString( + inputs.toolCallId, + inputs.tool_call_id, + inputs.call_id, + outputs.toolCallId, + outputs.tool_call_id, + outputs.call_id, + embedded?.tool_call_id, + embedded?.tool_use_id, + embedded?.call_id, + ); + const name = firstString(inputs.toolName, inputs.tool_name, run.name); + const call = matchPendingCall(pendingCalls, explicitId, name); + const callId = explicitId ?? call?.id; + const rawResult = firstDefined( + embedded?.content, + embedded?.output, + embedded?.result, + outputs.result, + outputs.output, + outputs.content, + run.error, + Object.keys(outputs).length > 0 ? outputs : undefined, + ); + if (rawResult === undefined) return undefined; + let content = resultText(rawResult); + if (run.error && rawResult === run.error && !/^error/i.test(content)) { + content = `Error: ${content}`; + } + return { + key: callId + ? `tool-result:${callId}` + : `tool-run:${firstString(run.id) ?? "unknown"}`, + stable: true, + event: { + type: "tool_result", + content, + ...(callId ? { callId } : {}), + ...(timestamp ? { timestamp } : {}), + }, + }; +} + +function embeddedToolMessage( + outputs: Record, +): Record | undefined { + const output = isObject(outputs.output) ? outputs.output : undefined; + const candidate = output ?? (isObject(outputs.message) ? outputs.message : undefined); + if (!candidate) return undefined; + return isObject(candidate.kwargs) ? candidate.kwargs : candidate; +} + +function matchPendingCall( + pendingCalls: PendingCall[], + id: string | undefined, + name: string | undefined, +): PendingCall | undefined { + let match = id + ? pendingCalls.find((call) => !call.consumed && call.id === id) + : undefined; + match ??= name + ? pendingCalls.find((call) => !call.consumed && call.name === name) + : undefined; + const remaining = pendingCalls.filter((call) => !call.consumed); + match ??= remaining.length === 1 ? remaining[0] : undefined; + if (match) match.consumed = true; + return match; +} + +function mergeItems(state: DecodeState, items: ConversationItem[]): void { + if (items.length === 0) return; + let overlap = Math.min(state.history.length, items.length); + while (overlap > 0) { + let matches = true; + const historyStart = state.history.length - overlap; + for (let index = 0; index < overlap; index += 1) { + if (!conversationItemsMatch(state.history[historyStart + index], items[index])) { + matches = false; + break; + } + } + if (matches) break; + overlap -= 1; + } + + const repeatedSnapshot = + overlap === 0 && + items.some( + (item) => + item.stable === true && + state.history.some((historyItem) => historyItem.key === item.key), + ); + + for (let index = overlap; index < items.length; index += 1) { + const item = items[index]; + if (!item) continue; + if ( + item.stable === true && + state.history.some((historyItem) => historyItem.key === item.key) + ) { + continue; + } + if ( + repeatedSnapshot && + state.history.some((historyItem) => conversationItemsMatch(historyItem, item)) + ) { + continue; + } + state.history.push(item); + if (!item.event) continue; + if (isDuplicateAdjacentMessage(state.events.at(-1), item.event)) continue; + state.events.push(item.event); + if (item.event.type === "tool_call") { + state.pendingCalls.push({ + ...(item.event.id ? { id: item.event.id } : {}), + ...(item.event.name ? { name: item.event.name } : {}), + consumed: false, + }); + } else if (item.event.type === "tool_result" && item.event.callId) { + matchPendingCall(state.pendingCalls, item.event.callId, undefined); + } + } +} + +function conversationItemsMatch( + previous: ConversationItem | undefined, + current: ConversationItem | undefined, +): boolean { + if (!previous || !current) return false; + if (previous.key === current.key) return true; + return ( + previous.semanticKey !== undefined && + previous.semanticKey === current.semanticKey && + (previous.stable !== true || current.stable !== true) + ); +} + +function isDuplicateAdjacentMessage( + previous: DecodedEvent | undefined, + current: DecodedEvent, +): boolean { + return ( + previous?.type === "message" && + current.type === "message" && + previous.role === current.role && + previous.content === current.content && + previous.timestamp?.getTime() === current.timestamp?.getTime() + ); +} + +function canonicalRole(value: string | undefined, constructorClass?: string): string { + if (constructorClass === "HumanMessage" || constructorClass === "ChatMessage") { + return "user"; + } + if (constructorClass === "AIMessage") return "assistant"; + if (constructorClass === "ToolMessage" || constructorClass === "FunctionMessage") { + return "tool"; + } + if (constructorClass === "SystemMessage") return "system"; + if (value === "human") return "user"; + if (value === "ai") return "assistant"; + if (value === "function") return "tool"; + return value ?? "assistant"; +} + +function constructorName(value: Record): string | undefined { + if (!Array.isArray(value.id)) return undefined; + const last = value.id[value.id.length - 1]; + return typeof last === "string" ? last : undefined; +} + +function contentText(value: unknown): string { + if (typeof value === "string") return value; + if (value === null || value === undefined) return ""; + if (Array.isArray(value)) return blocksText(value); + if (isObject(value) && typeof value.value === "string") return value.value; + return ""; +} + +function reasoningText(value: Record): string { + const direct = firstString( + value.reasoning, + value.thinking, + value.text, + value.content, + ); + if (direct) return direct; + if (Array.isArray(value.summary)) { + return value.summary + .map((item) => (isObject(item) ? contentText(item.text ?? item.content) : "")) + .filter(Boolean) + .join("\n"); + } + return ""; +} + +function resultText(value: unknown): string { + if (typeof value === "string") return value; + if (value === undefined || value === null) return ""; + if (Array.isArray(value)) return blocksText(value) || jsonString(value); + if (isObject(value)) { + if (isObject(value.kwargs)) return resultText(value.kwargs.content); + if (typeof value.content === "string") return value.content; + } + return jsonString(value); +} + +function firstString(...values: unknown[]): string | undefined { + return values.find((value): value is string => typeof value === "string" && value.length > 0); +} + +function firstDefined(...values: unknown[]): unknown { + return values.find((value) => value !== undefined && value !== null); +} + +function firstNonEmpty(...values: unknown[]): unknown { + return values.find( + (value) => + value !== undefined && + value !== null && + (!Array.isArray(value) || value.length > 0), + ); +} diff --git a/src/index.ts b/src/index.ts index 9182203..10d090b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ import { claudeCodeAdapter } from "./adapters/claude-code.js"; import { codexAdapter } from "./adapters/codex.js"; +import { langSmithAdapter } from "./adapters/langsmith.js"; import { lettaAdapter } from "./adapters/letta.js"; import { openHandsAdapter } from "./adapters/openhands.js"; import { decodeDeepAgentsCheckpoint } from "./adapters/deepagents.js"; @@ -18,6 +19,7 @@ import { NormalizationError } from "./types.js"; const ADAPTERS: Record = { "claude-code": claudeCodeAdapter, codex: codexAdapter, + langsmith: langSmithAdapter, letta: lettaAdapter, openhands: openHandsAdapter, }; diff --git a/src/types.ts b/src/types.ts index 199e9f1..b0df120 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,7 @@ export type TrajectorySource = | "claude-code" | "codex" + | "langsmith" | "letta" | "openhands"; diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 5b72570..90a09c4 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -14,6 +14,12 @@ const fixtures = [ { source: "claude-code", name: "claude-code/cleanup" }, { source: "codex", name: "codex/tool-calls" }, { source: "codex", name: "codex/cleanup" }, + { source: "langsmith", name: "langsmith/tool-call" }, + { source: "langsmith", name: "langsmith/cleanup" }, + { source: "langsmith", name: "langsmith/official-openai-responses" }, + { source: "langsmith", name: "langsmith/official-anthropic" }, + { source: "langsmith", name: "langsmith/deepagents" }, + { source: "langsmith", name: "langsmith/deepagents-code" }, { source: "letta", name: "letta/tool-call" }, { source: "letta", name: "letta/cleanup" }, { source: "letta", name: "letta/local-v3" }, @@ -36,6 +42,7 @@ describe("golden fixtures", () => { const input = fixtureText( fixture.name, fixture.source === "openhands" || + fixture.source === "langsmith" || (fixture.source === "letta" && !fixture.name.startsWith("letta/local-")) ? "input.json" : "input.jsonl", @@ -68,7 +75,7 @@ describe("public API", () => { test("rejects an unknown source", () => { expect(() => normalizeTranscript({ - source: "langsmith" as TrajectorySource, + source: "not-a-source" as TrajectorySource, transcript: "{}", }), ).toThrow( @@ -104,6 +111,444 @@ describe("public API", () => { ).toThrow(expect.objectContaining({ code: "invalid_input" })); }); + test("rejects an invalid LangSmith document shape", () => { + expect(() => + normalizeTranscript({ + source: "langsmith", + transcript: '{"runs":{}}', + }), + ).toThrow(expect.objectContaining({ code: "invalid_input" })); + }); + + test("accepts string output from a LangSmith LLM run", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { output: "hi" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records).toContainEqual( + expect.objectContaining({ role: "assistant", content: "hi" }), + ); + }); + + for (const integration of ["claude-code", "openai-codex"] as const) { + test(`keeps ${integration} LangSmith traces on the generic path`, () => { + const root = canonicalLangSmithRun({ + id: "trace-test", + trace_id: "trace-test", + run_type: "chain", + inputs: { messages: [{ role: "user", content: "root input" }] }, + outputs: { + messages: [ + { role: "user", content: "root input" }, + { role: "assistant", content: "root aggregate" }, + ], + }, + extra: { metadata: { ls_integration: integration } }, + }); + const child = canonicalLangSmithRun({ + id: "child-llm", + trace_id: "trace-test", + parent_run_id: "trace-test", + inputs: { messages: [{ role: "user", content: "generic input" }] }, + outputs: { message: { role: "assistant", content: "generic output" } }, + }); + + const result = normalizeTranscript({ + source: "langsmith", + transcript: JSON.stringify([root, child]), + }); + const contents = result.records.flatMap((record) => + "content" in record && typeof record.content === "string" + ? [record.content] + : [], + ); + + expect(contents).toContain("generic input"); + expect(contents).toContain("generic output"); + expect(contents).not.toContain("root aggregate"); + }); + } + + test("reconstructs Anthropic SSE stored as string output", () => { + const output = [ + "event: message_start", + 'data: {"type":"message_start","message":{"role":"assistant","content":[]}}', + "", + "event: content_block_start", + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}', + "", + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Check first."}}', + "", + "event: content_block_start", + 'data: {"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}', + "", + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Hello "}}', + "", + "event: content_block_delta", + 'data: {"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"world."}}', + "", + "event: message_stop", + 'data: {"type":"message_stop"}', + "", + ].join("\n"); + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { output }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records).toContainEqual( + expect.objectContaining({ role: "reasoning", content: "Check first." }), + ); + expect(result.records).toContainEqual( + expect.objectContaining({ role: "assistant", content: "Hello world." }), + ); + }); + + test("deduplicates a tool call repeated in native message fields", () => { + const call = { + id: "call-1", + name: "weather", + args: { city: "Paris" }, + }; + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "run-1", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "weather?" }] }, + outputs: { + role: "assistant", + content: [ + { + type: "tool_use", + id: call.id, + name: call.name, + input: call.args, + }, + ], + tool_calls: [call], + }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect( + result.records.filter( + (record) => record.role === "assistant" && record.content === null, + ), + ).toHaveLength(1); + expect(result.diagnostics).not.toContainEqual( + expect.objectContaining({ code: "duplicate_tool_call_id" }), + ); + }); + + test("accepts a canonical LangSmith runs envelope", () => { + const transcript = JSON.stringify({ + runs: [ + canonicalLangSmithRun({ + id: "llm", + run_type: "llm", + inputs: { messages: [{ role: "user", content: "hello" }] }, + outputs: { role: "assistant", content: "hi" }, + }), + ], + }); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + }); + + test("deduplicates identical adjacent LangSmith messages with different IDs", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "llm", + start_time: "2026-07-10T00:00:00Z", + inputs: { + messages: [ + { id: "message-1", role: "user", content: "hello" }, + { id: "message-2", role: "user", content: "hello" }, + ], + }, + outputs: { role: "assistant", content: "hi" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + }); + + test("preserves identical LangSmith messages from different turns", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "llm-1", + start_time: "2026-07-10T00:00:00Z", + inputs: { + messages: [{ id: "message-1", role: "user", content: "retry" }], + }, + outputs: { role: "assistant", content: "first" }, + }), + canonicalLangSmithRun({ + id: "llm-2", + start_time: "2026-07-10T00:00:02Z", + inputs: { + messages: [{ id: "message-2", role: "user", content: "retry" }], + }, + outputs: { role: "assistant", content: "second" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect( + result.records.filter( + (record) => record.role === "user" && record.content === "retry", + ), + ).toHaveLength(2); + }); + + test("serializes a bare LangSmith tool output object", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { input: [{ role: "user", content: "weather?" }] }, + outputs: { + output: [ + { + type: "function_call", + id: "fc-item", + call_id: "call-1", + name: "weather", + arguments: "{}", + }, + ], + }, + }), + canonicalLangSmithRun({ + id: "tool", + name: "weather", + run_type: "tool", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: {}, + outputs: { call_id: "call-1", temperature: 22 }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records).toContainEqual( + expect.objectContaining({ + role: "tool", + tool_call_id: "call-1", + content: '{"call_id":"call-1","temperature":22}', + }), + ); + expect(result.diagnostics).toEqual([]); + }); + + test("excludes LangSmith runs marked for exclusion from Messages view", () => { + const transcript = JSON.stringify([ + canonicalLangSmithRun({ + id: "internal", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + extra: { metadata: { ls_message_view_exclude: false } }, + inputs: { messages: [{ role: "user", content: "Classify this" }] }, + outputs: { role: "assistant", content: "billing" }, + }), + canonicalLangSmithRun({ + id: "visible", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: { messages: [{ role: "user", content: "Help me" }] }, + outputs: { role: "assistant", content: "Sure" }, + }), + ]); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "assistant", + ]); + expect(result.records).not.toContainEqual( + expect.objectContaining({ content: "Classify this" }), + ); + }); + + test("decodes Anthropic reasoning and tool blocks from LangSmith runs", () => { + const firstAssistant = { + id: "msg-1", + role: "assistant", + content: [ + { type: "thinking", thinking: "I should check." }, + { type: "text", text: "Checking." }, + { + type: "tool_use", + id: "toolu-1", + name: "weather", + input: { city: "Paris" }, + }, + ], + }; + const transcript = JSON.stringify([ + { + id: "llm-1", + run_type: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { messages: [{ role: "user", content: "weather?" }] }, + outputs: { message: firstAssistant }, + }, + { + id: "tool-1", + run_type: "tool", + name: "weather", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: { city: "Paris" }, + outputs: { output: { condition: "sunny" } }, + }, + { + id: "llm-2", + run_type: "llm", + start_time: "2026-07-10T00:00:04Z", + end_time: "2026-07-10T00:00:05Z", + inputs: { + messages: [ + { role: "user", content: "weather?" }, + firstAssistant, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu-1", content: "sunny" }, + ], + }, + ], + }, + outputs: { message: { role: "assistant", content: "It is sunny." } }, + }, + ].map(canonicalLangSmithRun)); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "reasoning", + "assistant", + "assistant", + "tool", + "assistant", + ]); + expect(result.records.filter((record) => record.role === "tool")).toHaveLength(1); + }); + + test("decodes OpenAI Responses items from LangSmith runs", () => { + const transcript = JSON.stringify([ + { + id: "response-1", + run_type: "llm", + start_time: "2026-07-10T00:00:00Z", + end_time: "2026-07-10T00:00:01Z", + inputs: { + instructions: "Be helpful.", + input: [{ type: "message", role: "user", content: "weather?" }], + }, + outputs: { + output: [ + { + type: "reasoning", + id: "reasoning-1", + summary: [{ type: "summary_text", text: "Check weather." }], + }, + { + type: "function_call", + call_id: "call-1", + name: "weather", + arguments: '{"city":"Paris"}', + }, + ], + }, + }, + { + id: "tool-1", + run_type: "tool", + name: "weather", + start_time: "2026-07-10T00:00:02Z", + end_time: "2026-07-10T00:00:03Z", + inputs: { call_id: "call-1" }, + outputs: { output: "sunny" }, + }, + { + id: "response-2", + run_type: "llm", + start_time: "2026-07-10T00:00:04Z", + end_time: "2026-07-10T00:00:05Z", + inputs: { + input: [ + { type: "message", role: "user", content: "weather?" }, + { + type: "function_call", + call_id: "call-1", + name: "weather", + arguments: '{"city":"Paris"}', + }, + { type: "function_call_output", call_id: "call-1", output: "sunny" }, + ], + }, + outputs: { + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "It is sunny." }], + }, + ], + }, + }, + ].map(canonicalLangSmithRun)); + + const result = normalizeTranscript({ source: "langsmith", transcript }); + + expect(result.records.map((record) => record.role)).toEqual([ + "meta", + "user", + "reasoning", + "assistant", + "tool", + "assistant", + ]); + }); + test("rejects a non-flat Letta document shape", () => { expect(() => normalizeTranscript({ @@ -389,6 +834,20 @@ function fixtureText(name: string, file: string): string { return readFileSync(fileURLToPath(url), "utf8"); } +function canonicalLangSmithRun( + run: Record, +): Record { + const id = typeof run.id === "string" ? run.id : "run"; + return { + id, + trace_id: "trace-test", + name: id, + run_type: "llm", + inputs: {}, + ...run, + }; +} + function codexMessages(user: string, assistant: string): string { return [codexMessage("user", user), codexMessage("assistant", assistant)].join("\n"); }