From 438f12ac7ee394b0cf198016e59e79e52d1459cf Mon Sep 17 00:00:00 2001 From: R Max Espinoza Date: Sat, 15 Aug 2026 23:10:08 +0000 Subject: [PATCH] feat: add dsh transcript source --- PARITY.md | 18 ++ README.md | 1 + fixtures/dsh/happy-path/expected.json | 11 + fixtures/dsh/happy-path/input.jsonl | 12 + python/src/trajectory/_types.py | 4 +- .../src/trajectory/_vendor/trajectory-cli.mjs | 283 +++++++++++++----- src/adapters/dsh/README.md | 25 ++ src/adapters/dsh/index.ts | 166 ++++++++++ src/adapters/dsh/list.ts | 18 ++ src/index.ts | 2 + src/listing.ts | 2 + src/types.ts | 1 + test/canonical.test.ts | 1 + test/listing.test.ts | 11 + test/normalize.test.ts | 27 ++ 15 files changed, 513 insertions(+), 69 deletions(-) create mode 100644 fixtures/dsh/happy-path/expected.json create mode 100644 fixtures/dsh/happy-path/input.jsonl create mode 100644 src/adapters/dsh/README.md create mode 100644 src/adapters/dsh/index.ts create mode 100644 src/adapters/dsh/list.ts diff --git a/PARITY.md b/PARITY.md index 13d1ec2..97f522f 100644 --- a/PARITY.md +++ b/PARITY.md @@ -155,6 +155,24 @@ JavaScript saver rejected Python's `msgpack` serializer type. The production adapter therefore delegates to the official Python saver and message reducer instead of decoding SQLite blobs or assuming cross-language wire compatibility. +## DeepSeek Harness (dsh) + +The dsh adapter was checked on 2026-08-15 against three local DeepSeek Harness +sessions without retaining or printing transcript content, paths, identifiers, +arguments, results, or other private fields. dsh persists a zstd-compressed +append-only JSONL event stream; the adapter intentionally accepts the +decompressed JSONL payload and does not add a zstd dependency. + +Observed sessions used a top-level `session` header; `request/header` model +configuration; `user/message` and aggregate `assistant/message` content blocks; +direct `tool/call` and `tool/result` lifecycle events; plus turn/step, +permission, title, request-context, and streaming chunk events. The decoder +maps message text/reasoning and direct tool lifecycle events, ignores chunks to +avoid duplicate assistant text, and uses source-native message/call ids for +canonical identity. Sanitized fixtures cover the header, model and cwd metadata, +turns, user and assistant messages, reasoning, a linked successful tool call and +result, and ignored chunks. + ## OpenCode native export The OpenCode adapter was implemented against the native-format parser in diff --git a/README.md b/README.md index 68a44f3..2d86c2e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ and is empty when the transcript required no recoverable cleanup. | [`codex`](src/adapters/codex/) | Native Codex rollout JSONL | `codex` | | [`copilot-cli`](src/adapters/copilot-cli/) | Native GitHub Copilot CLI event JSONL | `copilot-cli` | | [`cursor`](src/adapters/cursor/) | Cursor role/message content-block JSONL capture | `cursor` | +| [`dsh`](src/adapters/dsh/) | Decompressed DeepSeek Harness session JSONL event stream | `dsh` | | [`droid`](src/adapters/droid/) | Native Droid session JSONL | `droid` | | [`gemini-cli`](src/adapters/gemini-cli/) | Native Gemini CLI whole-session JSON | `gemini-cli` | | [`hermes`](src/adapters/hermes/) | Session-store message-row array or a `{ "session": {...}, "messages": [...] }` envelope | `hermes` | diff --git a/fixtures/dsh/happy-path/expected.json b/fixtures/dsh/happy-path/expected.json new file mode 100644 index 0000000..d669ca5 --- /dev/null +++ b/fixtures/dsh/happy-path/expected.json @@ -0,0 +1,11 @@ +{ + "records": [ + { "role": "meta", "source": "dsh", "cwd": "/home/user/project", "model": "deepseek-chat" }, + { "role": "user", "content": "List the project files.", "timestamp": "2026-08-15T22:07:26.800Z" }, + { "role": "assistant", "content": null, "tool_calls": [{ "id": "toolu-1", "name": "bash", "args": "{\"cmd\":\"ls\"}" }], "timestamp": "2026-08-15T22:07:27.000Z" }, + { "role": "reasoning", "content": "I should inspect the directory.", "timestamp": "2026-08-15T22:07:27.100Z" }, + { "role": "tool", "tool_call_id": "toolu-1", "content": "README.md\nsrc", "ok": true, "timestamp": "2026-08-15T22:07:27.200Z" }, + { "role": "assistant", "content": "The project contains README.md and src.", "timestamp": "2026-08-15T22:07:27.400Z" } + ], + "diagnostics": [] +} diff --git a/fixtures/dsh/happy-path/input.jsonl b/fixtures/dsh/happy-path/input.jsonl new file mode 100644 index 0000000..58454ec --- /dev/null +++ b/fixtures/dsh/happy-path/input.jsonl @@ -0,0 +1,12 @@ +{"type":"session","version":0,"id":"session-00000000-0000-0000-0000-000000000001","createdAt":1786831646414,"cwd":"/home/user/project","delegationDepth":0,"agentPreset":"standard"} +{"type":"permission/preset","seq":0,"time":1786831646606,"data":{"preset":"workspace-write"}} +{"type":"request/header","seq":1,"time":1786831646610,"data":{"header":{"system":"","tools":[{"name":"bash"}],"config":{"provider":"deepseek","model":"deepseek-chat","maxTokens":8192}},"reason":"initial"}} +{"type":"turn/start","seq":2,"time":1786831646700,"data":{"turn":1}} +{"type":"user/message","seq":3,"time":1786831646800,"data":{"id":"msg-user-1","role":"user","content":[{"type":"text","text":"List the project files."}],"source":{"kind":"interactive"}}} +{"type":"assistant/chunk","seq":4,"time":1786831646900,"data":{"turn":1,"step":1,"chunk":{"type":"text","text":"I will "}}} +{"type":"tool/call","seq":5,"time":1786831647000,"data":{"turn":1,"step":1,"callId":"toolu-1","name":"bash","arguments":"{\"cmd\":\"ls\"}"}} +{"type":"assistant/message","seq":6,"time":1786831647100,"data":{"turn":1,"step":1,"message":{"id":"msg-assistant-call","role":"assistant","content":[{"type":"reasoning","text":"I should inspect the directory."},{"type":"tool-call","id":"toolu-1","name":"bash","arguments":"{\"cmd\":\"ls\"}"}],"source":{"kind":"model"}}}} +{"type":"tool/result","seq":7,"time":1786831647200,"data":{"turn":1,"step":1,"message":{"id":"msg-tool-1","role":"tool","content":[{"type":"tool-result","toolCallId":"toolu-1","content":[{"type":"text","text":"README.md\nsrc"}],"isError":false}],"source":{"kind":"tool"}}}} +{"type":"assistant/chunk","seq":8,"time":1786831647300,"data":{"turn":1,"step":2,"chunk":{"type":"text","text":"The project "}}} +{"type":"assistant/message","seq":9,"time":1786831647400,"data":{"turn":1,"step":2,"message":{"id":"msg-assistant-final","role":"assistant","content":[{"type":"text","text":"The project contains README.md and src."}],"source":{"kind":"model"}}}} +{"type":"turn/end","seq":10,"time":1786831647500,"data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/python/src/trajectory/_types.py b/python/src/trajectory/_types.py index 8381d05..38071a2 100644 --- a/python/src/trajectory/_types.py +++ b/python/src/trajectory/_types.py @@ -3,11 +3,11 @@ from typing import Literal, TypedDict, Union TrajectorySource = Literal[ - "claude-code", "codex", "copilot-cli", "cursor", "droid", "gemini-cli", "hermes", "letta-code", "omp", "openclaw", "opencode", "openhands", "pi" + "claude-code", "codex", "copilot-cli", "cursor", "dsh", "droid", "gemini-cli", "hermes", "letta-code", "omp", "openclaw", "opencode", "openhands", "pi" ] CheckpointTrajectorySource = Literal["deepagents"] AnyTrajectorySource = Literal[ - "claude-code", "codex", "copilot-cli", "cursor", "droid", "gemini-cli", "hermes", "letta-code", "omp", "openclaw", "opencode", "openhands", "pi", "deepagents" + "claude-code", "codex", "copilot-cli", "cursor", "dsh", "droid", "gemini-cli", "hermes", "letta-code", "omp", "openclaw", "opencode", "openhands", "pi", "deepagents" ] ToolResultTruncationStrategy = Literal["head", "head-tail"] ToolResultPolicy = Literal["include", "omit"] diff --git a/python/src/trajectory/_vendor/trajectory-cli.mjs b/python/src/trajectory/_vendor/trajectory-cli.mjs index 2b314a1..8cdba0b 100644 --- a/python/src/trajectory/_vendor/trajectory-cli.mjs +++ b/python/src/trajectory/_vendor/trajectory-cli.mjs @@ -749,6 +749,138 @@ function invalidCursorTranscript() { return new NormalizationError("invalid_input", "Cursor transcript must be JSONL with role and message.content records."); } +// src/adapters/dsh/index.ts +var dshAdapter = { + source: "dsh", + decode(transcript) { + const diagnostics = []; + const events = []; + let cwd; + let sourceGroupId; + let createdAt; + let model; + for (const { value: record, line, byteOffset } of parseJsonLines(transcript, diagnostics)) { + if (record.type === "session") { + if (!sourceGroupId) + sourceGroupId = nonemptyString2(record.id); + if (!cwd) + cwd = nonemptyString2(record.cwd); + if (!createdAt) + createdAt = parseTimestamp(record.createdAt); + continue; + } + const data = isObject(record.data) ? record.data : {}; + if (record.type === "request/header") { + const header = isObject(data.header) ? data.header : data; + const config = isObject(header.config) ? header.config : header; + if (!model) + model = nonemptyString2(config.model); + continue; + } + if (record.type === "request/context") { + if (!model) + model = nonemptyString2(data.model); + continue; + } + const timestamp = parseTimestamp(record.time); + const sourceSequence = typeof record.seq === "number" ? record.seq : undefined; + const base = { + inputLine: line, + ...sourceSequence !== undefined ? { sourceSequence } : {}, + ...timestamp ? { timestamp } : {} + }; + const sourceIdentity = (id, componentIndex) => id ? { sourceRecordId: id, componentIndex } : { sourceOffset: byteOffset, sourceAnchorKind: "byte", componentIndex }; + if (record.type === "user/message") { + const message = messageData(data); + const content = blocksText(message.content); + if (content) { + events.push({ + type: "message", + role: "user", + content, + ...base, + ...sourceIdentity(nonemptyString2(message.id), 0) + }); + } + continue; + } + if (record.type === "assistant/message") { + const message = messageData(data); + let componentIndex = 0; + for (const block of messageBlocks(message.content)) { + if (block.type === "reasoning" && typeof block.text === "string") { + events.push({ + type: "reasoning", + content: block.text, + ...base, + ...sourceIdentity(nonemptyString2(message.id), componentIndex++) + }); + } else if (block.type === "text" && typeof block.text === "string") { + events.push({ + type: "message", + role: "assistant", + content: block.text, + ...base, + ...sourceIdentity(nonemptyString2(message.id), componentIndex++) + }); + } + } + continue; + } + if (record.type === "tool/call") { + const call = isObject(data.call) ? data.call : data; + const id = nonemptyString2(call.id) ?? nonemptyString2(call.callId); + const name = nonemptyString2(call.name); + events.push({ + type: "tool_call", + args: typeof call.arguments === "string" ? call.arguments : jsonString(call.arguments), + ...base, + ...sourceIdentity(id, 0), + ...id ? { id } : {}, + ...name ? { name } : {} + }); + continue; + } + if (record.type === "tool/result") { + const result = isObject(data.result) ? data.result : data; + const message = messageData(data); + const block = messageBlocks(message.content).find((candidate) => candidate.type === "tool-result"); + const callId = nonemptyString2(result.id) ?? nonemptyString2(result.callId) ?? (block ? nonemptyString2(block.toolCallId) : undefined); + const isError = typeof result.error === "boolean" ? result.error : block?.isError === true; + const content = typeof result.output === "string" ? result.output : typeof result.content === "string" ? result.content : block ? blocksText(block.content) : blocksText(message.content); + events.push({ + type: "tool_result", + content, + ...base, + ...sourceIdentity(callId, 0), + ...callId ? { callId } : {}, + ...typeof isError === "boolean" ? { ok: !isError } : {} + }); + } + } + return { + events, + context: { + source: "dsh", + ...cwd ? { cwd } : {}, + ...sourceGroupId ? { sourceGroupId } : {}, + ...createdAt ? { createdAt } : {}, + ...model ? { model } : {} + }, + diagnostics + }; + } +}; +function nonemptyString2(value) { + return typeof value === "string" && value ? value : undefined; +} +function messageData(data) { + return isObject(data.message) ? data.message : data; +} +function messageBlocks(content) { + return Array.isArray(content) ? content.filter(isObject) : []; +} + // src/adapters/droid/index.ts var TRANSPORT_TYPES2 = new Set([ "todo_state", @@ -859,8 +991,8 @@ var geminiCliAdapter = { if (messageType === "info") continue; const timestamp = parseTimestamp(message.timestamp); - const model = nonemptyString2(message.model); - const sourceRecordId = nonemptyString2(message.id); + const model = nonemptyString3(message.model); + const sourceRecordId = nonemptyString3(message.id); let componentIndex = 0; const emit = (event) => { events.push({ @@ -914,8 +1046,8 @@ var geminiCliAdapter = { for (const rawCall of message.toolCalls) { if (!isObject(rawCall)) continue; - const callId = nonemptyString2(rawCall.id); - const name = nonemptyString2(rawCall.name); + const callId = nonemptyString3(rawCall.id); + const name = nonemptyString3(rawCall.name); const callTimestamp = parseTimestamp(rawCall.timestamp) ?? timestamp; emit({ type: "tool_call", @@ -925,7 +1057,7 @@ var geminiCliAdapter = { ...callTimestamp ? { timestamp: callTimestamp } : {}, ...model ? { model } : {} }); - const status = nonemptyString2(rawCall.status); + const status = nonemptyString3(rawCall.status); const outputs = toolOutputs(rawCall.result); if (outputs.length === 0 && (!status || !TERMINAL_TOOL_STATUSES.has(status))) { continue; @@ -941,8 +1073,8 @@ var geminiCliAdapter = { }); } } - const sourceGroupId = nonemptyString2(document.sessionId); - const sourceGroupRequired = !sourceGroupId && !nonemptyString2(document.projectHash); + const sourceGroupId = nonemptyString3(document.sessionId); + const sourceGroupRequired = !sourceGroupId && !nonemptyString3(document.projectHash); const createdAt = parseTimestamp(document.startTime); return { events, @@ -968,7 +1100,7 @@ function parseGeminiDocument(transcript) { } return parsed; } -function nonemptyString2(value) { +function nonemptyString3(value) { return typeof value === "string" && value.length > 0 ? value : undefined; } function thoughtContent(value) { @@ -1277,7 +1409,7 @@ var lettaCodeAdapter = { for (const { value: row } of rows) { if (row.kind !== "reasoning") continue; - const sourceRecordId = nonemptyString3(row.source_message_id) ?? nonemptyString3(row.source_line_id); + const sourceRecordId = nonemptyString4(row.source_message_id) ?? nonemptyString4(row.source_line_id); if (sourceRecordId) reasoningRecordIds.add(sourceRecordId); } @@ -1300,8 +1432,8 @@ var lettaCodeAdapter = { continue; } const timestamp = parseTimestamp(row.captured_at); - const sourceMessageId = nonemptyString3(row.source_message_id); - const sourceLineId = nonemptyString3(row.source_line_id); + const sourceMessageId = nonemptyString4(row.source_message_id); + const sourceLineId = nonemptyString4(row.source_line_id); const sourceRecordId = sourceMessageId ?? sourceLineId; const sourceFields = sourceRecordId ? { sourceRecordId } : { sourceOffset: line - 1, @@ -1340,10 +1472,10 @@ var lettaCodeAdapter = { continue; } const callId = sourceLineId ?? sourceMessageId ?? `letta-code-tool-line-${line}`; - const name = nonemptyString3(row.name); + const name = nonemptyString4(row.name); events.push({ type: "tool_call", - args: nonemptyString3(row.argsText) ?? "{}", + args: nonemptyString4(row.argsText) ?? "{}", inputLine: line, ...sourceFields, componentIndex: 0, @@ -1378,7 +1510,7 @@ var lettaCodeAdapter = { }; } }; -function nonemptyString3(value) { +function nonemptyString4(value) { return typeof value === "string" && value.length > 0 ? value : undefined; } function invalidLettaCodeTranscript() { @@ -1689,9 +1821,9 @@ var openCodeAdapter = { continue; const info = isObject(message.info) ? message.info : {}; const role = info.role; - const messageId = nonemptyString4(info.id); + const messageId = nonemptyString5(info.id); const timestamp = parseTimestamp(isObject(info.time) ? info.time.created : undefined); - const model = nonemptyString4(info.modelID); + const model = nonemptyString5(info.modelID); const parts = Array.isArray(message.parts) ? message.parts : []; let messageComponentIndex = 0; let latestTimestamp; @@ -1708,7 +1840,7 @@ var openCodeAdapter = { const ordinal = partOrdinal++; if (!isObject(part)) continue; - const partId = nonemptyString4(part.id); + const partId = nonemptyString5(part.id); const sourceRecordId = partId ?? messageId; let partComponentIndex = 0; const emit = (event) => { @@ -1750,8 +1882,8 @@ var openCodeAdapter = { const stateTime = isObject(state.time) ? state.time : {}; const callTimestamp = orderedTimestamp(parseTimestamp(stateTime.start) ?? timestamp); const resultTimestamp = orderedTimestamp(parseTimestamp(stateTime.end) ?? callTimestamp); - const callId = nonemptyString4(part.callID); - const name = nonemptyString4(part.tool); + const callId = nonemptyString5(part.callID); + const name = nonemptyString5(part.tool); emit({ type: "tool_call", args: jsonString(state.input), @@ -1760,7 +1892,7 @@ var openCodeAdapter = { ...callTimestamp ? { timestamp: callTimestamp } : {}, ...model ? { model } : {} }); - const status = nonemptyString4(state.status); + const status = nonemptyString5(state.status); const output = state.output !== undefined ? stringContent2(state.output) : status === "error" ? errorContent(state.error) : undefined; if (output !== undefined) { emit({ @@ -1782,8 +1914,8 @@ var openCodeAdapter = { } } } - const cwd = nonemptyString4(sessionInfo.directory); - const sourceGroupId = nonemptyString4(sessionInfo.id); + const cwd = nonemptyString5(sessionInfo.directory); + const sourceGroupId = nonemptyString5(sessionInfo.id); const createdAt = parseTimestamp(isObject(sessionInfo.time) ? sessionInfo.time.created : undefined); return { events, @@ -1809,7 +1941,7 @@ function parseOpenCodeDocument(transcript) { } return parsed; } -function nonemptyString4(value) { +function nonemptyString5(value) { return typeof value === "string" && value.length > 0 ? value : undefined; } function stringContent2(value) { @@ -3110,9 +3242,24 @@ async function listDroidTrajectories(root) { return sortListings(items); } -// src/adapters/deepagents/list.ts +// src/adapters/dsh/list.ts import { homedir as homedir5 } from "node:os"; -import { join as join6 } from "node:path"; +import { basename as basename4, join as join6 } from "node:path"; +async function listDshTrajectories(root) { + const base = root ?? join6(homedir5(), ".dsh", "sessions"); + const items = []; + for (const path of collectFiles(base, ".jsonl.zstd", 4)) { + const sessionDir = basename4(join6(path, "..")); + const listing = listingFromFile(sessionDir, path); + if (listing) + items.push(listing); + } + return sortListings(items); +} + +// src/adapters/deepagents/list.ts +import { homedir as homedir6 } from "node:os"; +import { join as join7 } from "node:path"; async function listDeepAgentsTrajectories(root) { const path = resolveStorePath(root); if (!safeStat(path)) @@ -3133,13 +3280,13 @@ async function listDeepAgentsTrajectories(root) { } function resolveStorePath(root) { if (root === undefined) - return join6(homedir5(), ".deepagents", "sessions.db"); - return root.endsWith(".db") ? root : join6(root, "sessions.db"); + return join7(homedir6(), ".deepagents", "sessions.db"); + return root.endsWith(".db") ? root : join7(root, "sessions.db"); } // src/adapters/hermes/list.ts -import { homedir as homedir6 } from "node:os"; -import { join as join7 } from "node:path"; +import { homedir as homedir7 } from "node:os"; +import { join as join8 } from "node:path"; async function listHermesTrajectories(root) { const path = resolveStorePath2(root); if (!safeStat(path)) @@ -3166,27 +3313,27 @@ async function listHermesTrajectories(root) { } function resolveStorePath2(root) { if (root === undefined) - return join7(homedir6(), ".hermes", "state.db"); - return root.endsWith(".db") ? root : join7(root, "state.db"); + return join8(homedir7(), ".hermes", "state.db"); + return root.endsWith(".db") ? root : join8(root, "state.db"); } function numeric(value) { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined; } // src/adapters/letta-code/list.ts -import { homedir as homedir7 } from "node:os"; -import { join as join8 } from "node:path"; +import { homedir as homedir8 } from "node:os"; +import { join as join9 } from "node:path"; async function listLettaCodeTrajectories(root) { - const base = root ?? join8(homedir7(), ".letta", "transcripts"); + const base = root ?? join9(homedir8(), ".letta", "transcripts"); const items = []; for (const agent of safeReadDir(base)) { if (!agent.isDirectory) continue; - const agentPath = join8(base, agent.name); + const agentPath = join9(base, agent.name); for (const conversation of safeReadDir(agentPath)) { if (!conversation.isDirectory) continue; - const path = join8(agentPath, conversation.name, "transcript.jsonl"); + const path = join9(agentPath, conversation.name, "transcript.jsonl"); const listing = listingFromFile(`${agent.name}/${conversation.name}`, path); if (listing && (listing.sizeBytes ?? 0) > 0) items.push(listing); @@ -3197,21 +3344,21 @@ async function listLettaCodeTrajectories(root) { // src/adapters/openclaw/list.ts import { existsSync } from "node:fs"; -import { homedir as homedir8 } from "node:os"; -import { basename as basename4, join as join9 } from "node:path"; +import { homedir as homedir9 } from "node:os"; +import { basename as basename5, join as join10 } from "node:path"; async function listOpenClawTrajectories(root) { const base = root ?? defaultStateDir(); const items = []; - const agentsPath = join9(base, "agents"); + const agentsPath = join10(base, "agents"); for (const agent of safeReadDir(agentsPath)) { if (!agent.isDirectory) continue; - const sessionsPath = join9(agentsPath, agent.name, "sessions"); + const sessionsPath = join10(agentsPath, agent.name, "sessions"); for (const entry of safeReadDir(sessionsPath)) { if (!entry.isFile || !entry.name.endsWith(".jsonl")) continue; - const path = join9(sessionsPath, entry.name); - const listing = listingFromFile(basename4(entry.name, ".jsonl"), path); + const path = join10(sessionsPath, entry.name); + const listing = listingFromFile(basename5(entry.name, ".jsonl"), path); if (listing) items.push(listing); } @@ -3222,22 +3369,22 @@ function defaultStateDir() { const override = process.env.OPENCLAW_STATE_DIR?.trim() || process.env.CLAWDBOT_STATE_DIR?.trim(); if (override) return override; - const current = join9(homedir8(), ".openclaw"); + const current = join10(homedir9(), ".openclaw"); if (existsSync(current)) return current; - return join9(homedir8(), ".clawdbot"); + return join10(homedir9(), ".clawdbot"); } // src/adapters/openhands/list.ts -import { homedir as homedir9 } from "node:os"; -import { join as join10 } from "node:path"; +import { homedir as homedir10 } from "node:os"; +import { join as join11 } from "node:path"; async function listOpenHandsTrajectories(root) { - const base = root ?? join10(homedir9(), ".openhands", "sessions"); + const base = root ?? join11(homedir10(), ".openhands", "sessions"); const items = []; for (const entry of safeReadDir(base)) { if (!entry.isDirectory) continue; - const path = join10(base, entry.name); + const path = join11(base, entry.name); const facts = safeStat(path); items.push({ id: entry.name, @@ -3249,21 +3396,21 @@ async function listOpenHandsTrajectories(root) { } // src/adapters/pi/list.ts -import { homedir as homedir10 } from "node:os"; -import { basename as basename5, join as join11 } from "node:path"; +import { homedir as homedir11 } from "node:os"; +import { basename as basename6, join as join12 } from "node:path"; async function listPiTrajectories(root) { const base = root ?? defaultAgentDir(); const items = []; - const sessionsPath = join11(base, "sessions"); + const sessionsPath = join12(base, "sessions"); for (const project of safeReadDir(sessionsPath)) { if (!project.isDirectory) continue; - const projectPath = join11(sessionsPath, project.name); + const projectPath = join12(sessionsPath, project.name); for (const entry of safeReadDir(projectPath)) { if (!entry.isFile || !entry.name.endsWith(".jsonl")) continue; - const path = join11(projectPath, entry.name); - const listing = listingFromFile(basename5(entry.name, ".jsonl"), path); + const path = join12(projectPath, entry.name); + const listing = listingFromFile(basename6(entry.name, ".jsonl"), path); if (listing) items.push(listing); } @@ -3274,17 +3421,17 @@ function defaultAgentDir() { const override = process.env.PI_CODING_AGENT_DIR?.trim(); if (override) return override; - return join11(homedir10(), ".pi", "agent"); + return join12(homedir11(), ".pi", "agent"); } // src/adapters/omp/list.ts import { existsSync as existsSync2 } from "node:fs"; -import { homedir as homedir11 } from "node:os"; -import { basename as basename6, join as join12 } from "node:path"; +import { homedir as homedir12 } from "node:os"; +import { basename as basename7, join as join13 } from "node:path"; async function listOmpTrajectories(root) { const items = []; - const sessionsPath = root ? join12(root, "sessions") : resolveOmpSessionsPath({ - home: homedir11(), + const sessionsPath = root ? join13(root, "sessions") : resolveOmpSessionsPath({ + home: homedir12(), platform: process.platform, env: process.env, exists: existsSync2 @@ -3292,12 +3439,12 @@ async function listOmpTrajectories(root) { for (const project of safeReadDir(sessionsPath)) { if (!project.isDirectory) continue; - const projectPath = join12(sessionsPath, project.name); + const projectPath = join13(sessionsPath, project.name); for (const entry of safeReadDir(projectPath)) { if (!entry.isFile || !entry.name.endsWith(".jsonl")) continue; - const path = join12(projectPath, entry.name); - const listing = listingFromFile(basename6(entry.name, ".jsonl"), path); + const path = join13(projectPath, entry.name); + const listing = listingFromFile(basename7(entry.name, ".jsonl"), path); if (listing) items.push(listing); } @@ -3306,18 +3453,18 @@ async function listOmpTrajectories(root) { } function resolveOmpSessionsPath(options) { const profile = resolveProfile(options.env.OMP_PROFILE, options.env.PI_PROFILE); - const configRoot = join12(options.home, options.env.PI_CONFIG_DIR || ".omp", ...profile ? ["profiles", profile] : []); + const configRoot = join13(options.home, options.env.PI_CONFIG_DIR || ".omp", ...profile ? ["profiles", profile] : []); const agentOverride = profile ? undefined : options.env.PI_CODING_AGENT_DIR?.trim() || undefined; - const agentDir = agentOverride ?? join12(configRoot, "agent"); + const agentDir = agentOverride ?? join13(configRoot, "agent"); if (agentOverride === undefined && (options.platform === "linux" || options.platform === "darwin")) { const xdgData = options.env.XDG_DATA_HOME?.trim(); if (xdgData) { - const xdgRoot = join12(xdgData, "omp", ...profile ? ["profiles", profile] : []); + const xdgRoot = join13(xdgData, "omp", ...profile ? ["profiles", profile] : []); if (options.exists(xdgRoot)) - return join12(xdgRoot, "sessions"); + return join13(xdgRoot, "sessions"); } } - return join12(agentDir, "sessions"); + return join13(agentDir, "sessions"); } function resolveProfile(ompProfile, piProfile) { const value = (ompProfile !== undefined ? ompProfile : piProfile)?.trim(); @@ -3336,6 +3483,7 @@ var LISTERS = { "claude-code": listClaudeCodeTrajectories, codex: listCodexTrajectories, droid: listDroidTrajectories, + dsh: listDshTrajectories, deepagents: listDeepAgentsTrajectories, hermes: listHermesTrajectories, "letta-code": listLettaCodeTrajectories, @@ -3413,6 +3561,7 @@ var ADAPTERS = { codex: codexAdapter, "copilot-cli": copilotCliAdapter, cursor: cursorAdapter, + dsh: dshAdapter, droid: droidAdapter, "gemini-cli": geminiCliAdapter, hermes: hermesAdapter, diff --git a/src/adapters/dsh/README.md b/src/adapters/dsh/README.md new file mode 100644 index 0000000..14c2d54 --- /dev/null +++ b/src/adapters/dsh/README.md @@ -0,0 +1,25 @@ +# dsh + +DeepSeek Harness (`dsh`) stores each session as an append-only JSONL event +stream at `~/.dsh/sessions//session-/session.jsonl.zstd`. +Pass the **decompressed JSONL text** to +`normalizeTranscript({ source: "dsh", transcript })`. The `.zstd` container is +an on-disk storage detail; this package intentionally has no zstd dependency. + +| dsh event | Normalized output | +| --- | --- | +| `session` | meta session id, cwd, creation time | +| `request/header`, `request/context` | model metadata | +| `user/message` | user text blocks | +| `assistant/message` | aggregate assistant text and reasoning blocks | +| `tool/call` | assistant tool call | +| `tool/result` | linked tool result and native error status | +| `assistant/chunk`, turn/step boundaries, title, permission and request metadata | ignored | + +`tool/call` and `tool/result` are the authoritative tool lifecycle events. +Assistant streaming chunks are deliberately ignored, and embedded tool blocks in +aggregate messages are not replayed, preventing duplicate calls and results. + +`listTrajectories({ source: "dsh" })` discovers compressed session logs in the +standard store. It returns their `.jsonl.zstd` paths; callers decompress before +normalizing. diff --git a/src/adapters/dsh/index.ts b/src/adapters/dsh/index.ts new file mode 100644 index 0000000..678c17d --- /dev/null +++ b/src/adapters/dsh/index.ts @@ -0,0 +1,166 @@ +import type { + DecodedEvent, + DecodedSession, + SourceAdapter, +} from "../../internal.js"; +import type { Diagnostic } from "../../types.js"; +import { blocksText, isObject, jsonString, parseJsonLines, parseTimestamp } from "../shared.js"; + +/** Decode a decompressed DeepSeek Harness session JSONL stream. */ +export const dshAdapter: SourceAdapter = { + source: "dsh", + + decode(transcript: string): DecodedSession { + const diagnostics: Diagnostic[] = []; + const events: DecodedEvent[] = []; + let cwd: string | undefined; + let sourceGroupId: string | undefined; + let createdAt: Date | undefined; + let model: string | undefined; + + for (const { value: record, line, byteOffset } of parseJsonLines( + transcript, + diagnostics, + )) { + if (record.type === "session") { + if (!sourceGroupId) sourceGroupId = nonemptyString(record.id); + if (!cwd) cwd = nonemptyString(record.cwd); + if (!createdAt) createdAt = parseTimestamp(record.createdAt); + continue; + } + + const data = isObject(record.data) ? record.data : {}; + if (record.type === "request/header") { + const header = isObject(data.header) ? data.header : data; + const config = isObject(header.config) ? header.config : header; + if (!model) model = nonemptyString(config.model); + continue; + } + if (record.type === "request/context") { + if (!model) model = nonemptyString(data.model); + continue; + } + + const timestamp = parseTimestamp(record.time); + const sourceSequence = typeof record.seq === "number" ? record.seq : undefined; + const base = { + inputLine: line, + ...(sourceSequence !== undefined ? { sourceSequence } : {}), + ...(timestamp ? { timestamp } : {}), + }; + const sourceIdentity = ( + id: string | undefined, + componentIndex: number, + ) => + id + ? { sourceRecordId: id, componentIndex } + : { sourceOffset: byteOffset, sourceAnchorKind: "byte" as const, componentIndex }; + + if (record.type === "user/message") { + const message = messageData(data); + const content = blocksText(message.content); + if (content) { + events.push({ + type: "message", + role: "user", + content, + ...base, + ...sourceIdentity(nonemptyString(message.id), 0), + }); + } + continue; + } + + if (record.type === "assistant/message") { + const message = messageData(data); + let componentIndex = 0; + for (const block of messageBlocks(message.content)) { + if (block.type === "reasoning" && typeof block.text === "string") { + events.push({ + type: "reasoning", + content: block.text, + ...base, + ...sourceIdentity(nonemptyString(message.id), componentIndex++), + }); + } else if (block.type === "text" && typeof block.text === "string") { + events.push({ + type: "message", + role: "assistant", + content: block.text, + ...base, + ...sourceIdentity(nonemptyString(message.id), componentIndex++), + }); + } + } + continue; + } + + if (record.type === "tool/call") { + const call = isObject(data.call) ? data.call : data; + const id = nonemptyString(call.id) ?? nonemptyString(call.callId); + const name = nonemptyString(call.name); + events.push({ + type: "tool_call", + args: typeof call.arguments === "string" ? call.arguments : jsonString(call.arguments), + ...base, + ...sourceIdentity(id, 0), + ...(id ? { id } : {}), + ...(name ? { name } : {}), + }); + continue; + } + + if (record.type === "tool/result") { + const result = isObject(data.result) ? data.result : data; + const message = messageData(data); + const block = messageBlocks(message.content).find( + (candidate) => candidate.type === "tool-result", + ); + const callId = nonemptyString(result.id) ?? nonemptyString(result.callId) ?? + (block ? nonemptyString(block.toolCallId) : undefined); + const isError = typeof result.error === "boolean" + ? result.error + : block?.isError === true; + const content = typeof result.output === "string" + ? result.output + : typeof result.content === "string" + ? result.content + : block + ? blocksText(block.content) + : blocksText(message.content); + events.push({ + type: "tool_result", + content, + ...base, + ...sourceIdentity(callId, 0), + ...(callId ? { callId } : {}), + ...(typeof isError === "boolean" ? { ok: !isError } : {}), + }); + } + } + + return { + events, + context: { + source: "dsh", + ...(cwd ? { cwd } : {}), + ...(sourceGroupId ? { sourceGroupId } : {}), + ...(createdAt ? { createdAt } : {}), + ...(model ? { model } : {}), + }, + diagnostics, + }; + }, +}; + +function nonemptyString(value: unknown): string | undefined { + return typeof value === "string" && value ? value : undefined; +} + +function messageData(data: Record): Record { + return isObject(data.message) ? data.message : data; +} + +function messageBlocks(content: unknown): Record[] { + return Array.isArray(content) ? content.filter(isObject) : []; +} diff --git a/src/adapters/dsh/list.ts b/src/adapters/dsh/list.ts new file mode 100644 index 0000000..c859c3c --- /dev/null +++ b/src/adapters/dsh/list.ts @@ -0,0 +1,18 @@ +import { homedir } from "node:os"; +import { basename, join } from "node:path"; +import type { TrajectoryListing } from "../../listing.js"; +import { collectFiles, listingFromFile, sortListings } from "../listing-shared.js"; + +/** List compressed DeepSeek Harness session logs without decompressing them. */ +export async function listDshTrajectories( + root: string | undefined, +): Promise { + const base = root ?? join(homedir(), ".dsh", "sessions"); + const items: TrajectoryListing[] = []; + for (const path of collectFiles(base, ".jsonl.zstd", 4)) { + const sessionDir = basename(join(path, "..")); + const listing = listingFromFile(sessionDir, path); + if (listing) items.push(listing); + } + return sortListings(items); +} diff --git a/src/index.ts b/src/index.ts index 28ce245..69cb72a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import { claudeCodeAdapter } from "./adapters/claude-code/index.js"; import { codexAdapter } from "./adapters/codex/index.js"; import { copilotCliAdapter } from "./adapters/copilot-cli/index.js"; import { cursorAdapter } from "./adapters/cursor/index.js"; +import { dshAdapter } from "./adapters/dsh/index.js"; import { droidAdapter } from "./adapters/droid/index.js"; import { geminiCliAdapter } from "./adapters/gemini-cli/index.js"; import { hermesAdapter } from "./adapters/hermes/index.js"; @@ -34,6 +35,7 @@ const ADAPTERS: Record = { codex: codexAdapter, "copilot-cli": copilotCliAdapter, cursor: cursorAdapter, + dsh: dshAdapter, droid: droidAdapter, "gemini-cli": geminiCliAdapter, hermes: hermesAdapter, diff --git a/src/listing.ts b/src/listing.ts index 8b51398..e02eeda 100644 --- a/src/listing.ts +++ b/src/listing.ts @@ -13,6 +13,7 @@ import { listClaudeCodeTrajectories } from "./adapters/claude-code/list.js"; import { listCodexTrajectories } from "./adapters/codex/list.js"; import { listDroidTrajectories } from "./adapters/droid/list.js"; +import { listDshTrajectories } from "./adapters/dsh/list.js"; import { listDeepAgentsTrajectories } from "./adapters/deepagents/list.js"; import { listHermesTrajectories } from "./adapters/hermes/list.js"; import { listLettaCodeTrajectories } from "./adapters/letta-code/list.js"; @@ -64,6 +65,7 @@ const LISTERS: Partial> = { "claude-code": listClaudeCodeTrajectories, codex: listCodexTrajectories, droid: listDroidTrajectories, + dsh: listDshTrajectories, deepagents: listDeepAgentsTrajectories, hermes: listHermesTrajectories, "letta-code": listLettaCodeTrajectories, diff --git a/src/types.ts b/src/types.ts index 61238ba..72f99b7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -6,6 +6,7 @@ export type TrajectorySource = | "codex" | "copilot-cli" | "cursor" + | "dsh" | "droid" | "gemini-cli" | "hermes" diff --git a/test/canonical.test.ts b/test/canonical.test.ts index 2907bfc..172597e 100644 --- a/test/canonical.test.ts +++ b/test/canonical.test.ts @@ -43,6 +43,7 @@ const goldenFixtures = [ const additionalInvariantFixtures = [ { source: "copilot-cli", name: "copilot-cli/tool-calls" }, { source: "cursor", name: "cursor/tool-calls" }, + { source: "dsh", name: "dsh/happy-path" }, { source: "gemini-cli", name: "gemini-cli/tool-calls" }, { source: "opencode", name: "opencode/tool-calls" }, ] as const satisfies ReadonlyArray<{ diff --git a/test/listing.test.ts b/test/listing.test.ts index 9e8b5af..39cf235 100644 --- a/test/listing.test.ts +++ b/test/listing.test.ts @@ -85,6 +85,11 @@ beforeAll(() => { writeFileSync(join(openclawSessions, "oc-1.jsonl"), `{"type":"session"}\n`); writeFileSync(join(openclawSessions, "sessions.json"), "{}\n"); + // dsh: sessions are nested under cwd slugs and stored compressed. + const dshSession = join(base, "dsh", "home-user-project", "session-dsh-1"); + mkdirSync(dshSession, { recursive: true }); + writeFileSync(join(dshSession, "session.jsonl.zstd"), "compressed"); + // pi: /sessions//_.jsonl layout. const piSessions = join(base, "pi", "sessions", "-home-user-pi-demo--"); mkdirSync(piSessions, { recursive: true }); @@ -209,6 +214,12 @@ describe("listTrajectories", () => { expect(result.items.map((item) => item.id)).toEqual(["oc-1"]); }); + test("lists compressed dsh session logs", async () => { + const result = await listTrajectories({ source: "dsh", root: join(base, "dsh") }); + expect(result.items.map((item) => item.id)).toEqual(["session-dsh-1"]); + expect(result.items[0]?.path.endsWith("session.jsonl.zstd")).toBe(true); + }); + test("lists pi sessions and ignores files outside project directories", async () => { const result = await listTrajectories({ source: "pi", diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 91dbda0..c61bc29 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -20,6 +20,7 @@ const fixtures = [ { source: "copilot-cli", name: "copilot-cli/cleanup" }, { source: "cursor", name: "cursor/tool-calls" }, { source: "cursor", name: "cursor/cleanup" }, + { source: "dsh", name: "dsh/happy-path" }, { source: "droid", name: "droid/happy-path" }, { source: "gemini-cli", name: "gemini-cli/tool-calls" }, { source: "gemini-cli", name: "gemini-cli/cleanup" }, @@ -308,6 +309,32 @@ describe("public API", () => { ); }); + test("decodes dsh direct tool lifecycle events, ignores chunks, and tolerates malformed lines", () => { + const transcript = [ + fixtureText("dsh/happy-path", "input.jsonl"), + "{not json}", + ].join("\n"); + const result = normalizeTranscript({ source: "dsh", transcript }); + + expect(result.records.filter((record) => record.role === "assistant")).toHaveLength(2); + expect(result.records).toEqual(expect.arrayContaining([ + expect.objectContaining({ + role: "assistant", + content: null, + tool_calls: [{ id: "toolu-1", name: "bash", args: "{\"cmd\":\"ls\"}" }], + }), + expect.objectContaining({ + role: "tool", + tool_call_id: "toolu-1", + content: "README.md\nsrc", + ok: true, + }), + ])); + expect(result.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "invalid_json_line" }), + ])); + }); + test("always returns diagnostics", () => { const result = normalizeTranscript({ source: "codex",