From 19b565fe267982b857d4dc41125899953b9c888f Mon Sep 17 00:00:00 2001 From: Chal1ce <752193262@qq.com> Date: Fri, 14 Aug 2026 21:54:32 +0800 Subject: [PATCH 01/13] feat(runtime): add observability replay and evaluation --- packages/core/src/cli-runtime.ts | 105 ++++++++ packages/core/src/dscode-extension.ts | 265 +++++++++++++++++-- packages/core/src/evaluation.ts | 123 +++++++++ packages/core/src/home.ts | 8 + packages/core/src/index.ts | 37 +++ packages/core/src/observability.ts | 350 ++++++++++++++++++++++++++ packages/core/src/replay.ts | 299 ++++++++++++++++++++++ packages/core/src/runtime-options.ts | 7 + test/cli-integration.test.ts | 31 +++ test/observability.test.ts | 142 +++++++++++ 10 files changed, 1343 insertions(+), 24 deletions(-) create mode 100644 packages/core/src/evaluation.ts create mode 100644 packages/core/src/observability.ts create mode 100644 packages/core/src/replay.ts create mode 100644 test/observability.test.ts diff --git a/packages/core/src/cli-runtime.ts b/packages/core/src/cli-runtime.ts index de34aba..00ab71d 100644 --- a/packages/core/src/cli-runtime.ts +++ b/packages/core/src/cli-runtime.ts @@ -7,6 +7,16 @@ import { initializeDSCodeHome } from "./home.js"; import { installPiLoginSecretMask } from "./pi-login-mask.js"; import { installPiMarkdownCodeBlocks } from "./pi-markdown.js"; import { parseSupportedProviderId, type SupportedProviderId } from "./providers.js"; +import { + evaluateReplay, + formatEvaluationResult, + type EvaluationOptions, +} from "./evaluation.js"; +import { + compareTraceReports, + formatReplayReport, + readTraceFile, +} from "./replay.js"; import { parseRuntimeArgs, printDSCodeHelp } from "./runtime-options.js"; import { installDSCodeRuntimeBranding } from "./runtime-branding.js"; import { ensureDSCodeUiDefaults } from "./ui-defaults.js"; @@ -23,6 +33,11 @@ export async function runDSCode(argv: string[]): Promise { runWindowsSandboxLifecycle(windowsSandboxCommand); return; } + const traceCommand = parseTraceCommand(argv); + if (traceCommand) { + await runTraceCommand(traceCommand); + return; + } const parsed = parseRuntimeArgs(argv); if (parsed.help) { printDSCodeHelp(); @@ -61,6 +76,90 @@ export async function runDSCode(argv: string[]): Promise { }); } +interface TraceCommand { + command: "replay" | "evaluate"; + input: string; + compare?: string; + json: boolean; + evaluation: EvaluationOptions; +} + +function parseTraceCommand(argv: string[]): TraceCommand | undefined { + const command = argv[0]; + if (command !== "replay" && command !== "evaluate" && command !== "eval") return undefined; + const paths: string[] = []; + let json = false; + let compare = false; + const evaluation: EvaluationOptions = {}; + for (let index = 1; index < argv.length; index += 1) { + const argument = argv[index]!; + if (argument === "--json") { + json = true; + continue; + } + if (argument === "--compare") { + compare = true; + continue; + } + const [flag, inlineValue] = splitTraceFlag(argument); + if ( + flag === "--max-tool-calls" || + flag === "--max-duration-ms" || + flag === "--max-total-tokens" || + flag === "--max-cost" + ) { + const value = inlineValue ?? argv[++index]; + if (!value) throw new Error(`${flag} requires a value`); + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 0) throw new Error(`${flag} requires a non-negative number`); + if (flag === "--max-tool-calls") evaluation.maxToolCalls = parsed; + if (flag === "--max-duration-ms") evaluation.maxDurationMs = parsed; + if (flag === "--max-total-tokens") evaluation.maxTotalTokens = parsed; + if (flag === "--max-cost") evaluation.maxCost = parsed; + continue; + } + if (argument === "--allow-errors") { + evaluation.failOnErrors = false; + continue; + } + if (argument.startsWith("-")) throw new Error(`unknown trace option: ${argument}`); + paths.push(argument); + } + if (compare) { + if (paths.length !== 2) throw new Error("replay --compare requires baseline and candidate trace files"); + return { command: "replay", input: paths[0]!, compare: paths[1]!, json, evaluation }; + } + if (paths.length !== 1) throw new Error(`${command} requires one trace file`); + return { + command: command === "eval" ? "evaluate" : command, + input: paths[0]!, + json, + evaluation, + }; +} + +async function runTraceCommand(command: TraceCommand): Promise { + const report = await readTraceFile(command.input); + if (command.compare) { + const comparison = compareTraceReports(report, await readTraceFile(command.compare)); + process.stdout.write( + command.json + ? `${JSON.stringify(comparison, null, 2)}\n` + : `${formatReplayReport(comparison.baseline)}\n\nCandidate\n${formatReplayReport(comparison.candidate)}\n\nDifferences\n${comparison.differences.join("\n") || "none"}\n`, + ); + if (!comparison.candidate.valid) process.exitCode = 1; + return; + } + if (command.command === "replay") { + process.stdout.write(`${formatReplayReport(report, command.json)}\n`); + if (!report.valid) process.exitCode = 1; + return; + } + const result = evaluateReplay(report, command.evaluation); + process.stdout.write(`${formatEvaluationResult(result, command.json)}\n`); + if (!result.passed) process.exitCode = 1; +} + /** Process-oriented wrapper used by the terminal and bundled RPC entry points. */ export async function runDSCodeProcess(argv: string[]): Promise { try { @@ -84,6 +183,12 @@ interface AuthCommand { providerId?: SupportedProviderId; } +function splitTraceFlag(argument: string): [string, string | undefined] { + if (!argument.startsWith("--") || !argument.includes("=")) return [argument, undefined]; + const index = argument.indexOf("="); + return [argument.slice(0, index), argument.slice(index + 1)]; +} + function parseAuthCommand(argv: string[]): AuthCommand | undefined { const command = argv[0]; if (command === "login" || command === "logout") { diff --git a/packages/core/src/dscode-extension.ts b/packages/core/src/dscode-extension.ts index ed72374..cd9d245 100644 --- a/packages/core/src/dscode-extension.ts +++ b/packages/core/src/dscode-extension.ts @@ -30,6 +30,13 @@ import { registerLocalImageInput } from "./image-input.js"; import { partitionSessionFile } from "./home.js"; import { ManagedProcessRegistry, type ManagedProcessResult } from "./managed-process.js"; import { MCPManager } from "./mcp.js"; +import { + AgentRuntimeTrace, + classifyHttpStatus, + summarizeContent, + summarizeValue, + usageFromPiUsage, +} from "./observability.js"; import { applyWorkspacePatch, type ApplyPatchResult } from "./patch.js"; import { formatPlanForExecution, @@ -137,11 +144,17 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte let toolsBeforePlan: string[] | undefined; let projectCommands: string[] = []; let lastAgentFailed = false; + let traceErrorRecorded = false; let sessionPartition = Promise.resolve(); let planState: PlanState | undefined; let lastOfferedPlanRevision = 0; const access = new SessionAccessController(options.sandbox, options.network); + const trace = new AgentRuntimeTrace(); + const pendingModelSpans: string[] = []; + const pendingCompactionSpans: string[] = []; + let traceSequence = 0; const effectiveAccess = (): EffectiveAccess => access.effective(permission); + const nextTraceSpan = (kind: string): string => `${kind}:${++traceSequence}`; const queueSessionPartition = (ctx: ExtensionContext): Promise => { const sessionFile = ctx.sessionManager.getSessionFile(); @@ -223,11 +236,51 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte }; pi.on("before_provider_request", (event, ctx) => { + const spanId = nextTraceSpan("model"); + pendingModelSpans.push(spanId); + trace.record({ + type: "model_request", + spanId, + status: "started", + provider: ctx.model?.provider ?? options.providerId, + model: ctx.model?.id ?? options.modelId, + input: summarizeValue(event.payload), + }); if (ctx.model?.provider !== "deepseek" || options.transport !== "responses") return; return optimizeDeepSeekResponsesPayload(event.payload, { webSearch: options.webSearch }); }); + pi.on("after_provider_response", (event, ctx) => { + const spanId = pendingModelSpans[0] ?? nextTraceSpan("model-response"); + const failed = event.status >= 400; + trace.record({ + type: "provider_response", + spanId, + status: failed ? "failed" : "completed", + provider: ctx.model?.provider ?? options.providerId, + model: ctx.model?.id ?? options.modelId, + attributes: { + httpStatus: event.status, + responseHeaderCount: Object.keys(event.headers).length, + }, + }); + if (failed) { + lastAgentFailed = true; + pendingModelSpans.shift(); + if (!traceErrorRecorded) { + traceErrorRecorded = true; + trace.recordError( + nextTraceSpan("error"), + classifyHttpStatus(event.status), + "provider response failed", + { httpStatus: event.status }, + ); + } + } + }); + pi.on("session_start", async (_event, ctx) => { + trace.setSession(ctx.sessionManager.getSessionId()); checkpoints.length = 0; undone.clear(); projectCommands = await discoverProjectCommands(ctx.cwd); @@ -265,17 +318,37 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte }); pi.on("agent_settled", async (_event, ctx) => { + trace.endRun(lastAgentFailed ? "failed" : "completed"); + await trace.flush(); await queueSessionPartition(ctx); }); pi.on("session_shutdown", async (_event, ctx) => { + trace.shutdown("cancelled"); + await trace.flush(); await queueSessionPartition(ctx); processes.dispose(); await mcp.close(); }); - pi.on("before_agent_start", async (event) => { + pi.on("before_agent_start", async (event, ctx) => { lastAgentFailed = false; + traceErrorRecorded = false; + pendingModelSpans.length = 0; + const traceId = trace.startRun({ + sessionId: ctx.sessionManager.getSessionId(), + provider: options.providerId, + model: options.modelId, + }); + trace.record({ + type: "agent_start", + spanId: `agent:${traceId}`, + status: "started", + provider: ctx.model?.provider ?? options.providerId, + model: ctx.model?.id ?? options.modelId, + input: summarizeValue(event.prompt), + attributes: { attachedImageCount: event.images?.length ?? 0 }, + }); const currentAccess = effectiveAccess(); const personalizationPrompt = await loadPersonalizationPrompt(options.personalizationFile); const systemPrompt = composePersonalizedSystemPrompt( @@ -302,25 +375,63 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte }); pi.on("tool_call", async (event, ctx) => { + trace.record({ + type: "tool_call", + spanId: `tool:${event.toolCallId}`, + status: "started", + toolName: event.toolName, + toolCallId: event.toolCallId, + input: summarizeValue(event.input), + attributes: { inputKeyCount: Object.keys(event.input).length }, + }); + const blockTool = (reason: string, code = "policy") => { + trace.record({ + type: "approval", + spanId: `approval:${event.toolCallId}`, + status: "failed", + toolName: event.toolName, + toolCallId: event.toolCallId, + attributes: { decision: "denied", code }, + }); + trace.record({ + type: "tool_result", + spanId: `tool:${event.toolCallId}`, + status: "failed", + toolName: event.toolName, + toolCallId: event.toolCallId, + error: { code, message: "tool execution blocked" }, + output: summarizeValue(reason), + }); + return { block: true, reason }; + }; + const recordApproval = (decision: "approved" | "denied", code = "approval") => { + trace.record({ + type: "approval", + spanId: `approval:${event.toolCallId}`, + status: decision === "approved" ? "completed" : "failed", + toolName: event.toolName, + toolCallId: event.toolCallId, + attributes: { decision, code }, + }); + }; if ( event.toolName === "bash" || event.toolName === "run_command" || event.toolName === "edit" || event.toolName === "write" ) { - return { - block: true, - reason: - event.toolName === "bash" || event.toolName === "run_command" - ? "This shell tool bypasses DSCode's managed OS sandbox. Use exec_command instead." - : "This write tool bypasses DSCode checkpoints. Use apply_patch instead.", - }; + return blockTool( + event.toolName === "bash" || event.toolName === "run_command" + ? "This shell tool bypasses DSCode's managed OS sandbox. Use exec_command instead." + : "This write tool bypasses DSCode checkpoints. Use apply_patch instead.", + "bypass", + ); } if (permission === "plan" && !planAllowedTools.has(event.toolName)) { - return { - block: true, - reason: `Plan mode does not allow ${event.toolName}. Run /plan to leave plan mode.`, - }; + return blockTool( + `Plan mode does not allow ${event.toolName}. Run /plan to leave plan mode.`, + "plan_mode", + ); } const externalMcp = event.toolName.startsWith("mcp__"); const command = @@ -331,10 +442,10 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte : undefined; const dangerousCommand = command !== undefined && classifyCommand(command) === "dangerous"; if (permission === "plan" && dangerousCommand) { - return { - block: true, - reason: "Plan mode blocks destructive commands. Leave plan mode before running this command.", - }; + return blockTool( + "Plan mode blocks destructive commands. Leave plan mode before running this command.", + "dangerous_command", + ); } const needsApproval = permission === "ask" || @@ -359,18 +470,18 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte return; } if (!ctx.hasUI) { - return { - block: true, - reason: - "This action requires an interactive approval UI. Use --permission full for an explicitly trusted non-interactive run.", - }; + return blockTool( + "This action requires an interactive approval UI. Use --permission full for an explicitly trusted non-interactive run.", + "non_interactive", + ); } if (dangerousCommand) { const approved = await ctx.ui.confirm( "Run destructive command?", `${command}\n\nThis may delete data or alter system/process state.`, ); - if (!approved) return { block: true, reason: "Destructive command denied by user" }; + if (!approved) return blockTool("Destructive command denied by user", "user_denied"); + recordApproval("approved", "destructive_command"); } else if ( event.toolName === "apply_patch" && isRecord(event.input) && @@ -378,17 +489,82 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte ) { for (const section of patchApprovalSections(event.input.input)) { const approved = await ctx.ui.confirm(`Apply ${section.file}?`, section.patch); - if (!approved) return { block: true, reason: `Denied ${section.file} by user` }; + if (!approved) return blockTool(`Denied ${section.file} by user`, "user_denied"); } + recordApproval("approved", "patch"); } else { const approved = await ctx.ui.confirm( `Allow ${event.toolName}?`, approvalSummary(event.toolName, event.input), ); - if (!approved) return { block: true, reason: "Denied by user" }; + if (!approved) return blockTool("Denied by user", "user_denied"); + recordApproval("approved"); } }); + pi.on("tool_result", (event) => { + trace.record({ + type: "tool_result", + spanId: `tool:${event.toolCallId}`, + status: event.isError ? "failed" : "completed", + toolName: event.toolName, + toolCallId: event.toolCallId, + content: summarizeContent(event.content), + output: summarizeValue(event.details), + ...(event.usage ? { usage: usageFromPiUsage(event.usage) } : {}), + ...(event.isError ? { error: { code: "tool", message: "tool execution failed" } } : {}), + }); + }); + + pi.on("turn_start", (event) => { + trace.record({ + type: "turn_start", + spanId: `turn:${event.turnIndex}`, + turnId: `turn:${event.turnIndex}`, + status: "started", + attributes: { turnIndex: event.turnIndex }, + }); + }); + + pi.on("turn_end", (event) => { + trace.record({ + type: "turn_end", + spanId: `turn:${event.turnIndex}`, + turnId: `turn:${event.turnIndex}`, + status: "completed", + attributes: { toolResultCount: event.toolResults.length }, + }); + }); + + pi.on("session_before_compact", (event) => { + const spanId = nextTraceSpan("compaction"); + pendingCompactionSpans.push(spanId); + trace.record({ + type: "compaction", + spanId, + status: "started", + attributes: { + reason: event.reason, + willRetry: event.willRetry, + branchEntries: event.branchEntries.length, + }, + }); + }); + + pi.on("session_compact", (event) => { + const spanId = pendingCompactionSpans.shift() ?? nextTraceSpan("compaction-result"); + trace.record({ + type: "compaction", + spanId, + status: "completed", + attributes: { + reason: event.reason, + willRetry: event.willRetry, + tokensBefore: event.compactionEntry.tokensBefore, + }, + }); + }); + pi.on("user_bash", (_event, ctx) => { const operations: BashOperations = { exec: async (command, cwd, execution) => { @@ -420,12 +596,53 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte pi.on("message_end", (event) => { if (event.message.role !== "assistant") return; + const modelSpanId = pendingModelSpans.shift() ?? nextTraceSpan("model-response"); + const stopReason = + "stopReason" in event.message && typeof event.message.stopReason === "string" + ? event.message.stopReason + : undefined; + const usage = + "usage" in event.message && event.message.usage + ? usageFromPiUsage(event.message.usage) + : undefined; + trace.record({ + type: "model_response", + spanId: modelSpanId, + status: stopReason === "error" || stopReason === "aborted" ? "failed" : "completed", + provider: options.providerId, + model: options.modelId, + ...(usage ? { usage } : {}), + content: summarizeContent(event.message.content), + ...(stopReason ? { attributes: { stopReason } } : {}), + }); lastAgentFailed = "stopReason" in event.message && (event.message.stopReason === "error" || event.message.stopReason === "aborted"); + if (lastAgentFailed && !traceErrorRecorded) { + traceErrorRecorded = true; + trace.recordError( + nextTraceSpan("error"), + event.message.stopReason === "aborted" ? "aborted" : "provider", + "model response failed", + { + stopReason: + "stopReason" in event.message && typeof event.message.stopReason === "string" + ? event.message.stopReason + : "unknown", + }, + ); + } }); pi.on("agent_end", async (_event, ctx) => { + const traceId = trace.getTraceId(); + if (traceId) { + trace.record({ + type: "agent_end", + spanId: `agent:${traceId}`, + status: lastAgentFailed ? "failed" : "completed", + }); + } if ( permission !== "plan" || ctx.mode !== "tui" || diff --git a/packages/core/src/evaluation.ts b/packages/core/src/evaluation.ts new file mode 100644 index 0000000..a766362 --- /dev/null +++ b/packages/core/src/evaluation.ts @@ -0,0 +1,123 @@ +import type { AgentTraceEvent } from "./observability.js"; +import { replayTraceLines, summarizeTrace, type TraceReplayReport } from "./replay.js"; + +export interface EvaluationOptions { + maxToolCalls?: number; + maxDurationMs?: number; + maxTotalTokens?: number; + maxCost?: number; + requireSuccessfulRun?: boolean; + failOnErrors?: boolean; +} + +export interface EvaluationResult { + passed: boolean; + turns: number; + modelRequests: number; + toolCalls: number; + toolFailures: number; + durationMs: number; + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cost: number; + approvals: number; + errors: number; + violations: string[]; + replay: TraceReplayReport; +} + +export function evaluateTrace( + events: AgentTraceEvent[], + options: EvaluationOptions = {}, +): EvaluationResult { + return evaluateReplay(summarizeTrace(events), options); +} + +export function evaluateTraceLines( + lines: string[], + options: EvaluationOptions = {}, +): EvaluationResult { + return evaluateReplay(replayTraceLines(lines), options); +} + +export function evaluateReplay( + replay: TraceReplayReport, + options: EvaluationOptions = {}, +): EvaluationResult { + const violations = [...replay.violations]; + const requireSuccessfulRun = options.requireSuccessfulRun ?? true; + const failOnErrors = options.failOnErrors ?? true; + if (requireSuccessfulRun && replay.finalStatus !== "completed") { + violations.push(`run did not complete successfully: ${replay.finalStatus ?? "missing"}`); + } + if (failOnErrors && replay.counts.errors > 0) { + violations.push(`runtime errors: ${replay.counts.errors}`); + } + if (options.maxToolCalls !== undefined && replay.counts.toolCalls > options.maxToolCalls) { + violations.push(`tool call budget exceeded: ${replay.counts.toolCalls} > ${options.maxToolCalls}`); + } + if (options.maxDurationMs !== undefined && replay.durationMs > options.maxDurationMs) { + violations.push(`duration budget exceeded: ${replay.durationMs}ms > ${options.maxDurationMs}ms`); + } + if (options.maxTotalTokens !== undefined && replay.usage.totalTokens > options.maxTotalTokens) { + violations.push( + `token budget exceeded: ${replay.usage.totalTokens} > ${options.maxTotalTokens}`, + ); + } + if (options.maxCost !== undefined && replay.usage.cost > options.maxCost) { + violations.push(`cost budget exceeded: $${replay.usage.cost} > $${options.maxCost}`); + } + return { + passed: violations.length === 0, + turns: replay.counts.turns, + modelRequests: replay.counts.modelRequests, + toolCalls: replay.counts.toolCalls, + toolFailures: replay.counts.toolFailures, + durationMs: replay.durationMs, + inputTokens: replay.usage.inputTokens, + outputTokens: replay.usage.outputTokens, + cacheReadTokens: replay.usage.cacheReadTokens, + cost: replay.usage.cost, + approvals: replay.counts.approvals, + errors: replay.counts.errors, + violations, + replay, + }; +} + +export function formatEvaluationResult(result: EvaluationResult, json = false): string { + if (json) { + return JSON.stringify( + { + passed: result.passed, + turns: result.turns, + modelRequests: result.modelRequests, + toolCalls: result.toolCalls, + toolFailures: result.toolFailures, + durationMs: result.durationMs, + inputTokens: result.inputTokens, + outputTokens: result.outputTokens, + cacheReadTokens: result.cacheReadTokens, + cost: result.cost, + approvals: result.approvals, + errors: result.errors, + violations: result.violations, + }, + null, + 2, + ); + } + return [ + `Evaluation ${result.passed ? "PASSED" : "FAILED"}`, + `turns ${result.turns}`, + `models ${result.modelRequests}`, + `tools ${result.toolCalls} (${result.toolFailures} failure(s))`, + `duration ${result.durationMs}ms`, + `tokens ${result.inputTokens} input · ${result.outputTokens} output · ${result.cacheReadTokens} cache read`, + `cost $${result.cost.toFixed(4)}`, + `approvals ${result.approvals}`, + `errors ${result.errors}`, + ...(result.violations.length ? ["violations ", ...result.violations.map((value) => ` ${value}`)] : []), + ].join("\n"); +} diff --git a/packages/core/src/home.ts b/packages/core/src/home.ts index 13a7666..d9f704d 100644 --- a/packages/core/src/home.ts +++ b/packages/core/src/home.ts @@ -35,6 +35,12 @@ export function getDSCodeArchivedSessionsDir(): string { ); } +export function getDSCodeTracesDir(): string { + return resolveHomePath( + process.env.DSCODE_TRACE_DIR ?? path.join(getDSCodeHome(), "traces"), + ); +} + /** Configure the underlying runtime to use DSCode-owned paths only. */ export async function initializeDSCodeHome(): Promise { const home = getDSCodeHome(); @@ -51,6 +57,8 @@ export async function initializeDSCodeHome(): Promise { await fs.chmod(sessions, 0o700).catch(() => undefined); await fs.mkdir(getDSCodeArchivedSessionsDir(), { recursive: true, mode: 0o700 }); await fs.chmod(getDSCodeArchivedSessionsDir(), 0o700).catch(() => undefined); + await fs.mkdir(getDSCodeTracesDir(), { recursive: true, mode: 0o700 }); + await fs.chmod(getDSCodeTracesDir(), 0o700).catch(() => undefined); await partitionExistingSessions(sessions); return home; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0902a1c..d4d94ba 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -41,6 +41,7 @@ export { getDSCodeArchivedSessionsDir, getDSCodeHome, getDSCodeSessionsDir, + getDSCodeTracesDir, initializeDSCodeHome, migrateLegacyDSCodeHome, partitionExistingSessions, @@ -97,3 +98,39 @@ export { type PersonalizationTone, } from "./personalization.js"; export { DSCODE_VERSION } from "./version.js"; +export { + AGENT_TRACE_SCHEMA_VERSION, + AgentRuntimeTrace, + classifyHttpStatus, + summarizeContent, + summarizeValue, + usageFromPiUsage, + type AgentRuntimeTraceOptions, + type AgentTraceError, + type AgentTraceEvent, + type AgentTraceEventInput, + type AgentTraceEventType, + type AgentTraceStatus, + type AgentTraceUsage, + type TraceContentSummary, + type TraceValueSummary, +} from "./observability.js"; +export { + compareTraceReports, + formatReplayReport, + readTraceFile, + replayTraceLines, + summarizeTrace, + type TraceComparison, + type TraceCounts, + type TraceReplayReport, + type TraceUsageSummary, +} from "./replay.js"; +export { + evaluateReplay, + evaluateTrace, + evaluateTraceLines, + formatEvaluationResult, + type EvaluationOptions, + type EvaluationResult, +} from "./evaluation.js"; diff --git a/packages/core/src/observability.ts b/packages/core/src/observability.ts new file mode 100644 index 0000000..fdbed93 --- /dev/null +++ b/packages/core/src/observability.ts @@ -0,0 +1,350 @@ +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { getDSCodeTracesDir } from "./home.js"; + +export const AGENT_TRACE_SCHEMA_VERSION = 1 as const; + +export type AgentTraceEventType = + | "run_start" + | "agent_start" + | "turn_start" + | "turn_end" + | "model_request" + | "provider_response" + | "model_response" + | "tool_call" + | "tool_result" + | "approval" + | "compaction" + | "agent_end" + | "run_end" + | "error" + | "session_shutdown"; + +export type AgentTraceStatus = "started" | "completed" | "failed" | "cancelled"; + +export interface AgentTraceUsage { + input?: number; + output?: number; + cacheRead?: number; + cacheWrite?: number; + total?: number; + cost?: number; +} + +export interface TraceValueSummary { + kind: "text" | "json" | "binary" | "unknown"; + bytes: number; + sha256: string; +} + +export interface TraceContentSummary { + items: number; + textItems: number; + imageItems: number; + bytes: number; + sha256: string; +} + +export interface AgentTraceError { + code: string; + message: string; +} + +export interface AgentTraceEvent { + schemaVersion: typeof AGENT_TRACE_SCHEMA_VERSION; + traceId: string; + sessionId?: string; + runId?: string; + turnId?: string; + spanId: string; + parentSpanId?: string; + timestamp: string; + type: AgentTraceEventType; + status?: AgentTraceStatus; + durationMs?: number; + provider?: string; + model?: string; + toolName?: string; + toolCallId?: string; + usage?: AgentTraceUsage; + error?: AgentTraceError; + input?: TraceValueSummary; + output?: TraceValueSummary; + content?: TraceContentSummary; + attributes?: Record; +} + +export type AgentTraceEventInput = Omit< + AgentTraceEvent, + "schemaVersion" | "traceId" | "sessionId" | "runId" | "timestamp" | "durationMs" +> & { + durationMs?: number; +}; + +export interface AgentRuntimeTraceOptions { + traceDir?: string; + enabled?: boolean; + now?: () => Date; + createId?: () => string; +} + +interface ActiveRun { + traceId: string; + runId: string; + sessionId?: string; + spanId: string; + filePath: string; + startedAt: number; + closed: boolean; +} + +/** + * Small, local-first event sink for agent runs. Writes are queued so telemetry + * cannot block the agent loop; callers can await flush at lifecycle boundaries. + */ +export class AgentRuntimeTrace { + private readonly traceDir: string; + private readonly enabled: boolean; + private readonly now: () => Date; + private readonly createId: () => string; + private sessionId: string | undefined; + private activeRun: ActiveRun | undefined; + private readonly spanStarts = new Map(); + private writeQueue: Promise = Promise.resolve(); + + constructor(options: AgentRuntimeTraceOptions = {}) { + this.traceDir = path.resolve(options.traceDir ?? getDSCodeTracesDir()); + this.enabled = + options.enabled ?? + !["0", "false", "off"].includes((process.env.DSCODE_TRACE ?? "1").toLowerCase()); + this.now = options.now ?? (() => new Date()); + this.createId = options.createId ?? randomUUID; + } + + setSession(sessionId: string | undefined): void { + this.sessionId = sessionId; + } + + startRun(metadata: { + sessionId?: string; + provider?: string; + model?: string; + } = {}): string { + if (this.activeRun && !this.activeRun.closed) this.endRun("cancelled"); + this.spanStarts.clear(); + const traceId = this.createId(); + const runId = this.createId(); + const sessionId = metadata.sessionId ?? this.sessionId; + const startedAt = this.now().getTime(); + this.activeRun = { + traceId, + runId, + ...(sessionId ? { sessionId } : {}), + spanId: `run:${runId}`, + filePath: path.join(this.traceDir, `${traceFileTimestamp(startedAt)}-${traceId}.jsonl`), + startedAt, + closed: false, + }; + this.record({ + type: "run_start", + spanId: `run:${runId}`, + status: "started", + ...(metadata.provider ? { provider: metadata.provider } : {}), + ...(metadata.model ? { model: metadata.model } : {}), + }); + return traceId; + } + + record(event: AgentTraceEventInput): void { + const run = this.activeRun; + if (!run || run.closed) return; + const timestamp = this.now(); + const timestampMs = timestamp.getTime(); + const durationMs = + event.durationMs ?? + (event.status !== "started" && this.spanStarts.has(event.spanId) + ? Math.max(0, timestampMs - this.spanStarts.get(event.spanId)!) + : undefined); + if (event.status === "started") this.spanStarts.set(event.spanId, timestampMs); + if (event.status !== "started") this.spanStarts.delete(event.spanId); + + const traceEvent: AgentTraceEvent = { + schemaVersion: AGENT_TRACE_SCHEMA_VERSION, + traceId: run.traceId, + ...(run.sessionId ? { sessionId: run.sessionId } : {}), + runId: run.runId, + timestamp: timestamp.toISOString(), + ...event, + ...(durationMs === undefined ? {} : { durationMs }), + }; + this.enqueue(traceEvent, run.filePath); + } + + recordError( + spanId: string, + code: string, + message = "runtime error", + attributes?: Record, + ): void { + this.record({ + type: "error", + spanId, + status: "failed", + error: { code, message: sanitizeErrorMessage(message) }, + ...(attributes ? { attributes } : {}), + }); + } + + endRun(status: Exclude = "completed"): void { + const run = this.activeRun; + if (!run || run.closed) return; + this.record({ type: "run_end", spanId: run.spanId, status }); + run.closed = true; + this.spanStarts.delete(run.spanId); + } + + shutdown(reason: "completed" | "cancelled" = "cancelled"): void { + if (this.activeRun && !this.activeRun.closed) { + this.record({ + type: "session_shutdown", + spanId: `shutdown:${this.activeRun.runId}`, + status: reason === "completed" ? "completed" : "cancelled", + }); + this.endRun(reason); + } + } + + getTracePath(): string | undefined { + return this.activeRun?.filePath; + } + + getTraceId(): string | undefined { + return this.activeRun?.traceId; + } + + async flush(): Promise { + await this.writeQueue; + } + + private enqueue(event: AgentTraceEvent, filePath: string): void { + if (!this.enabled) return; + const line = `${JSON.stringify(event)}\n`; + this.writeQueue = this.writeQueue + .then(async () => { + await fs.mkdir(this.traceDir, { recursive: true, mode: 0o700 }); + await fs.appendFile(filePath, line, { encoding: "utf8", mode: 0o600 }); + await fs.chmod(filePath, 0o600).catch(() => undefined); + }) + .catch(() => undefined); + } +} + +export function summarizeValue(value: unknown): TraceValueSummary { + if (typeof value === "string") return summarizeBytes("text", value); + if (isBinaryValue(value)) return summarizeBytes("binary", value); + const serialized = safeSerialize(value); + return summarizeBytes(serialized === undefined ? "unknown" : "json", serialized ?? ""); +} + +export function summarizeContent(content: unknown): TraceContentSummary { + const items = Array.isArray(content) ? content : []; + let textItems = 0; + let imageItems = 0; + let bytes = 0; + const digests: string[] = []; + for (const item of items) { + if (!isRecord(item)) { + digests.push(safeSerialize(item) ?? "unknown"); + continue; + } + if (item.type === "text" || item.type === "output_text") { + textItems += 1; + const text = typeof item.text === "string" ? item.text : ""; + bytes += Buffer.byteLength(text, "utf8"); + digests.push(text); + } else if (item.type === "image" || item.type === "input_image") { + imageItems += 1; + const data = typeof item.data === "string" ? item.data : ""; + bytes += Buffer.byteLength(data, "utf8"); + digests.push(data); + } else { + const serialized = safeSerialize(item) ?? "unknown"; + bytes += Buffer.byteLength(serialized, "utf8"); + digests.push(serialized); + } + } + return { + items: items.length, + textItems, + imageItems, + bytes, + sha256: digest(digests.join("\n")), + }; +} + +export function usageFromPiUsage(usage: { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + totalTokens: number; + cost: { total: number }; +}): AgentTraceUsage { + return { + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + total: usage.totalTokens, + cost: usage.cost.total, + }; +} + +export function classifyHttpStatus(status: number): string { + if (status === 401 || status === 403) return "auth"; + if (status === 408 || status === 504) return "timeout"; + if (status === 429) return "rate_limit"; + if (status >= 400 && status < 500) return "provider_request"; + if (status >= 500) return "provider_unavailable"; + return "unknown"; +} + +function summarizeBytes(kind: TraceValueSummary["kind"], value: string | Uint8Array): TraceValueSummary { + const bytes = typeof value === "string" ? Buffer.byteLength(value, "utf8") : value.byteLength; + return { kind, bytes, sha256: digest(value) }; +} + +function digest(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function safeSerialize(value: unknown): string | undefined { + try { + return JSON.stringify(value, (_key, nested) => + typeof nested === "bigint" ? `${nested.toString()}n` : nested, + ); + } catch { + return undefined; + } +} + +function isBinaryValue(value: unknown): value is Uint8Array { + return value instanceof Uint8Array || Buffer.isBuffer(value); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function sanitizeErrorMessage(message: string): string { + return message + .replace(/https?:\/\/\S+/gi, "") + .replace(/(?:sk|key|token|secret)[-_a-z0-9]*\s*[:=]\s*\S+/gi, "") + .slice(0, 160); +} + +function traceFileTimestamp(timestamp: number): string { + return new Date(timestamp).toISOString().replace(/[:.]/g, "-"); +} diff --git a/packages/core/src/replay.ts b/packages/core/src/replay.ts new file mode 100644 index 0000000..5520720 --- /dev/null +++ b/packages/core/src/replay.ts @@ -0,0 +1,299 @@ +import fs from "node:fs/promises"; +import type { + AgentTraceEvent, + AgentTraceEventType, + AgentTraceStatus, + AgentTraceUsage, +} from "./observability.js"; +import { AGENT_TRACE_SCHEMA_VERSION } from "./observability.js"; + +export interface TraceUsageSummary { + inputTokens: number; + outputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + totalTokens: number; + cost: number; +} + +export interface TraceCounts { + runs: number; + turns: number; + modelRequests: number; + providerResponses: number; + modelResponses: number; + toolCalls: number; + toolResults: number; + toolFailures: number; + approvals: number; + errors: number; + compactions: number; +} + +export interface TraceReplayReport { + source?: string; + valid: boolean; + events: number; + invalidLines: number; + parseErrors: string[]; + traceIds: string[]; + durationMs: number; + counts: TraceCounts; + usage: TraceUsageSummary; + spans: { + total: number; + closed: number; + unclosed: string[]; + }; + unmatchedToolCalls: string[]; + missingRunEnds: string[]; + finalStatus?: AgentTraceStatus; + violations: string[]; +} + +export interface TraceComparison { + baseline: TraceReplayReport; + candidate: TraceReplayReport; + deltas: Record; + differences: string[]; +} + +const TRACE_EVENT_TYPES = new Set([ + "run_start", + "agent_start", + "turn_start", + "turn_end", + "model_request", + "provider_response", + "model_response", + "tool_call", + "tool_result", + "approval", + "compaction", + "agent_end", + "run_end", + "error", + "session_shutdown", +]); + +export async function readTraceFile(filePath: string): Promise { + const content = await fs.readFile(filePath, "utf8"); + return replayTraceLines(content.split(/\r?\n/), filePath); +} + +export function replayTraceLines(lines: string[], source?: string): TraceReplayReport { + const events: AgentTraceEvent[] = []; + const parseErrors: string[] = []; + let invalidLines = 0; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]?.trim(); + if (!line) continue; + try { + const value: unknown = JSON.parse(line); + if (!isTraceEvent(value)) throw new Error("invalid event envelope"); + events.push(value); + } catch (error) { + invalidLines += 1; + if (parseErrors.length < 10) { + parseErrors.push(`line ${index + 1}: ${error instanceof Error ? error.message : "invalid JSON"}`); + } + } + } + return summarizeTrace(events, { + ...(source ? { source } : {}), + parseErrors, + invalidLines, + }); +} + +export function summarizeTrace( + events: AgentTraceEvent[], + options: { source?: string; parseErrors?: string[]; invalidLines?: number } = {}, +): TraceReplayReport { + const counts: TraceCounts = { + runs: 0, + turns: 0, + modelRequests: 0, + providerResponses: 0, + modelResponses: 0, + toolCalls: 0, + toolResults: 0, + toolFailures: 0, + approvals: 0, + errors: 0, + compactions: 0, + }; + const usage: TraceUsageSummary = { + inputTokens: 0, + outputTokens: 0, + cacheReadTokens: 0, + cacheWriteTokens: 0, + totalTokens: 0, + cost: 0, + }; + const traceIds = new Set(); + const starts = new Map(); + const closed = new Set(); + const toolCalls = new Set(); + const toolResults = new Set(); + const runStarts = new Set(); + const runEnds = new Set(); + let firstTimestamp: number | undefined; + let lastTimestamp: number | undefined; + let finalStatus: AgentTraceStatus | undefined; + + for (const event of events) { + traceIds.add(event.traceId); + const timestamp = Date.parse(event.timestamp); + if (Number.isFinite(timestamp)) { + firstTimestamp = firstTimestamp === undefined ? timestamp : Math.min(firstTimestamp, timestamp); + lastTimestamp = lastTimestamp === undefined ? timestamp : Math.max(lastTimestamp, timestamp); + } + if (event.status === "started") starts.set(event.spanId, event); + if (event.status !== "started") closed.add(event.spanId); + addUsage(usage, event.usage); + switch (event.type) { + case "run_start": + counts.runs += 1; + runStarts.add(event.runId ?? event.spanId); + break; + case "turn_start": + counts.turns += 1; + break; + case "model_request": + counts.modelRequests += 1; + break; + case "provider_response": + counts.providerResponses += 1; + break; + case "model_response": + counts.modelResponses += 1; + break; + case "tool_call": + counts.toolCalls += 1; + if (event.toolCallId) toolCalls.add(event.toolCallId); + break; + case "tool_result": + counts.toolResults += 1; + if (event.status === "failed") counts.toolFailures += 1; + if (event.toolCallId) toolResults.add(event.toolCallId); + break; + case "approval": + counts.approvals += 1; + break; + case "error": + counts.errors += 1; + break; + case "compaction": + counts.compactions += 1; + break; + case "run_end": + runEnds.add(event.runId ?? event.spanId); + finalStatus = event.status; + break; + default: + break; + } + } + + const unclosed = [...starts.keys()].filter((spanId) => !closed.has(spanId)); + const unmatchedToolCalls = [...toolCalls].filter((toolCallId) => !toolResults.has(toolCallId)); + const missingRunEnds = [...runStarts].filter((runId) => !runEnds.has(runId)); + const parseErrors = options.parseErrors ?? []; + const violations = [ + ...parseErrors.map((error) => `trace parse error: ${error}`), + ...(unclosed.length ? [`unclosed spans: ${unclosed.join(", ")}`] : []), + ...(unmatchedToolCalls.length + ? [`unmatched tool calls: ${unmatchedToolCalls.join(", ")}`] + : []), + ...(missingRunEnds.length ? [`runs without run_end: ${missingRunEnds.join(", ")}`] : []), + ]; + return { + ...(options.source ? { source: options.source } : {}), + valid: violations.length === 0, + events: events.length, + invalidLines: options.invalidLines ?? 0, + parseErrors, + traceIds: [...traceIds], + durationMs: + firstTimestamp === undefined || lastTimestamp === undefined + ? 0 + : Math.max(0, lastTimestamp - firstTimestamp), + counts, + usage, + spans: { total: starts.size + [...closed].filter((spanId) => !starts.has(spanId)).length, closed: closed.size, unclosed }, + unmatchedToolCalls, + missingRunEnds, + ...(finalStatus ? { finalStatus } : {}), + violations, + }; +} + +export function compareTraceReports( + baseline: TraceReplayReport, + candidate: TraceReplayReport, +): TraceComparison { + const deltas: Record = { + durationMs: candidate.durationMs - baseline.durationMs, + modelRequests: candidate.counts.modelRequests - baseline.counts.modelRequests, + modelResponses: candidate.counts.modelResponses - baseline.counts.modelResponses, + toolCalls: candidate.counts.toolCalls - baseline.counts.toolCalls, + toolFailures: candidate.counts.toolFailures - baseline.counts.toolFailures, + errors: candidate.counts.errors - baseline.counts.errors, + totalTokens: candidate.usage.totalTokens - baseline.usage.totalTokens, + cost: candidate.usage.cost - baseline.usage.cost, + }; + const differences = Object.entries(deltas) + .filter(([, value]) => value !== 0) + .map(([key, value]) => `${key}: ${formatDelta(value)}`); + if (baseline.valid !== candidate.valid) differences.push(`valid: ${String(baseline.valid)} -> ${String(candidate.valid)}`); + if (baseline.finalStatus !== candidate.finalStatus) { + differences.push(`finalStatus: ${baseline.finalStatus ?? "none"} -> ${candidate.finalStatus ?? "none"}`); + } + return { baseline, candidate, deltas, differences }; +} + +export function formatReplayReport(report: TraceReplayReport, json = false): string { + if (json) return JSON.stringify(report, null, 2); + return [ + `Trace ${report.source ?? report.traceIds[0] ?? "unknown"}`, + `valid ${report.valid ? "yes" : "no"}`, + `events ${report.events}`, + `duration ${report.durationMs}ms`, + `runs ${report.counts.runs} (${report.finalStatus ?? "open"})`, + `turns ${report.counts.turns}`, + `models ${report.counts.modelRequests} request(s), ${report.counts.modelResponses} response(s)`, + `tools ${report.counts.toolCalls} call(s), ${report.counts.toolFailures} failure(s)`, + `usage ${report.usage.totalTokens} tokens · $${report.usage.cost.toFixed(4)}`, + `spans ${report.spans.closed}/${report.spans.total} closed`, + ...(report.violations.length ? ["violations ", ...report.violations.map((value) => ` ${value}`)] : []), + ].join("\n"); +} + +function isTraceEvent(value: unknown): value is AgentTraceEvent { + if (typeof value !== "object" || value === null) return false; + const event = value as Partial; + return ( + event.schemaVersion === AGENT_TRACE_SCHEMA_VERSION && + typeof event.traceId === "string" && + typeof event.spanId === "string" && + typeof event.timestamp === "string" && + Number.isFinite(Date.parse(event.timestamp)) && + typeof event.type === "string" && + TRACE_EVENT_TYPES.has(event.type as AgentTraceEventType) + ); +} + +function addUsage(summary: TraceUsageSummary, usage: AgentTraceUsage | undefined): void { + if (!usage) return; + summary.inputTokens += usage.input ?? 0; + summary.outputTokens += usage.output ?? 0; + summary.cacheReadTokens += usage.cacheRead ?? 0; + summary.cacheWriteTokens += usage.cacheWrite ?? 0; + summary.totalTokens += usage.total ?? 0; + summary.cost += usage.cost ?? 0; +} + +function formatDelta(value: number): string { + return Number.isInteger(value) ? (value > 0 ? `+${value}` : String(value)) : `${value > 0 ? "+" : ""}${value.toFixed(4)}`; +} diff --git a/packages/core/src/runtime-options.ts b/packages/core/src/runtime-options.ts index 0dcebde..f46dc8f 100644 --- a/packages/core/src/runtime-options.ts +++ b/packages/core/src/runtime-options.ts @@ -209,6 +209,8 @@ export function printDSCodeHelp(): void { Usage: dscode [options] [prompt] + dscode replay [--json] + dscode evaluate [--json] [budget options] dscode -p "task" Non-interactive text mode dscode --mode json -p "task" JSONL/CI mode dscode --mode rpc IDE/RPC server @@ -238,6 +240,11 @@ Session and editor features: DSCode commands: /plan /permissions /effort /base-url /status /undo /checkpoints /diff /jobs /mcp /agents /doctor +Runtime traces and evaluation: + dscode replay --compare Compare two trace projections + --max-tool-calls --max-duration-ms --max-total-tokens --max-cost + --allow-errors Do not fail evaluation on runtime errors + Authentication: dscode login [provider] Sign in to a supported model provider dscode logout [provider] Remove the selected provider credential diff --git a/test/cli-integration.test.ts b/test/cli-integration.test.ts index 89c2072..eeb0af0 100644 --- a/test/cli-integration.test.ts +++ b/test/cli-integration.test.ts @@ -1,10 +1,13 @@ +import fs from "node:fs/promises"; import { spawn } from "node:child_process"; import http from "node:http"; +import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; describe("DSCode Pi integration", () => { let server: http.Server | undefined; + let traceDir: string | undefined; afterEach(async () => { if (!server) return; @@ -12,9 +15,12 @@ describe("DSCode Pi integration", () => { server!.close((error) => (error ? reject(error) : resolve())), ); server = undefined; + if (traceDir) await fs.rm(traceDir, { recursive: true, force: true }); + traceDir = undefined; }); it("runs a JSONL turn through the DeepSeek Responses adapter", async () => { + traceDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-cli-trace-")); let payload: Record | undefined; server = http.createServer(async (request, response) => { const body: Buffer[] = []; @@ -100,6 +106,7 @@ describe("DSCode Pi integration", () => { DEEPSEEK_API_KEY: "test-only-key", PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0", + DSCODE_TRACE_DIR: traceDir, }, ); @@ -115,9 +122,20 @@ describe("DSCode Pi integration", () => { expect.objectContaining({ type: "custom", name: "apply_patch" }), ]), ); + const traceFiles = (await fs.readdir(traceDir)).filter((file) => file.endsWith(".jsonl")); + expect(traceFiles).toHaveLength(1); + const trace = await fs.readFile(path.join(traceDir, traceFiles[0]!), "utf8"); + const events = trace + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { type: string }); + expect(events.map((event) => event.type)).toEqual( + expect.arrayContaining(["run_start", "model_request", "provider_response", "model_response", "run_end"]), + ); }, 15_000); it("returns a non-zero CI exit code on provider failure", async () => { + traceDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-cli-error-trace-")); server = http.createServer(async (_request, response) => { response.writeHead(401, { "content-type": "application/json" }); response.end(JSON.stringify({ error: { message: "invalid test key" } })); @@ -144,11 +162,24 @@ describe("DSCode Pi integration", () => { DEEPSEEK_API_KEY: "invalid-test-key", PI_SKIP_VERSION_CHECK: "1", PI_TELEMETRY: "0", + DSCODE_TRACE_DIR: traceDir, }, ); expect(execution.exitCode).not.toBe(0); expect(`${execution.stdout}\n${execution.stderr}`).toContain("invalid test key"); + const traceFiles = (await fs.readdir(traceDir)).filter((file) => file.endsWith(".jsonl")); + expect(traceFiles).toHaveLength(1); + const traceLines = (await fs.readFile(path.join(traceDir, traceFiles[0]!), "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as { type: string; status?: string }); + expect(traceLines).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "error" }), + expect.objectContaining({ type: "run_end", status: "failed" }), + ]), + ); }, 15_000); }); diff --git a/test/observability.test.ts b/test/observability.test.ts new file mode 100644 index 0000000..41d414b --- /dev/null +++ b/test/observability.test.ts @@ -0,0 +1,142 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { AgentRuntimeTrace, summarizeValue } from "../packages/core/src/observability.js"; +import { evaluateTrace } from "../packages/core/src/evaluation.js"; +import { readTraceFile, replayTraceLines } from "../packages/core/src/replay.js"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +describe("agent runtime observability", () => { + it("writes a redacted, replayable JSONL trace without blocking record calls", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-trace-")); + temporaryDirectories.push(directory); + let timestamp = 1_700_000_000_000; + const ids = ["trace-1", "run-1"]; + const trace = new AgentRuntimeTrace({ + traceDir: directory, + now: () => new Date(timestamp += 10), + createId: () => ids.shift() ?? "generated-id", + }); + trace.setSession("session-1"); + trace.startRun({ provider: "deepseek", model: "test-model" }); + trace.record({ + type: "agent_start", + spanId: "agent:run-1", + status: "started", + input: summarizeValue("api_key=do-not-store-this"), + }); + trace.record({ + type: "model_request", + spanId: "model:1", + status: "started", + input: summarizeValue({ prompt: "private prompt" }), + }); + trace.record({ + type: "model_response", + spanId: "model:1", + status: "completed", + usage: { input: 10, output: 4, total: 14, cost: 0.002 }, + }); + trace.record({ + type: "tool_call", + spanId: "tool:1", + status: "started", + toolCallId: "call-1", + toolName: "read_file", + input: summarizeValue({ path: "/private/file.txt" }), + }); + trace.record({ + type: "tool_result", + spanId: "tool:1", + status: "completed", + toolCallId: "call-1", + toolName: "read_file", + output: summarizeValue("private file contents"), + }); + trace.record({ type: "agent_end", spanId: "agent:run-1", status: "completed" }); + trace.endRun("completed"); + await trace.flush(); + + const tracePath = trace.getTracePath(); + expect(tracePath).toBeDefined(); + const raw = await fs.readFile(tracePath!, "utf8"); + expect(raw).not.toContain("private prompt"); + expect(raw).not.toContain("private file contents"); + expect(raw).not.toContain("do-not-store-this"); + + const report = await readTraceFile(tracePath!); + expect(report.valid).toBe(true); + expect(report.counts.modelRequests).toBe(1); + expect(report.counts.toolCalls).toBe(1); + expect(report.counts.toolFailures).toBe(0); + expect(report.usage.totalTokens).toBe(14); + expect(report.usage.cost).toBe(0.002); + }); + + it("reports malformed and structurally incomplete traces", () => { + const report = replayTraceLines([ + JSON.stringify({ + schemaVersion: 1, + traceId: "trace-1", + runId: "run-1", + spanId: "run:run-1", + timestamp: new Date().toISOString(), + type: "run_start", + status: "started", + }), + JSON.stringify({ + schemaVersion: 1, + traceId: "trace-1", + runId: "run-1", + spanId: "tool:call-1", + timestamp: new Date().toISOString(), + type: "tool_call", + status: "started", + toolCallId: "call-1", + }), + "not-json", + ]); + + expect(report.valid).toBe(false); + expect(report.invalidLines).toBe(1); + expect(report.unmatchedToolCalls).toEqual(["call-1"]); + expect(report.missingRunEnds).toEqual(["run-1"]); + expect(report.violations.join("\n")).toContain("trace parse error"); + }); + + it("evaluates success and budget constraints from the same event projection", () => { + const timestamp = new Date().toISOString(); + const events = [ + { + schemaVersion: 1 as const, + traceId: "trace-1", + runId: "run-1", + spanId: "run:run-1", + timestamp, + type: "run_start" as const, + status: "started" as const, + }, + { + schemaVersion: 1 as const, + traceId: "trace-1", + runId: "run-1", + spanId: "run:run-1", + timestamp, + type: "run_end" as const, + status: "completed" as const, + }, + ]; + const result = evaluateTrace(events, { maxToolCalls: 0 }); + expect(result.passed).toBe(true); + expect(result.violations).toEqual([]); + expect(evaluateTrace(events, { maxToolCalls: -1 }).passed).toBe(false); + }); +}); From 36a4a1adf06d11b948cc0c8f65cb57c4fa62f1f7 Mon Sep 17 00:00:00 2001 From: Chal1ce <752193262@qq.com> Date: Fri, 14 Aug 2026 22:28:23 +0800 Subject: [PATCH 02/13] feat(runtime): add deterministic provider fixture replay --- packages/core/src/cli-runtime.ts | 154 +++++++++++ packages/core/src/dscode-extension.ts | 47 +++- packages/core/src/evaluation.ts | 3 +- packages/core/src/fixture.ts | 377 ++++++++++++++++++++++++++ packages/core/src/index.ts | 16 ++ packages/core/src/replay.ts | 4 + packages/core/src/runtime-options.ts | 8 + test/cli-integration.test.ts | 172 ++++++++++++ test/fixture.test.ts | 113 ++++++++ 9 files changed, 888 insertions(+), 6 deletions(-) create mode 100644 packages/core/src/fixture.ts create mode 100644 test/fixture.test.ts diff --git a/packages/core/src/cli-runtime.ts b/packages/core/src/cli-runtime.ts index 00ab71d..c1a1a83 100644 --- a/packages/core/src/cli-runtime.ts +++ b/packages/core/src/cli-runtime.ts @@ -1,3 +1,7 @@ +import fs from "node:fs/promises"; +import { spawn } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; import process from "node:process"; import pc from "picocolors"; import { ensureFirstRunAuth, runAuthCommand } from "./auth.js"; @@ -12,6 +16,7 @@ import { formatEvaluationResult, type EvaluationOptions, } from "./evaluation.js"; +import { readAgentFixtureFile } from "./fixture.js"; import { compareTraceReports, formatReplayReport, @@ -80,6 +85,8 @@ interface TraceCommand { command: "replay" | "evaluate"; input: string; compare?: string; + execute?: boolean; + prompt?: string; json: boolean; evaluation: EvaluationOptions; } @@ -90,6 +97,8 @@ function parseTraceCommand(argv: string[]): TraceCommand | undefined { const paths: string[] = []; let json = false; let compare = false; + let execute = false; + let prompt: string | undefined; const evaluation: EvaluationOptions = {}; for (let index = 1; index < argv.length; index += 1) { const argument = argv[index]!; @@ -102,6 +111,18 @@ function parseTraceCommand(argv: string[]): TraceCommand | undefined { continue; } const [flag, inlineValue] = splitTraceFlag(argument); + if (flag === "--execute") { + const value = inlineValue ?? argv[++index]; + if (!value) throw new Error("--execute requires a fixture file"); + execute = true; + paths.push(value); + continue; + } + if (flag === "--prompt") { + prompt = inlineValue ?? argv[++index]; + if (!prompt) throw new Error("--prompt requires a value"); + continue; + } if ( flag === "--max-tool-calls" || flag === "--max-duration-ms" || @@ -126,6 +147,7 @@ function parseTraceCommand(argv: string[]): TraceCommand | undefined { paths.push(argument); } if (compare) { + if (execute) throw new Error("replay --execute cannot be combined with --compare"); if (paths.length !== 2) throw new Error("replay --compare requires baseline and candidate trace files"); return { command: "replay", input: paths[0]!, compare: paths[1]!, json, evaluation }; } @@ -133,12 +155,18 @@ function parseTraceCommand(argv: string[]): TraceCommand | undefined { return { command: command === "eval" ? "evaluate" : command, input: paths[0]!, + ...(execute ? { execute } : {}), + ...(prompt ? { prompt } : {}), json, evaluation, }; } async function runTraceCommand(command: TraceCommand): Promise { + if (command.execute) { + await runFixtureReplayCommand(command); + return; + } const report = await readTraceFile(command.input); if (command.compare) { const comparison = compareTraceReports(report, await readTraceFile(command.compare)); @@ -160,6 +188,132 @@ async function runTraceCommand(command: TraceCommand): Promise { if (!result.passed) process.exitCode = 1; } +async function runFixtureReplayCommand(command: TraceCommand): Promise { + const fixturePath = path.resolve(command.input); + const fixture = await readAgentFixtureFile(fixturePath); + if (fixture.provider !== "deepseek") { + throw new Error(`Fixture provider "${fixture.provider}" is not supported by this runtime yet`); + } + const runDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-fixture-replay-")); + const traceDirectory = path.join(runDirectory, "traces"); + const homeDirectory = path.join(runDirectory, "home"); + try { + const nestedExitCode = await runFixtureRuntime({ + fixturePath, + fixtureModel: fixture.model, + prompt: command.prompt ?? "Replay fixture", + homeDirectory, + traceDirectory, + }); + const traceFiles = (await fs.readdir(traceDirectory)) + .filter((file) => file.endsWith(".jsonl")) + .sort(); + if (traceFiles.length === 0) { + throw new Error("Fixture replay completed without producing a trace"); + } + const report = await readTraceFile(path.join(traceDirectory, traceFiles[traceFiles.length - 1]!)); + const result = evaluateReplay( + report, + { ...(fixture.evaluation ?? {}), ...command.evaluation }, + ); + const assertionViolations = fixtureAssertionViolations(fixture, report); + result.violations.push(...assertionViolations); + if (nestedExitCode !== 0 && report.finalStatus === "completed") { + result.violations.push(`fixture runtime exited with code ${nestedExitCode}`); + } + result.passed = result.violations.length === 0; + process.stdout.write(`${formatEvaluationResult(result, command.json)}\n`); + process.exitCode = result.passed ? 0 : 1; + } finally { + await fs.rm(runDirectory, { recursive: true, force: true }); + } +} + +interface FixtureRuntimeOptions { + fixturePath: string; + fixtureModel: string; + prompt: string; + homeDirectory: string; + traceDirectory: string; +} + +async function runFixtureRuntime(options: FixtureRuntimeOptions): Promise { + const invocation = currentCliInvocation(); + const child = spawn( + invocation.command, + [ + ...invocation.args, + "--provider", + "deepseek", + "--model", + options.fixtureModel, + "--permission", + "full", + "--mode", + "json", + "--print", + "--no-session", + "--no-approve", + options.prompt, + ], + { + env: { + ...process.env, + DSCODE_HOME: options.homeDirectory, + DSCODE_SESSIONS_DIR: path.join(options.homeDirectory, "sessions"), + DSCODE_TRACE_DIR: options.traceDirectory, + DSCODE_TRACE: "1", + DSCODE_FIXTURE_PATH: options.fixturePath, + DEEPSEEK_API_KEY: "dscode-fixture", + PI_TELEMETRY: "0", + PI_SKIP_VERSION_CHECK: "1", + }, + stdio: "ignore", + }, + ); + return await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve(code ?? 1)); + }); +} + +function currentCliInvocation(): { command: string; args: string[] } { + const [nodePath, launcher, source] = process.argv; + if (!nodePath || !launcher) throw new Error("Could not determine the current DSCode CLI invocation"); + if (source?.endsWith(".ts")) { + return { command: nodePath, args: [launcher, source] }; + } + return { command: nodePath, args: [launcher] }; +} + +function fixtureAssertionViolations( + fixture: Awaited>, + report: Awaited>, +): string[] { + const violations: string[] = []; + if (report.counts.modelResponses !== fixture.responses.length) { + violations.push( + `fixture responses consumed: ${report.counts.modelResponses} != ${fixture.responses.length}`, + ); + } + const assertions = fixture.assertions; + if (!assertions) return violations; + if (assertions.finalStatus && report.finalStatus !== assertions.finalStatus) { + violations.push(`expected final status ${assertions.finalStatus}, got ${report.finalStatus ?? "missing"}`); + } + if (assertions.modelResponses !== undefined && report.counts.modelResponses !== assertions.modelResponses) { + violations.push( + `expected model responses ${assertions.modelResponses}, got ${report.counts.modelResponses}`, + ); + } + if (assertions.toolNames && JSON.stringify(report.toolNames) !== JSON.stringify(assertions.toolNames)) { + violations.push( + `expected tool sequence ${JSON.stringify(assertions.toolNames)}, got ${JSON.stringify(report.toolNames)}`, + ); + } + return violations; +} + /** Process-oriented wrapper used by the terminal and bundled RPC entry points. */ export async function runDSCodeProcess(argv: string[]): Promise { try { diff --git a/packages/core/src/dscode-extension.ts b/packages/core/src/dscode-extension.ts index cd9d245..21fc718 100644 --- a/packages/core/src/dscode-extension.ts +++ b/packages/core/src/dscode-extension.ts @@ -30,6 +30,11 @@ import { registerLocalImageInput } from "./image-input.js"; import { partitionSessionFile } from "./home.js"; import { ManagedProcessRegistry, type ManagedProcessResult } from "./managed-process.js"; import { MCPManager } from "./mcp.js"; +import { + AgentFixtureRecorder, + AgentFixtureReplay, + readAgentFixtureFileSync, +} from "./fixture.js"; import { AgentRuntimeTrace, classifyHttpStatus, @@ -150,6 +155,12 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte let lastOfferedPlanRevision = 0; const access = new SessionAccessController(options.sandbox, options.network); const trace = new AgentRuntimeTrace(); + const fixturePath = process.env.DSCODE_FIXTURE_PATH?.trim(); + const fixture = fixturePath ? readAgentFixtureFileSync(fixturePath) : undefined; + const fixtureReplay = fixture ? new AgentFixtureReplay(fixture) : undefined; + const fixtureRecorder = options.fixtureCapturePath + ? new AgentFixtureRecorder(options.fixtureCapturePath, options.providerId, options.modelId) + : undefined; const pendingModelSpans: string[] = []; const pendingCompactionSpans: string[] = []; let traceSequence = 0; @@ -184,7 +195,7 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte ctx.ui.setWidget("dscode-plan", planWidgetLines(planState, ctx)); }; - registerDeepSeekProvider(pi, options); + registerDeepSeekProvider(pi, options, fixtureReplay); registerLocalImageInput(pi); registerNaturalExit(pi); registerSessionCommands(pi); @@ -319,12 +330,14 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte pi.on("agent_settled", async (_event, ctx) => { trace.endRun(lastAgentFailed ? "failed" : "completed"); + await fixtureRecorder?.finish(); await trace.flush(); await queueSessionPartition(ctx); }); pi.on("session_shutdown", async (_event, ctx) => { trace.shutdown("cancelled"); + await fixtureRecorder?.finish(); await trace.flush(); await queueSessionPartition(ctx); processes.dispose(); @@ -516,6 +529,19 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte }); }); + pi.on("message_start", (event) => { + if (event.message.role !== "assistant" || pendingModelSpans.length > 0) return; + const spanId = nextTraceSpan("model"); + pendingModelSpans.push(spanId); + trace.record({ + type: "model_request", + spanId, + status: "started", + provider: event.message.provider, + model: event.message.model, + }); + }); + pi.on("turn_start", (event) => { trace.record({ type: "turn_start", @@ -596,6 +622,7 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte pi.on("message_end", (event) => { if (event.message.role !== "assistant") return; + fixtureRecorder?.record(event.message); const modelSpanId = pendingModelSpans.shift() ?? nextTraceSpan("model-response"); const stopReason = "stopReason" in event.message && typeof event.message.stopReason === "string" @@ -979,8 +1006,16 @@ export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExte }; } -function registerDeepSeekProvider(pi: ExtensionAPI, options: DSCodeRuntimeOptions): void { - const api = options.transport === "responses" ? "openai-responses" : "openai-completions"; +function registerDeepSeekProvider( + pi: ExtensionAPI, + options: DSCodeRuntimeOptions, + fixtureReplay?: AgentFixtureReplay, +): void { + const api = fixtureReplay + ? "dscode-fixture" + : options.transport === "responses" + ? "openai-responses" + : "openai-completions"; const models = [ { id: "deepseek-v4-flash", @@ -1009,9 +1044,11 @@ function registerDeepSeekProvider(pi: ExtensionAPI, options: DSCodeRuntimeOption pi.registerProvider("deepseek", { name: "DeepSeek", baseUrl: options.baseUrl, - apiKey: "$DEEPSEEK_API_KEY", + apiKey: fixtureReplay ? "dscode-fixture" : "$DEEPSEEK_API_KEY", api, - authHeader: true, + ...(fixtureReplay + ? { streamSimple: (model, context, streamOptions) => fixtureReplay.stream(model, context, streamOptions) } + : { authHeader: true }), models: models.map((model) => ({ id: model.id, name: model.name, diff --git a/packages/core/src/evaluation.ts b/packages/core/src/evaluation.ts index a766362..4bd206c 100644 --- a/packages/core/src/evaluation.ts +++ b/packages/core/src/evaluation.ts @@ -93,6 +93,7 @@ export function formatEvaluationResult(result: EvaluationResult, json = false): passed: result.passed, turns: result.turns, modelRequests: result.modelRequests, + modelResponses: result.replay.counts.modelResponses, toolCalls: result.toolCalls, toolFailures: result.toolFailures, durationMs: result.durationMs, @@ -111,7 +112,7 @@ export function formatEvaluationResult(result: EvaluationResult, json = false): return [ `Evaluation ${result.passed ? "PASSED" : "FAILED"}`, `turns ${result.turns}`, - `models ${result.modelRequests}`, + `models ${result.modelRequests} request(s), ${result.replay.counts.modelResponses} response(s)`, `tools ${result.toolCalls} (${result.toolFailures} failure(s))`, `duration ${result.durationMs}ms`, `tokens ${result.inputTokens} input · ${result.outputTokens} output · ${result.cacheReadTokens} cache read`, diff --git a/packages/core/src/fixture.ts b/packages/core/src/fixture.ts new file mode 100644 index 0000000..3aec5ec --- /dev/null +++ b/packages/core/src/fixture.ts @@ -0,0 +1,377 @@ +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { + createAssistantMessageEventStream, + type Api, + type AssistantMessage, + type AssistantMessageEventStream, + type Context, + type Model, + type SimpleStreamOptions, + type StopReason, + type ThinkingContent, + type TextContent, + type ToolCall, + type Usage, +} from "@earendil-works/pi-ai"; +import type { AgentTraceStatus } from "./observability.js"; + +export const AGENT_FIXTURE_SCHEMA_VERSION = 1 as const; +export const AGENT_FIXTURE_KIND = "dscode-agent-fixture" as const; + +export type FixtureContentBlock = TextContent | ThinkingContent | ToolCall; +export type FixtureStopReason = Exclude; + +export interface FixtureUsage { + input: number; + output: number; + cacheRead: number; + cacheWrite: number; + totalTokens: number; + cost: Usage["cost"]; +} + +export interface AgentFixtureResponse { + content: FixtureContentBlock[]; + stopReason: FixtureStopReason; + usage?: FixtureUsage; + responseId?: string; + errorMessage?: string; + delayMs?: number; +} + +export interface AgentFixtureEvaluation { + maxToolCalls?: number; + maxDurationMs?: number; + maxTotalTokens?: number; + maxCost?: number; + requireSuccessfulRun?: boolean; + failOnErrors?: boolean; +} + +export interface AgentFixtureAssertions { + finalStatus?: AgentTraceStatus; + toolNames?: string[]; + modelResponses?: number; +} + +export interface AgentFixture { + schemaVersion: typeof AGENT_FIXTURE_SCHEMA_VERSION; + kind: typeof AGENT_FIXTURE_KIND; + provider: string; + model: string; + responses: AgentFixtureResponse[]; + evaluation?: AgentFixtureEvaluation; + assertions?: AgentFixtureAssertions; +} + +export class AgentFixtureRecorder { + private readonly responses: AgentFixtureResponse[] = []; + private finished = false; + + constructor( + private readonly filePath: string, + private readonly provider: string, + private readonly model: string, + ) {} + + record(message: AssistantMessage): void { + if (this.finished || message.role !== "assistant") return; + this.responses.push({ + content: message.content.map(cloneFixtureContent), + stopReason: message.stopReason === "pending" ? "error" : message.stopReason, + usage: cloneUsage(message.usage), + ...(message.responseId ? { responseId: message.responseId } : {}), + ...(message.errorMessage ? { errorMessage: message.errorMessage } : {}), + }); + } + + async finish(): Promise { + if (this.finished) return; + this.finished = true; + await writeAgentFixture(this.filePath, { + schemaVersion: AGENT_FIXTURE_SCHEMA_VERSION, + kind: AGENT_FIXTURE_KIND, + provider: this.provider, + model: this.model, + responses: this.responses, + }); + } +} + +export class AgentFixtureReplay { + private nextResponse = 0; + private calls = 0; + + constructor(private readonly fixture: AgentFixture) {} + + stream( + model: Model, + _context: Context, + options?: SimpleStreamOptions, + ): AssistantMessageEventStream { + const stream = createAssistantMessageEventStream(); + const response = this.fixture.responses[this.nextResponse++]; + this.calls += 1; + void this.resolve(stream, model, response, options?.signal); + return stream; + } + + consumedResponses(): number { + return this.nextResponse; + } + + callCount(): number { + return this.calls; + } + + pendingResponses(): number { + return Math.max(0, this.fixture.responses.length - this.nextResponse); + } + + private async resolve( + stream: AssistantMessageEventStream, + model: Model, + response: AgentFixtureResponse | undefined, + signal: AbortSignal | undefined, + ): Promise { + try { + if (response?.delayMs) await delay(response.delayMs, signal); + const message = response + ? messageFromFixture(response, model) + : errorMessage(model, "Fixture has no response for this model request"); + if (signal?.aborted) { + const aborted = { ...message, stopReason: "aborted" as const, errorMessage: "Request was aborted" }; + stream.push({ type: "error", reason: "aborted", error: aborted }); + stream.end(aborted); + return; + } + const partial: AssistantMessage = { ...message, content: [], stopReason: "pending" }; + stream.push({ type: "start", partial }); + if (message.stopReason === "pending") { + const error = { ...message, stopReason: "error" as const }; + stream.push({ type: "error", reason: "error", error }); + stream.end(error); + return; + } + if (message.stopReason === "error" || message.stopReason === "aborted") { + stream.push({ type: "error", reason: message.stopReason, error: message }); + stream.end(message); + return; + } + stream.push({ type: "done", reason: message.stopReason, message }); + stream.end(message); + } catch (error) { + const message = errorMessage(model, error instanceof Error ? error.message : String(error)); + stream.push({ type: "error", reason: "error", error: message }); + stream.end(message); + } + } +} + +export function readAgentFixtureFileSync(filePath: string): AgentFixture { + return parseAgentFixture(fs.readFileSync(filePath, "utf8"), filePath); +} + +export async function readAgentFixtureFile(filePath: string): Promise { + return parseAgentFixture(await fsp.readFile(filePath, "utf8"), filePath); +} + +export async function writeAgentFixture(filePath: string, fixture: AgentFixture): Promise { + await fsp.mkdir(path.dirname(path.resolve(filePath)), { recursive: true, mode: 0o700 }); + await fsp.writeFile(filePath, `${JSON.stringify(fixture, null, 2)}\n`, { + encoding: "utf8", + mode: 0o600, + }); + await fsp.chmod(filePath, 0o600).catch(() => undefined); +} + +function parseAgentFixture(raw: string, source: string): AgentFixture { + let value: unknown; + try { + value = JSON.parse(raw); + } catch { + throw new Error(`Invalid agent fixture JSON: ${source}`); + } + if (!isRecord(value)) throw new Error(`Invalid agent fixture envelope: ${source}`); + if ( + value.schemaVersion !== AGENT_FIXTURE_SCHEMA_VERSION || + value.kind !== AGENT_FIXTURE_KIND || + typeof value.provider !== "string" || + typeof value.model !== "string" || + !Array.isArray(value.responses) || + value.responses.length === 0 + ) { + throw new Error(`Invalid agent fixture schema: ${source}`); + } + const responses = value.responses.map((response, index) => parseResponse(response, `${source} response ${index + 1}`)); + return { + schemaVersion: AGENT_FIXTURE_SCHEMA_VERSION, + kind: AGENT_FIXTURE_KIND, + provider: value.provider, + model: value.model, + responses, + ...(isRecord(value.evaluation) ? { evaluation: parseEvaluation(value.evaluation) } : {}), + ...(isRecord(value.assertions) ? { assertions: parseAssertions(value.assertions) } : {}), + }; +} + +function parseResponse(value: unknown, source: string): AgentFixtureResponse { + if (!isRecord(value) || !Array.isArray(value.content) || typeof value.stopReason !== "string") { + throw new Error(`Invalid agent fixture response: ${source}`); + } + if (!["stop", "length", "toolUse", "error", "aborted"].includes(value.stopReason)) { + throw new Error(`Invalid fixture stopReason: ${source}`); + } + const content = value.content.map((block, index) => parseContent(block, `${source} content ${index + 1}`)); + const response: AgentFixtureResponse = { + content, + stopReason: value.stopReason as FixtureStopReason, + ...(typeof value.responseId === "string" ? { responseId: value.responseId } : {}), + ...(typeof value.errorMessage === "string" ? { errorMessage: value.errorMessage } : {}), + ...(typeof value.delayMs === "number" && value.delayMs >= 0 ? { delayMs: value.delayMs } : {}), + }; + if (isRecord(value.usage)) response.usage = parseUsage(value.usage, source); + return response; +} + +function parseContent(value: unknown, source: string): FixtureContentBlock { + if (!isRecord(value) || typeof value.type !== "string") throw new Error(`Invalid fixture content: ${source}`); + if (value.type === "text" && typeof value.text === "string") return { type: "text", text: value.text }; + if (value.type === "thinking" && typeof value.thinking === "string") { + return { type: "thinking", thinking: value.thinking }; + } + if ( + value.type === "toolCall" && + typeof value.id === "string" && + typeof value.name === "string" && + isRecord(value.arguments) + ) { + return { type: "toolCall", id: value.id, name: value.name, arguments: value.arguments }; + } + throw new Error(`Invalid fixture content: ${source}`); +} + +function parseUsage(value: Record, source: string): FixtureUsage { + const cost = isRecord(value.cost) ? value.cost : undefined; + const numbers = ["input", "output", "cacheRead", "cacheWrite", "totalTokens"]; + if (!numbers.every((key) => typeof value[key] === "number") || !cost) { + throw new Error(`Invalid fixture usage: ${source}`); + } + const costKeys = ["input", "output", "cacheRead", "cacheWrite", "total"]; + if (!costKeys.every((key) => typeof cost[key] === "number")) { + throw new Error(`Invalid fixture cost: ${source}`); + } + return { + input: value.input as number, + output: value.output as number, + cacheRead: value.cacheRead as number, + cacheWrite: value.cacheWrite as number, + totalTokens: value.totalTokens as number, + cost: { + input: cost.input as number, + output: cost.output as number, + cacheRead: cost.cacheRead as number, + cacheWrite: cost.cacheWrite as number, + total: cost.total as number, + }, + }; +} + +function parseEvaluation(value: Record): AgentFixtureEvaluation { + const result: AgentFixtureEvaluation = {}; + for (const key of ["maxToolCalls", "maxDurationMs", "maxTotalTokens", "maxCost"] as const) { + if (typeof value[key] === "number" && value[key] >= 0) result[key] = value[key]; + } + if (typeof value.requireSuccessfulRun === "boolean") result.requireSuccessfulRun = value.requireSuccessfulRun; + if (typeof value.failOnErrors === "boolean") result.failOnErrors = value.failOnErrors; + return result; +} + +function parseAssertions(value: Record): AgentFixtureAssertions { + const result: AgentFixtureAssertions = {}; + if (value.finalStatus === "completed" || value.finalStatus === "failed" || value.finalStatus === "cancelled") { + result.finalStatus = value.finalStatus; + } + if (Array.isArray(value.toolNames) && value.toolNames.every((name) => typeof name === "string")) { + result.toolNames = value.toolNames as string[]; + } + if (typeof value.modelResponses === "number" && value.modelResponses >= 0) result.modelResponses = value.modelResponses; + return result; +} + +function messageFromFixture(response: AgentFixtureResponse, model: Model): AssistantMessage { + return { + role: "assistant", + content: response.content.map(cloneFixtureContent), + api: model.api, + provider: model.provider, + model: model.id, + usage: response.usage ?? emptyUsage(), + stopReason: response.stopReason, + ...(response.responseId ? { responseId: response.responseId } : {}), + ...(response.errorMessage ? { errorMessage: response.errorMessage } : {}), + timestamp: Date.now(), + }; +} + +function errorMessage(model: Model, message: string): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: emptyUsage(), + stopReason: "error", + errorMessage: message, + timestamp: Date.now(), + }; +} + +function emptyUsage(): Usage { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function cloneUsage(usage: Usage): FixtureUsage { + return { + input: usage.input, + output: usage.output, + cacheRead: usage.cacheRead, + cacheWrite: usage.cacheWrite, + totalTokens: usage.totalTokens, + cost: { ...usage.cost }, + }; +} + +function cloneFixtureContent(block: FixtureContentBlock): FixtureContentBlock { + if (block.type === "text") return { type: "text", text: block.text }; + if (block.type === "thinking") return { type: "thinking", thinking: block.thinking }; + return { type: "toolCall", id: block.id, name: block.name, arguments: structuredClone(block.arguments) }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function delay(milliseconds: number, signal: AbortSignal | undefined): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(new Error("Request was aborted")); + return; + } + const timer = setTimeout(resolve, milliseconds); + signal?.addEventListener("abort", () => { + clearTimeout(timer); + reject(new Error("Request was aborted")); + }, { once: true }); + }); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d4d94ba..daabc8e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -134,3 +134,19 @@ export { type EvaluationOptions, type EvaluationResult, } from "./evaluation.js"; +export { + AGENT_FIXTURE_KIND, + AGENT_FIXTURE_SCHEMA_VERSION, + AgentFixtureRecorder, + AgentFixtureReplay, + readAgentFixtureFile, + readAgentFixtureFileSync, + writeAgentFixture, + type AgentFixture, + type AgentFixtureAssertions, + type AgentFixtureEvaluation, + type AgentFixtureResponse, + type FixtureContentBlock, + type FixtureStopReason, + type FixtureUsage, +} from "./fixture.js"; diff --git a/packages/core/src/replay.ts b/packages/core/src/replay.ts index 5520720..e31f80e 100644 --- a/packages/core/src/replay.ts +++ b/packages/core/src/replay.ts @@ -46,6 +46,7 @@ export interface TraceReplayReport { unclosed: string[]; }; unmatchedToolCalls: string[]; + toolNames: string[]; missingRunEnds: string[]; finalStatus?: AgentTraceStatus; violations: string[]; @@ -136,6 +137,7 @@ export function summarizeTrace( const closed = new Set(); const toolCalls = new Set(); const toolResults = new Set(); + const toolNames: string[] = []; const runStarts = new Set(); const runEnds = new Set(); let firstTimestamp: number | undefined; @@ -171,6 +173,7 @@ export function summarizeTrace( break; case "tool_call": counts.toolCalls += 1; + if (event.toolName) toolNames.push(event.toolName); if (event.toolCallId) toolCalls.add(event.toolCallId); break; case "tool_result": @@ -223,6 +226,7 @@ export function summarizeTrace( usage, spans: { total: starts.size + [...closed].filter((spanId) => !starts.has(spanId)).length, closed: closed.size, unclosed }, unmatchedToolCalls, + toolNames, missingRunEnds, ...(finalStatus ? { finalStatus } : {}), violations, diff --git a/packages/core/src/runtime-options.ts b/packages/core/src/runtime-options.ts index f46dc8f..eab8b2d 100644 --- a/packages/core/src/runtime-options.ts +++ b/packages/core/src/runtime-options.ts @@ -41,6 +41,7 @@ export interface DSCodeRuntimeOptions { activeTools: string[]; toolsExplicit: boolean; personalizationFile?: string; + fixtureCapturePath?: string; } export interface ParsedRuntimeArgs { @@ -76,6 +77,7 @@ export function parseRuntimeArgs(argv: string[]): ParsedRuntimeArgs { let activeTools: string[] | undefined; let toolsExplicit = false; const personalizationFile = process.env.DSCODE_PERSONALIZATION_FILE?.trim(); + let fixtureCapturePath = process.env.DSCODE_FIXTURE_CAPTURE?.trim(); let help = false; let version = false; let yolo = false; @@ -129,6 +131,8 @@ export function parseRuntimeArgs(argv: string[]): ParsedRuntimeArgs { } else if (flag === "--no-tools") { activeTools = []; toolsExplicit = true; + } else if (flag === "--record-fixture") { + fixtureCapturePath = path.resolve(takeValue()); } else if (flag === "--no-resume") { // Pi starts a new persisted session unless --continue/--resume is passed. } else if (flag === "--help" || flag === "-h") { @@ -180,6 +184,7 @@ export function parseRuntimeArgs(argv: string[]): ParsedRuntimeArgs { activeTools, toolsExplicit, ...(personalizationFile ? { personalizationFile: path.resolve(personalizationFile) } : {}), + ...(fixtureCapturePath ? { fixtureCapturePath: path.resolve(fixtureCapturePath) } : {}), }, piArgs: forwarded, help, @@ -210,6 +215,7 @@ export function printDSCodeHelp(): void { Usage: dscode [options] [prompt] dscode replay [--json] + dscode replay --execute --prompt "task" [--json] dscode evaluate [--json] [budget options] dscode -p "task" Non-interactive text mode dscode --mode json -p "task" JSONL/CI mode @@ -229,6 +235,7 @@ DSCode options: --sandbox read-only|workspace-write|danger-full-access --network Pre-authorize command network access for this run --web Enable DeepSeek server-side web search + --record-fixture Explicitly capture assistant responses for offline replay -y, --yes YOLO: trust project, skip approvals, allow host + network Session and editor features: @@ -242,6 +249,7 @@ DSCode commands: Runtime traces and evaluation: dscode replay --compare Compare two trace projections + dscode replay --execute --prompt Execute a deterministic fixture replay --max-tool-calls --max-duration-ms --max-total-tokens --max-cost --allow-errors Do not fail evaluation on runtime errors diff --git a/test/cli-integration.test.ts b/test/cli-integration.test.ts index eeb0af0..90b02b3 100644 --- a/test/cli-integration.test.ts +++ b/test/cli-integration.test.ts @@ -4,6 +4,7 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { readAgentFixtureFile, writeAgentFixture } from "../packages/core/src/fixture.js"; describe("DSCode Pi integration", () => { let server: http.Server | undefined; @@ -21,6 +22,7 @@ describe("DSCode Pi integration", () => { it("runs a JSONL turn through the DeepSeek Responses adapter", async () => { traceDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-cli-trace-")); + const fixturePath = path.join(traceDir, "recorded.json"); let payload: Record | undefined; server = http.createServer(async (request, response) => { const body: Buffer[] = []; @@ -97,6 +99,8 @@ describe("DSCode Pi integration", () => { "--print", "--no-session", "--no-approve", + "--record-fixture", + fixturePath, "reply once", ], { @@ -132,6 +136,12 @@ describe("DSCode Pi integration", () => { expect(events.map((event) => event.type)).toEqual( expect.arrayContaining(["run_start", "model_request", "provider_response", "model_response", "run_end"]), ); + expect(await readAgentFixtureFile(fixturePath)).toMatchObject({ + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [{ content: [{ type: "text", text: "mock response" }], stopReason: "stop" }], + }); + expect((await fs.stat(fixturePath)).mode & 0o777).toBe(0o600); }, 15_000); it("returns a non-zero CI exit code on provider failure", async () => { @@ -181,6 +191,168 @@ describe("DSCode Pi integration", () => { ]), ); }, 15_000); + + it("executes a deterministic fixture through the real agent runtime", async () => { + const fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-execute-fixture-")); + const fixturePath = path.join(fixtureDir, "case.json"); + await writeAgentFixture(fixturePath, { + schemaVersion: 1, + kind: "dscode-agent-fixture", + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { content: [{ type: "text", text: "fixture response" }], stopReason: "stop" }, + ], + assertions: { finalStatus: "completed", modelResponses: 1 }, + }); + const execution = await spawnCapture( + process.execPath, + [ + path.resolve("node_modules/tsx/dist/cli.mjs"), + "src/cli.ts", + "replay", + "--execute", + fixturePath, + "--prompt", + "reply once", + "--json", + ], + { + ...process.env, + DSCODE_TRACE: "0", + PI_SKIP_VERSION_CHECK: "1", + PI_TELEMETRY: "0", + }, + ); + await fs.rm(fixtureDir, { recursive: true, force: true }); + + expect(execution.exitCode, execution.stderr).toBe(0); + const result = JSON.parse(execution.stdout) as { + passed: boolean; + modelRequests: number; + modelResponses: number; + violations: string[]; + }; + expect(result).toMatchObject({ + passed: true, + modelRequests: 1, + modelResponses: 1, + violations: [], + }); + }, 15_000); + + it("replays fixture tool calls through the real tool loop", async () => { + const fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-tool-fixture-")); + const fixturePath = path.join(fixtureDir, "case.json"); + await writeAgentFixture(fixturePath, { + schemaVersion: 1, + kind: "dscode-agent-fixture", + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { + content: [ + { + type: "toolCall", + id: "call-fixture-1", + name: "exec_command", + arguments: { cmd: "printf fixture-tool" }, + }, + ], + stopReason: "toolUse", + }, + { content: [{ type: "text", text: "tool completed" }], stopReason: "stop" }, + ], + assertions: { + finalStatus: "completed", + toolNames: ["exec_command"], + modelResponses: 2, + }, + }); + const execution = await spawnCapture( + process.execPath, + [ + path.resolve("node_modules/tsx/dist/cli.mjs"), + "src/cli.ts", + "replay", + "--execute", + fixturePath, + "--prompt", + "run the fixture tool", + "--json", + ], + { + ...process.env, + DSCODE_TRACE: "0", + PI_SKIP_VERSION_CHECK: "1", + PI_TELEMETRY: "0", + }, + ); + await fs.rm(fixtureDir, { recursive: true, force: true }); + + expect(execution.exitCode, execution.stderr).toBe(0); + const result = JSON.parse(execution.stdout) as { + passed: boolean; + modelResponses: number; + violations: string[]; + }; + expect(result).toMatchObject({ + passed: true, + modelResponses: 2, + violations: [], + }); + }, 15_000); + + it("allows an explicitly expected fixture error", async () => { + const fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-error-fixture-")); + const fixturePath = path.join(fixtureDir, "case.json"); + await writeAgentFixture(fixturePath, { + schemaVersion: 1, + kind: "dscode-agent-fixture", + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { content: [], stopReason: "error", errorMessage: "fixture rate limit" }, + { content: [], stopReason: "error", errorMessage: "fixture rate limit" }, + { content: [], stopReason: "error", errorMessage: "fixture rate limit" }, + { content: [], stopReason: "error", errorMessage: "fixture rate limit" }, + ], + evaluation: { requireSuccessfulRun: false, failOnErrors: false }, + assertions: { finalStatus: "failed", modelResponses: 4 }, + }); + const execution = await spawnCapture( + process.execPath, + [ + path.resolve("node_modules/tsx/dist/cli.mjs"), + "src/cli.ts", + "replay", + "--execute", + fixturePath, + "--prompt", + "exercise the error path", + "--json", + ], + { + ...process.env, + DSCODE_TRACE: "0", + PI_SKIP_VERSION_CHECK: "1", + PI_TELEMETRY: "0", + }, + ); + await fs.rm(fixtureDir, { recursive: true, force: true }); + + expect(execution.exitCode, `${execution.stderr}\n${execution.stdout}`).toBe(0); + const result = JSON.parse(execution.stdout) as { + passed: boolean; + modelResponses: number; + violations: string[]; + }; + expect(result).toMatchObject({ + passed: true, + modelResponses: 4, + violations: [], + }); + }, 30_000); }); function sendEvent(response: http.ServerResponse, event: Record): void { diff --git a/test/fixture.test.ts b/test/fixture.test.ts new file mode 100644 index 0000000..d3b672f --- /dev/null +++ b/test/fixture.test.ts @@ -0,0 +1,113 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + AGENT_FIXTURE_KIND, + AGENT_FIXTURE_SCHEMA_VERSION, + AgentFixtureReplay, + readAgentFixtureFile, + writeAgentFixture, + type AgentFixture, +} from "../packages/core/src/fixture.js"; +import type { Model } from "@earendil-works/pi-ai"; + +describe("agent fixtures", () => { + it("round-trips a fixture with tool calls and streams deterministic responses", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-fixture-")); + const fixturePath = path.join(directory, "case.json"); + const fixture: AgentFixture = { + schemaVersion: AGENT_FIXTURE_SCHEMA_VERSION, + kind: AGENT_FIXTURE_KIND, + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { + content: [ + { + type: "toolCall", + id: "call-1", + name: "exec_command", + arguments: { cmd: "printf fixture" }, + }, + ], + stopReason: "toolUse", + }, + { content: [{ type: "text", text: "fixture complete" }], stopReason: "stop" }, + ], + assertions: { finalStatus: "completed", toolNames: ["exec_command"], modelResponses: 2 }, + }; + await writeAgentFixture(fixturePath, fixture); + const loaded = await readAgentFixtureFile(fixturePath); + expect(loaded).toEqual(fixture); + + const replay = new AgentFixtureReplay(loaded); + const model = { + api: "dscode-fixture", + provider: "deepseek", + id: "deepseek-v4-flash", + } as Model<"dscode-fixture">; + const first = replay.stream(model, { messages: [] }); + const firstEvents = []; + for await (const event of first) firstEvents.push(event); + expect(firstEvents.at(-1)).toMatchObject({ type: "done", reason: "toolUse" }); + expect(await first.result()).toMatchObject({ + stopReason: "toolUse", + content: [{ type: "toolCall", name: "exec_command" }], + }); + expect(replay.consumedResponses()).toBe(1); + await fs.rm(directory, { recursive: true, force: true }); + }); + + it("turns an error fixture response into an assistant error event", async () => { + const fixture: AgentFixture = { + schemaVersion: AGENT_FIXTURE_SCHEMA_VERSION, + kind: AGENT_FIXTURE_KIND, + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { + content: [], + stopReason: "error", + errorMessage: "fixture rate limit", + }, + ], + }; + const replay = new AgentFixtureReplay(fixture); + const model = { + api: "dscode-fixture", + provider: "deepseek", + id: "deepseek-v4-flash", + } as Model<"dscode-fixture">; + const stream = replay.stream(model, { messages: [] }); + const events = []; + for await (const event of stream) events.push(event); + expect(events.at(-1)).toMatchObject({ type: "error", reason: "error" }); + expect(await stream.result()).toMatchObject({ stopReason: "error", errorMessage: "fixture rate limit" }); + }); + + it("injects response delays and aborted responses", async () => { + const fixture: AgentFixture = { + schemaVersion: AGENT_FIXTURE_SCHEMA_VERSION, + kind: AGENT_FIXTURE_KIND, + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { content: [], stopReason: "aborted", delayMs: 1, errorMessage: "fixture cancelled" }, + ], + }; + const replay = new AgentFixtureReplay(fixture); + const model = { + api: "dscode-fixture", + provider: "deepseek", + id: "deepseek-v4-flash", + } as Model<"dscode-fixture">; + const started = Date.now(); + const stream = replay.stream(model, { messages: [] }); + expect(await stream.result()).toMatchObject({ + stopReason: "aborted", + errorMessage: "fixture cancelled", + }); + expect(Date.now() - started).toBeGreaterThanOrEqual(1); + }); +}); From a2404edfdbd87dc21139e78ad86d1924adf9be86 Mon Sep 17 00:00:00 2001 From: Chal1ce <752193262@qq.com> Date: Fri, 14 Aug 2026 22:55:44 +0800 Subject: [PATCH 03/13] fix(runtime): address fixture replay review findings --- packages/core/src/cli-runtime.ts | 3 ++ packages/core/src/fixture.ts | 50 ++++++++++++++++++++++++++------ test/cli-integration.test.ts | 22 ++++++++++++++ test/fixture.test.ts | 25 ++++++++++++++++ 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/packages/core/src/cli-runtime.ts b/packages/core/src/cli-runtime.ts index c1a1a83..07e142c 100644 --- a/packages/core/src/cli-runtime.ts +++ b/packages/core/src/cli-runtime.ts @@ -112,6 +112,9 @@ function parseTraceCommand(argv: string[]): TraceCommand | undefined { } const [flag, inlineValue] = splitTraceFlag(argument); if (flag === "--execute") { + if (command !== "replay") { + throw new Error("--execute is only supported by replay"); + } const value = inlineValue ?? argv[++index]; if (!value) throw new Error("--execute requires a fixture file"); execute = true; diff --git a/packages/core/src/fixture.ts b/packages/core/src/fixture.ts index 3aec5ec..a44a01e 100644 --- a/packages/core/src/fixture.ts +++ b/packages/core/src/fixture.ts @@ -142,9 +142,7 @@ export class AgentFixtureReplay { ? messageFromFixture(response, model) : errorMessage(model, "Fixture has no response for this model request"); if (signal?.aborted) { - const aborted = { ...message, stopReason: "aborted" as const, errorMessage: "Request was aborted" }; - stream.push({ type: "error", reason: "aborted", error: aborted }); - stream.end(aborted); + endAborted(stream, model); return; } const partial: AssistantMessage = { ...message, content: [], stopReason: "pending" }; @@ -163,6 +161,10 @@ export class AgentFixtureReplay { stream.push({ type: "done", reason: message.stopReason, message }); stream.end(message); } catch (error) { + if (signal?.aborted || isFixtureAbortError(error)) { + endAborted(stream, model); + return; + } const message = errorMessage(model, error instanceof Error ? error.message : String(error)); stream.push({ type: "error", reason: "error", error: message }); stream.end(message); @@ -170,6 +172,18 @@ export class AgentFixtureReplay { } } +function endAborted( + stream: AssistantMessageEventStream, + model: Model, +): void { + const aborted = { + ...errorMessage(model, "Request was aborted"), + stopReason: "aborted" as const, + }; + stream.push({ type: "error", reason: "aborted", error: aborted }); + stream.end(aborted); +} + export function readAgentFixtureFileSync(filePath: string): AgentFixture { return parseAgentFixture(fs.readFileSync(filePath, "utf8"), filePath); } @@ -365,13 +379,31 @@ function isRecord(value: unknown): value is Record { function delay(milliseconds: number, signal: AbortSignal | undefined): Promise { return new Promise((resolve, reject) => { if (signal?.aborted) { - reject(new Error("Request was aborted")); + reject(new FixtureAbortError()); return; } - const timer = setTimeout(resolve, milliseconds); - signal?.addEventListener("abort", () => { - clearTimeout(timer); - reject(new Error("Request was aborted")); - }, { once: true }); + let timer: ReturnType | undefined; + const onAbort = () => { + if (timer !== undefined) clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(new FixtureAbortError()); + }; + timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, milliseconds); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); }); } + +class FixtureAbortError extends Error { + constructor() { + super("Request was aborted"); + this.name = "AbortError"; + } +} + +function isFixtureAbortError(error: unknown): boolean { + return error instanceof FixtureAbortError; +} diff --git a/test/cli-integration.test.ts b/test/cli-integration.test.ts index 90b02b3..f67a650 100644 --- a/test/cli-integration.test.ts +++ b/test/cli-integration.test.ts @@ -192,6 +192,28 @@ describe("DSCode Pi integration", () => { ); }, 15_000); + it("rejects fixture execution on evaluate", async () => { + const execution = await spawnCapture( + process.execPath, + [ + path.resolve("node_modules/tsx/dist/cli.mjs"), + "src/cli.ts", + "evaluate", + "--execute", + "ignored-fixture.json", + "--json", + ], + { + ...process.env, + PI_SKIP_VERSION_CHECK: "1", + PI_TELEMETRY: "0", + }, + ); + + expect(execution.exitCode).not.toBe(0); + expect(execution.stderr).toContain("--execute is only supported by replay"); + }, 15_000); + it("executes a deterministic fixture through the real agent runtime", async () => { const fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), "dscode-execute-fixture-")); const fixturePath = path.join(fixtureDir, "case.json"); diff --git a/test/fixture.test.ts b/test/fixture.test.ts index d3b672f..9d0bc0e 100644 --- a/test/fixture.test.ts +++ b/test/fixture.test.ts @@ -110,4 +110,29 @@ describe("agent fixtures", () => { }); expect(Date.now() - started).toBeGreaterThanOrEqual(1); }); + + it("preserves an abort raised while a response is delayed", async () => { + const fixture: AgentFixture = { + schemaVersion: AGENT_FIXTURE_SCHEMA_VERSION, + kind: AGENT_FIXTURE_KIND, + provider: "deepseek", + model: "deepseek-v4-flash", + responses: [ + { content: [{ type: "text", text: "late" }], stopReason: "stop", delayMs: 50 }, + ], + }; + const replay = new AgentFixtureReplay(fixture); + const model = { + api: "dscode-fixture", + provider: "deepseek", + id: "deepseek-v4-flash", + } as Model<"dscode-fixture">; + const controller = new AbortController(); + const stream = replay.stream(model, { messages: [] }, { signal: controller.signal }); + queueMicrotask(() => controller.abort()); + const events = []; + for await (const event of stream) events.push(event); + expect(events.at(-1)).toMatchObject({ type: "error", reason: "aborted" }); + expect(await stream.result()).toMatchObject({ stopReason: "aborted" }); + }); }); From de05ee271bfb849a714be9ed8b9bba846ac7c27b Mon Sep 17 00:00:00 2001 From: Chal1ce <752193262@qq.com> Date: Fri, 14 Aug 2026 23:23:53 +0800 Subject: [PATCH 04/13] fix(runtime): harden MCP registration and runtime options --- packages/core/src/dscode-extension.ts | 7 ++++++- packages/core/src/mcp.ts | 28 +++++++++++++++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/core/src/dscode-extension.ts b/packages/core/src/dscode-extension.ts index 21fc718..5767929 100644 --- a/packages/core/src/dscode-extension.ts +++ b/packages/core/src/dscode-extension.ts @@ -134,7 +134,12 @@ const applyPatchParameters = Type.Object({ }), }); -export function createDSCodeExtension(options: DSCodeRuntimeOptions): InlineExtension { +export function createDSCodeExtension(inputOptions: DSCodeRuntimeOptions): InlineExtension { + // Keep the extension's provider and tool configuration stable for its lifetime. + const options = Object.freeze({ + ...inputOptions, + activeTools: [...inputOptions.activeTools], + }); return { name: "dscode", factory(pi) { diff --git a/packages/core/src/mcp.ts b/packages/core/src/mcp.ts index f2b19a0..dd2ac0e 100644 --- a/packages/core/src/mcp.ts +++ b/packages/core/src/mcp.ts @@ -46,7 +46,7 @@ export class MCPManager { this.errors.push("Project is not trusted; project MCP servers were not started."); } const configs = await loadMcpConfigs(ctx.cwd, trusted); - for (const [serverName, config] of Object.entries(configs)) { + for (const [serverName, config] of Object.entries(configs).sort(([left], [right]) => compareText(left, right))) { if (config.disabled) continue; try { await this.connectServer(pi, ctx, serverName, config); @@ -65,7 +65,7 @@ export class MCPManager { } toolNames(): string[] { - return this.servers.flatMap((server) => server.tools); + return this.servers.flatMap((server) => server.tools).sort(compareText); } async close(): Promise { @@ -105,10 +105,22 @@ export class MCPManager { } const listed = await client.listTools(); - const registered: string[] = []; - for (const tool of listed.tools) { - const localName = `mcp__${sanitizeName(serverName)}__${sanitizeName(tool.name)}`; - registered.push(localName); + const existingNames = new Set(this.servers.flatMap((server) => server.tools)); + const prepared = [...listed.tools] + .sort((left, right) => compareText(left.name, right.name)) + .map((tool) => ({ + tool, + localName: `mcp__${sanitizeName(serverName)}__${sanitizeName(tool.name)}`, + })); + const registered = prepared.map(({ localName }) => localName); + const seenNames = new Set(existingNames); + for (const { localName } of prepared) { + if (seenNames.has(localName)) { + throw new Error(`MCP tool name collision after sanitization: ${localName}`); + } + seenNames.add(localName); + } + for (const { tool, localName } of prepared) { pi.registerTool({ name: localName, label: `${serverName}: ${tool.title ?? tool.name}`, @@ -190,6 +202,10 @@ function sanitizeName(value: string): string { return value.replace(/[^a-zA-Z0-9_-]/g, "_"); } +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + interface FormattedMcpResult { content: Array; text: string; From 18e07c0fd2744b2681ec745e1e62d295eece4ad3 Mon Sep 17 00:00:00 2001 From: Chal1ce <752193262@qq.com> Date: Sun, 16 Aug 2026 12:03:54 +0800 Subject: [PATCH 05/13] fix(desktop): reduce long conversation render work --- apps/desktop/src/renderer/App.tsx | 73 +++++++++++++++---- .../src/renderer/lib/conversation.test.ts | 16 +++- apps/desktop/src/renderer/lib/conversation.ts | 29 ++++++++ apps/desktop/src/renderer/styles.css | 6 +- 4 files changed, 106 insertions(+), 18 deletions(-) diff --git a/apps/desktop/src/renderer/App.tsx b/apps/desktop/src/renderer/App.tsx index 7f6eeb6..3dfdac5 100644 --- a/apps/desktop/src/renderer/App.tsx +++ b/apps/desktop/src/renderer/App.tsx @@ -1,4 +1,5 @@ import { + memo, useCallback, useEffect, useMemo, @@ -56,6 +57,7 @@ import { X, } from "lucide-react"; import type { + AgentEvent, AgentSnapshot, AgentSessionStats, AuthUiEvent, @@ -76,8 +78,10 @@ import type { import { AUTH_PROMPT_CANCEL_VALUE } from "../shared/types"; import { applyAgentEvent, + coalesceStreamingAgentEvents, getAssistantActivity, groupConversation, + isStreamingAgentEvent, normalizeMessages, optimisticUserMessage, splitAssistantTurn, @@ -191,6 +195,32 @@ export default function App() { const sidebarFooterRef = useRef(null); const previewRequestRef = useRef(0); const inspectorResizeCleanupRef = useRef<() => void>(() => undefined); + const pendingAgentEventsRef = useRef([]); + const pendingAgentFrameRef = useRef(undefined); + const previewFileCandidatesRef = useRef([]); + + const flushPendingAgentEvents = useCallback(() => { + const frame = pendingAgentFrameRef.current; + if (frame !== undefined) window.cancelAnimationFrame(frame); + pendingAgentFrameRef.current = undefined; + const pending = pendingAgentEventsRef.current; + if (pending.length === 0) return; + pendingAgentEventsRef.current = []; + const events = coalesceStreamingAgentEvents(pending); + setMessages((current) => events.reduce((next, event) => applyAgentEvent(next, event), current)); + }, []); + + const enqueueAgentEvent = useCallback((event: AgentEvent) => { + if (!isStreamingAgentEvent(event)) { + flushPendingAgentEvents(); + setMessages((current) => applyAgentEvent(current, event)); + return; + } + pendingAgentEventsRef.current.push(event); + if (pendingAgentFrameRef.current === undefined) { + pendingAgentFrameRef.current = window.requestAnimationFrame(flushPendingAgentEvents); + } + }, [flushPendingAgentEvents]); const hydrateSnapshot = useCallback((snapshot: AgentSnapshot) => { setMessages(normalizeMessages(snapshot.messages)); @@ -399,7 +429,7 @@ export default function App() { setUiRequest(request); } } - setMessages((current) => applyAgentEvent(current, event)); + enqueueAgentEvent(event); }); const offError = window.dscode.agent.onError((message) => { if (isAgentSessionClosedError(message)) return; @@ -425,8 +455,12 @@ export default function App() { offError(); offAuth(); offCommand?.(); + const frame = pendingAgentFrameRef.current; + if (frame !== undefined) window.cancelAnimationFrame(frame); + pendingAgentFrameRef.current = undefined; + pendingAgentEventsRef.current = []; }; - }, [refreshSessionStats, refreshSessions, t]); + }, [enqueueAgentEvent, refreshSessionStats, refreshSessions, t]); useEffect(() => { if (!accountMenuOpen) return; @@ -654,7 +688,6 @@ export default function App() { setThemes(await window.dscode.themes.list()); }; - const allTools = useMemo(() => messages.flatMap((message) => message.tools), [messages]); const previewFileCandidates = useMemo(() => { const seen = new Set(); const files: string[] = []; @@ -676,12 +709,11 @@ export default function App() { } if (files.length === 6) break; } - for (const tool of [...allTools].reverse()) { - addFile(toolFilePath(tool)); - if (files.length === 6) break; - } + const previous = previewFileCandidatesRef.current; + if (sameStringArray(previous, files)) return previous; + previewFileCandidatesRef.current = files; return files; - }, [activeSession, allTools, messages, workspace]); + }, [activeSession, messages, workspace]); useEffect(() => { let cancelled = false; @@ -698,7 +730,10 @@ export default function App() { if (!cancelled) setRecentPreviewFiles([]); }); return () => { cancelled = true; }; - }, [previewFileCandidates]); + }, [activeSession, previewFileCandidates, workspace]); + const handlePreviewFile = useCallback((filePath: string) => { + void openFilePreview(filePath); + }, [openFilePreview]); const conversationGroups = useMemo(() => groupConversation(messages), [messages]); const latestAssistantGroup = [...conversationGroups].reverse().find((group) => group.type === "assistant"); const activeAssistantGroupId = running ? latestAssistantGroup?.id : undefined; @@ -971,7 +1006,7 @@ export default function App() { messages={group.messages} active={group.id === activeAssistantGroupId} showReasoningProcess={showReasoningProcess} - onPreviewFile={(filePath) => void openFilePreview(filePath)} + onPreviewFile={handlePreviewFile} /> ))} {running && !uiRequest && !activeAssistantHasWork && ( @@ -1324,7 +1359,7 @@ function EmptyState({ workspace, onSuggest }: { workspace?: string; onSuggest(va ); } -function UserMessage({ message, onPreview }: { message: ChatMessage; onPreview(image: PreviewImage): void }) { +const UserMessage = memo(function UserMessage({ message, onPreview }: { message: ChatMessage; onPreview(image: PreviewImage): void }) { const { t } = useI18n(); return (
0 ? " has-images" : ""}`}> @@ -1347,7 +1382,7 @@ function UserMessage({ message, onPreview }: { message: ChatMessage; onPreview(i {message.queued && {t("status.queued")}}
); -} +}); function ImageLightbox({ image, onClose }: { image: PreviewImage; onClose(): void }) { const { t } = useI18n(); @@ -1373,7 +1408,7 @@ function ImageLightbox({ image, onClose }: { image: PreviewImage; onClose(): voi ); } -function AssistantTurn({ +const AssistantTurn = memo(function AssistantTurn({ messages, active, showReasoningProcess, @@ -1405,7 +1440,13 @@ function AssistantTurn({ ))} ); -} +}, (previous, next) => ( + previous.active === next.active + && previous.showReasoningProcess === next.showReasoningProcess + && previous.onPreviewFile === next.onPreviewFile + && previous.messages.length === next.messages.length + && previous.messages.every((message, index) => message === next.messages[index]) +)); function MarkdownContent({ text, className = "markdown-body", onPreviewFile }: { text: string; className?: string; onPreviewFile?(filePath: string): void }) { return ( @@ -2610,6 +2651,10 @@ function imageDataUrl(image: ChatImage): string { return image.data.startsWith("data:") ? image.data : `data:${image.mimeType};base64,${image.data}`; } +function sameStringArray(left: string[], right: string[]): boolean { + return left.length === right.length && left.every((value, index) => value === right[index]); +} + type Translator = ReturnType["t"]; function permissionLabel(value: PermissionMode, t: Translator): string { diff --git a/apps/desktop/src/renderer/lib/conversation.test.ts b/apps/desktop/src/renderer/lib/conversation.test.ts index 1f108aa..5521c94 100644 --- a/apps/desktop/src/renderer/lib/conversation.test.ts +++ b/apps/desktop/src/renderer/lib/conversation.test.ts @@ -1,7 +1,21 @@ import { describe, expect, it } from "vitest"; -import { applyAgentEvent, getAssistantActivity, groupConversation, normalizeMessages, optimisticUserMessage, splitAssistantTurn } from "./conversation"; +import { applyAgentEvent, coalesceStreamingAgentEvents, getAssistantActivity, groupConversation, normalizeMessages, optimisticUserMessage, splitAssistantTurn } from "./conversation"; describe("conversation events", () => { + it("coalesces high-frequency streaming updates by payload key", () => { + const firstMessageUpdate = { type: "message_update", message: { role: "assistant", content: [{ type: "text", text: "A" }] } }; + const secondMessageUpdate = { type: "message_update", message: { role: "assistant", content: [{ type: "text", text: "AB" }] } }; + const firstToolUpdate = { type: "tool_execution_update", toolCallId: "tool-1", partialResult: "A" }; + const secondToolUpdate = { type: "tool_execution_update", toolCallId: "tool-1", partialResult: "AB" }; + + expect(coalesceStreamingAgentEvents([ + firstMessageUpdate, + firstToolUpdate, + secondMessageUpdate, + secondToolUpdate, + ])).toEqual([secondMessageUpdate, secondToolUpdate]); + }); + it("normalizes assistant text, thinking, and tool calls", () => { const messages = normalizeMessages([ { diff --git a/apps/desktop/src/renderer/lib/conversation.ts b/apps/desktop/src/renderer/lib/conversation.ts index b9d1c43..778d547 100644 --- a/apps/desktop/src/renderer/lib/conversation.ts +++ b/apps/desktop/src/renderer/lib/conversation.ts @@ -129,6 +129,35 @@ export function getAssistantActivity(messages: ChatMessage[]): AssistantActivity : "thinking"; } +export function isStreamingAgentEvent(event: AgentEvent): boolean { + return event.type === "message_update" || event.type === "tool_execution_update"; +} + +/** Keep only the latest payload for high-frequency streaming updates in a frame. */ +export function coalesceStreamingAgentEvents(events: AgentEvent[]): AgentEvent[] { + if (events.length < 2) return events; + const result: AgentEvent[] = []; + for (const event of events) { + const key = streamingEventKey(event); + if (key) { + for (let index = result.length - 1; index >= 0; index -= 1) { + if (streamingEventKey(result[index]!) === key) { + result.splice(index, 1); + break; + } + } + } + result.push(event); + } + return result; +} + +function streamingEventKey(event: AgentEvent): string | undefined { + if (event.type === "message_update") return "message_update"; + if (event.type !== "tool_execution_update" || typeof event.toolCallId !== "string") return undefined; + return `tool_execution_update:${event.toolCallId}`; +} + export function applyAgentEvent(messages: ChatMessage[], event: AgentEvent): ChatMessage[] { if (event.type === "message_start" || event.type === "message_update" || event.type === "message_end") { const raw = isRecord(event.message) ? event.message : undefined; diff --git a/apps/desktop/src/renderer/styles.css b/apps/desktop/src/renderer/styles.css index 70f7431..95a3608 100644 --- a/apps/desktop/src/renderer/styles.css +++ b/apps/desktop/src/renderer/styles.css @@ -154,7 +154,7 @@ kbd { margin-left: auto; color: var(--faint); font-family: -apple-system, sans-s .context-card-footer > div:last-child { align-items: flex-end; } .context-card-footer span { color: var(--faint); font-size: 9px; } .context-card-footer strong { max-width: 160px; overflow: hidden; font-size: 10.5px; font-weight: 580; text-overflow: ellipsis; white-space: nowrap; font-variant-numeric: tabular-nums; } -.message-scroll { min-height: 0; flex: 1; overflow-x: hidden; overflow-y: auto; overscroll-behavior: contain; scroll-behavior: smooth; } +.message-scroll { min-height: 0; flex: 1; overflow-x: hidden; overflow-y: auto; overscroll-behavior: contain; scroll-behavior: auto; } .messages { width: min(100%, 820px); padding: 50px 32px 165px; margin: 0 auto; } .context-card-visible .messages { width: min(calc(100% - 300px), 820px); margin-left: auto; margin-right: 300px; } .context-card-visible .composer-stack { width: min(calc(100% - 300px), 780px); margin-left: auto; margin-right: 300px; } @@ -170,7 +170,7 @@ kbd { margin-left: auto; color: var(--faint); font-family: -apple-system, sans-s .suggestion-grid button { min-height: 72px; padding: 13px; display: flex; flex-direction: column; align-items: flex-start; justify-content: space-between; border: 1px solid var(--line); border-radius: 9px; background: color-mix(in srgb, var(--surface) 72%, transparent); color: var(--muted); text-align: left; } .suggestion-grid button:hover { background: var(--surface); color: var(--ink); border-color: color-mix(in srgb, var(--line) 65%, var(--ink)); } -.user-message { width: fit-content; max-width: 80%; margin: 0 0 29px auto; padding: 10px 13px; border: 1px solid var(--line); border-radius: 13px 13px 4px 13px; background: var(--surface); line-height: 1.55; white-space: pre-wrap; box-shadow: 0 1px 1px rgba(30,30,27,.025); } +.user-message { width: fit-content; max-width: 80%; margin: 0 0 29px auto; padding: 10px 13px; border: 1px solid var(--line); border-radius: 13px 13px 4px 13px; background: var(--surface); line-height: 1.55; white-space: pre-wrap; box-shadow: 0 1px 1px rgba(30,30,27,.025); content-visibility: auto; contain-intrinsic-size: auto 64px; } .user-message small { display: block; color: var(--faint); margin-top: 5px; font-size: 10px; } .user-message.has-images { max-width: min(80%, 440px); padding: 6px; } .user-message.has-images .user-message-text { padding: 5px 7px 3px; } @@ -183,7 +183,7 @@ kbd { margin-left: auto; color: var(--faint); font-family: -apple-system, sans-s .message-images.multiple { grid-template-columns: repeat(2, minmax(0, 1fr)); width: min(360px, 64vw); } .message-images.multiple .message-image-button { aspect-ratio: 4 / 3; } .message-images.multiple img { width: 100%; height: 100%; object-fit: cover; } -.assistant-message { margin-bottom: 34px; line-height: 1.62; font-size: 14px; overflow-wrap: anywhere; } +.assistant-message { margin-bottom: 34px; line-height: 1.62; font-size: 14px; overflow-wrap: anywhere; content-visibility: auto; contain-intrinsic-size: auto 180px; } .assistant-message:last-child { margin-bottom: 8px; } .assistant-response + .assistant-response { margin-top: 13px; } .work-log { margin: 0 0 17px; color: var(--muted); font-size: 12px; } From b03923d8c4a8d67df26756d7305af5a9f56775f3 Mon Sep 17 00:00:00 2001 From: Chal1ce <752193262@qq.com> Date: Sun, 16 Aug 2026 15:49:02 +0800 Subject: [PATCH 06/13] fix(desktop): edit and replace last user message --- apps/desktop/src/renderer/App.tsx | 252 ++++++++++++++++++++++--- apps/desktop/src/renderer/lib/i18n.tsx | 6 + apps/desktop/src/renderer/styles.css | 18 +- packages/core/src/session-commands.ts | 64 ++++++- test/session-commands.test.ts | 56 ++++++ 5 files changed, 372 insertions(+), 24 deletions(-) diff --git a/apps/desktop/src/renderer/App.tsx b/apps/desktop/src/renderer/App.tsx index 3dfdac5..55e6db9 100644 --- a/apps/desktop/src/renderer/App.tsx +++ b/apps/desktop/src/renderer/App.tsx @@ -23,6 +23,7 @@ import { CircleAlert, CircleStop, Code2, + Copy, ExternalLink, Eye, File as FileIcon, @@ -46,6 +47,7 @@ import { Paperclip, PanelLeft, PanelRight, + Pencil, Plus, Search, Settings, @@ -163,6 +165,9 @@ export default function App() { const [sessionStats, setSessionStats] = useState(); const [draft, setDraft] = useState(""); const [attachments, setAttachments] = useState([]); + const [editingMessageId, setEditingMessageId] = useState(); + const [editingText, setEditingText] = useState(""); + const [editingSubmitting, setEditingSubmitting] = useState(false); const [previewImage, setPreviewImage] = useState(); const [running, setRunning] = useState(false); const [loading, setLoading] = useState(true); @@ -250,6 +255,23 @@ export default function App() { } }, []); + const readAgentSnapshot = useCallback(async (): Promise => { + const [state, messageData, models, thinkingLevels, stats] = await Promise.all([ + window.dscode.agent.command>("get_state"), + window.dscode.agent.command<{ messages: unknown[] }>("get_messages"), + window.dscode.agent.command<{ models: AgentSnapshot["models"] }>("get_available_models"), + window.dscode.agent.command<{ levels: string[] }>("get_available_thinking_levels"), + window.dscode.agent.command("get_session_stats").catch(() => undefined), + ]); + return { + state, + messages: messageData.messages, + models: models.models, + thinkingLevels: thinkingLevels.levels, + ...(stats ? { stats } : {}), + }; + }, []); + const openFilePreview = useCallback(async (filePath: string) => { const requestId = ++previewRequestRef.current; setContextCardOpen(false); @@ -305,6 +327,9 @@ export default function App() { const providerStatuses = behavior?.providerStatuses ?? providers; const status = providerStatuses.find((candidate) => candidate.id === nextProvider); setUiRequest(undefined); + setEditingMessageId(undefined); + setEditingText(""); + setEditingSubmitting(false); setActiveSession(sessionPath); activeCwdRef.current = cwd; setWorkspace(projectPath); @@ -398,7 +423,9 @@ export default function App() { permission: "auto", sandbox: "workspace-write", }); - if (!cancelled) hydrateSnapshot(snapshot); + if (!cancelled) { + hydrateSnapshot(snapshot); + } } catch (error) { if (!cancelled && !isAgentSessionClosedError(error)) { setToast({ message: cleanError(error instanceof Error ? error.message : String(error)), type: "error" }); @@ -580,18 +607,61 @@ export default function App() { await startAgent(session.cwd, session.path, undefined, projectPath); }; + const editMessage = useCallback((message: ChatMessage) => { + if (running || editingMessageId) return; + setEditingMessageId(message.id); + setEditingText(message.text); + }, [editingMessageId, running]); + + const cancelEditingMessage = useCallback(() => { + if (editingSubmitting) return; + setEditingMessageId(undefined); + setEditingText(""); + }, [editingSubmitting]); + + const submitEditingMessage = useCallback(async () => { + if (!editingMessageId || editingSubmitting || running) return; + const original = messages.find((message) => message.id === editingMessageId); + if (!original) return; + const text = editingText.trim(); + const messageImages = original.images.map(({ data, mimeType }) => ({ data, mimeType })); + if (!text && messageImages.length === 0) return; + const images = messageImages.map((image) => ({ type: "image", ...image })); + setEditingSubmitting(true); + setRunning(true); + try { + await window.dscode.agent.command("prompt", { message: "/edit-last" }); + const snapshot = await readAgentSnapshot(); + hydrateSnapshot(snapshot); + setEditingMessageId(undefined); + setEditingText(""); + setMessages((current) => [...current, optimisticUserMessage(text || t("composer.attachedImage"), false, messageImages)]); + await window.dscode.agent.command("prompt", { + message: text || t("composer.describeImage"), + ...(images.length ? { images } : {}), + }); + } catch (error) { + setRunning(false); + if (!isAgentSessionClosedError(error)) { + setToast({ message: cleanError(error instanceof Error ? error.message : String(error)), type: "error" }); + } + } finally { + setEditingSubmitting(false); + } + }, [editingMessageId, editingSubmitting, editingText, hydrateSnapshot, messages, readAgentSnapshot, running, t]); + const sendMessage = async () => { const text = draft.trim(); if (!text && attachments.length === 0) return; const alreadyRunning = running; - setDraft(""); const queued = alreadyRunning; const messageImages = attachments.map(({ data, mimeType }) => ({ data, mimeType })); - setMessages((current) => [...current, optimisticUserMessage(text || t("composer.attachedImage"), queued, messageImages)]); const images = messageImages.map((image) => ({ type: "image", ...image })); - setAttachments([]); setRunning(true); try { + setDraft(""); + setMessages((current) => [...current, optimisticUserMessage(text || t("composer.attachedImage"), queued, messageImages)]); + setAttachments([]); await window.dscode.agent.command(alreadyRunning ? "steer" : "prompt", { message: text || t("composer.describeImage"), ...(images.length ? { images } : {}), @@ -735,6 +805,7 @@ export default function App() { void openFilePreview(filePath); }, [openFilePreview]); const conversationGroups = useMemo(() => groupConversation(messages), [messages]); + const editableUserMessageId = [...conversationGroups].reverse().find((group) => group.type === "user")?.id; const latestAssistantGroup = [...conversationGroups].reverse().find((group) => group.type === "assistant"); const activeAssistantGroupId = running ? latestAssistantGroup?.id : undefined; const activeAssistantHasWork = latestAssistantGroup?.type === "assistant" @@ -1000,7 +1071,20 @@ export default function App() {
{conversationGroups.map((group) => ( group.type === "user" - ? + ? void submitEditingMessage()} + onEditCancel={cancelEditingMessage} + /> : { + if (!message.text) return; + try { + if (navigator.clipboard?.writeText) { + await navigator.clipboard.writeText(message.text); + } else { + const fallback = document.createElement("textarea"); + fallback.value = message.text; + fallback.setAttribute("readonly", "true"); + fallback.style.position = "fixed"; + fallback.style.opacity = "0"; + document.body.appendChild(fallback); + fallback.select(); + document.execCommand("copy"); + fallback.remove(); + } + setCopied(true); + window.setTimeout(() => setCopied(false), 1_200); + } catch { + setCopied(false); + } + }; + return ( -
0 ? " has-images" : ""}`}> - {message.images.length > 0 && ( -
1 ? " multiple" : ""}`}> - {message.images.map((image, index) => ( - - ))} +
+
0 ? " has-images" : ""}${editing ? " is-editing" : ""}`}> + {message.images.length > 0 && ( +
1 ? " multiple" : ""}`}> + {message.images.map((image, index) => ( + + ))} +
+ )} + {editing ? ( +