From 3a29574b7694038163f6d8cf3358261b4f34fec3 Mon Sep 17 00:00:00 2001 From: chiragbiradar Date: Wed, 15 Jul 2026 09:41:59 +0530 Subject: [PATCH 1/2] feat(eval): add agent-notes benchmark arm to the SWE-chat planning harness Adds a 'notes' runner to swechat-plan: the agent plans with a docs/agent-notes.md maintained by an agent across the same prior sessions used to build Greplica memory, giving the benchmark a realistic non-Greplica docs baseline alongside the graph-export 'docs' arm. - evals/swechat-plan/build-notes.ts: replays the case's prior sessions in order (same transcripts, same reconstructed edits as build-memory) but has the agent maintain a single agent-notes.md instead of Greplica proposals; stores the final notes as a checked-in notes-seeds/agent-notes.md with a manifest. Notes are committed into the replay snapshots so git reset/clean between sessions cannot drop them. - run.ts: --runner notes seeds docs/agent-notes.md from the case's notes-seeds, prompts the agent to navigate from the notes, blocks greplica/codegraph as in baseline, audits notes-file usage (first_notes_memory_command, notes_memory_first_navigation_used) and forbids notes access from other runners; records notes size stats next to the docs-arm stats. - build-memory.ts: export the transcript/edit replay helpers so build-notes reuses them instead of duplicating. - package.json: eval:swechat-plan-build-notes script. --- evals/swechat-plan/build-memory.ts | 8 +- evals/swechat-plan/build-notes.ts | 331 +++++++++++++++++++++++++++++ evals/swechat-plan/run.ts | 66 +++++- package.json | 1 + 4 files changed, 398 insertions(+), 8 deletions(-) create mode 100644 evals/swechat-plan/build-notes.ts diff --git a/evals/swechat-plan/build-memory.ts b/evals/swechat-plan/build-memory.ts index 8384656..e2000b9 100644 --- a/evals/swechat-plan/build-memory.ts +++ b/evals/swechat-plan/build-memory.ts @@ -47,7 +47,7 @@ interface CaseConfig { }; } -interface SessionSpec { +export interface SessionSpec { index: number; sessionId: string; createdAt: string; @@ -87,7 +87,7 @@ interface Replay { edits: ExtractedEdits; } -interface ExtractedEdits { +export interface ExtractedEdits { files: Array<{ path: string; original: string; final: string; editCount: number }>; warnings: string[]; } @@ -416,7 +416,7 @@ function sessionSpecs(context: Context): SessionSpec[] { })); } -function extractClaudeEdits(transcript: string, repoDirName: string): ExtractedEdits { +export function extractClaudeEdits(transcript: string, repoDirName: string): ExtractedEdits { const states = new Map(); const warnings: string[] = []; for (const [lineIndex, line] of transcript.split("\n").entries()) { @@ -461,7 +461,7 @@ function extractClaudeEdits(transcript: string, repoDirName: string): ExtractedE }; } -function claudeTranscriptToMarkdown(transcript: string, spec: SessionSpec): string { +export function claudeTranscriptToMarkdown(transcript: string, spec: SessionSpec): string { const sections = [ "# Historical SWE-chat Session Transcript", "", diff --git a/evals/swechat-plan/build-notes.ts b/evals/swechat-plan/build-notes.ts new file mode 100644 index 0000000..e838110 --- /dev/null +++ b/evals/swechat-plan/build-notes.ts @@ -0,0 +1,331 @@ +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + findRepoRoot, + readJson, + run, + runOrThrow, + timestamp, + valueAfter, + writeJson, +} from "../lib/common.js"; +import { runCodexAgent } from "../../libs/agent-runner/codex.js"; +import type { AgentRunResult } from "../../libs/agent-runner/types.js"; +import { loadRepoEnv } from "../../libs/env/load-local-env.js"; +import { + claudeTranscriptToMarkdown, + extractClaudeEdits, + type ExtractedEdits, + type SessionSpec, +} from "./build-memory.js"; + +interface CaseConfig { + case_id: string; + dataset: { + prior_sessions?: Array<{ + session_id: string; + created_at?: string; + checkpoint_id?: string; + title?: string; + }>; + transcript_root?: string; + }; + repo: { + full_name: string; + base_commit: string; + }; + notes?: { + seed_path?: string; + }; +} + +interface Replay { + spec: SessionSpec; + transcriptMarkdownPath: string; + edits: ExtractedEdits; +} + +interface Args { + caseId: string; + agentModel?: string; + runRoot?: string; + fixtureOnly: boolean; +} + +interface Context { + repoRoot: string; + caseDir: string; + runDir: string; + targetRepoDir: string; + transcriptMarkdownDir: string; + codexHomeDir: string; + notesRepoPath: string; + storedNotesPath: string; + storedManifestPath: string; + config: CaseConfig; +} + +interface GenerationStep { + kind: "bootstrap" | "update"; + session_id?: string; + generation: AgentRunResult; + notes_chars: number; +} + +export async function main(argv = process.argv.slice(2), defaultCaseId?: string): Promise { + const args = parseArgs(argv, defaultCaseId); + const context = prepareRun(args); + const model = args.agentModel ?? "gpt-5.5"; + const replays = prepareReplayArtifacts(context); + + if (args.fixtureOnly) { + writeJson(resolve(context.runDir, "manifest.json"), buildManifest(context, model, replays, [])); + console.log("SWE-chat notes generation fixture prep passed."); + console.log(`Run directory: ${context.runDir}`); + return; + } + + await prepareTargetRepo(context); + commitCleanRepoSnapshot(context, `repo snapshot ${context.config.repo.base_commit}`); + + const steps: GenerationStep[] = []; + const bootstrap = await runNotesAgent(context, model, bootstrapNotesPrompt(context), "00-bootstrap"); + steps.push({ kind: "bootstrap", generation: bootstrap, notes_chars: readNotes(context).length }); + commitCleanRepoSnapshot(context, "bootstrap agent notes"); + + for (const replay of replays) { + prepareSessionOriginalWorkingTree(context, replay); + applyPostSessionWorkingTree(context, replay); + const label = `${String(replay.spec.index).padStart(2, "0")}-${replay.spec.sessionId.slice(0, 8)}`; + const generation = await runNotesAgent(context, model, updateNotesPrompt(context, replay), label); + steps.push({ kind: "update", session_id: replay.spec.sessionId, generation, notes_chars: readNotes(context).length }); + commitCleanRepoSnapshot(context, `agent notes after session ${replay.spec.sessionId}`); + } + + const notes = readNotes(context); + if (notes.trim().length === 0) throw new Error(`Notes generation produced an empty ${context.notesRepoPath}.`); + mkdirSync(dirname(context.storedNotesPath), { recursive: true }); + writeFileSync(context.storedNotesPath, notes); + const manifest = buildManifest(context, model, replays, steps, notes); + writeJson(resolve(context.runDir, "manifest.json"), manifest); + writeJson(context.storedManifestPath, manifest); + + console.log("SWE-chat notes generation passed."); + console.log(`Run directory: ${context.runDir}`); + console.log(`Stored notes: ${context.storedNotesPath}`); +} + +function prepareRun(args: Args): Context { + const repoRoot = findRepoRoot(import.meta.url); + loadRepoEnv(repoRoot); + const caseDir = resolve(repoRoot, "evals/cases", args.caseId); + const config = readJson(resolve(caseDir, "case.json")); + if (config.case_id !== args.caseId) throw new Error(`Unexpected case id in case.json: ${config.case_id}`); + const runDir = resolve(args.runRoot ?? resolve(repoRoot, "eval-runs", timestamp(), config.case_id, "build-notes")); + mkdirSync(runDir, { recursive: true }); + const storedNotesPath = resolve(caseDir, config.notes?.seed_path ?? "notes-seeds/agent-notes.md"); + return { + repoRoot, + caseDir, + runDir, + targetRepoDir: resolve(runDir, "target-repo"), + transcriptMarkdownDir: resolve(runDir, "transcripts"), + codexHomeDir: resolve(runDir, "notes-codex-home"), + notesRepoPath: "docs/agent-notes.md", + storedNotesPath, + storedManifestPath: resolve(dirname(storedNotesPath), "manifest.json"), + config, + }; +} + +function prepareReplayArtifacts(context: Context): Replay[] { + mkdirSync(context.transcriptMarkdownDir, { recursive: true }); + return sessionSpecs(context).map((spec) => { + const transcriptPath = resolve( + context.repoRoot, + context.config.dataset.transcript_root ?? ".context/swechat-data/transcripts", + `${spec.sessionId}.jsonl`, + ); + if (!existsSync(transcriptPath)) throw new Error(`Missing transcript: ${transcriptPath}`); + const transcript = readFileSync(transcriptPath, "utf8"); + const transcriptMarkdownPath = resolve( + context.transcriptMarkdownDir, + `${String(spec.index).padStart(2, "0")}-${spec.sessionId.slice(0, 8)}.messages.md`, + ); + writeFileSync(transcriptMarkdownPath, claudeTranscriptToMarkdown(transcript, spec)); + const edits = extractClaudeEdits(transcript, repoName(context.config.repo.full_name)); + if (edits.files.length === 0) edits.warnings.push("no successful file edits reconstructed; replay is transcript-only"); + return { spec, transcriptMarkdownPath, edits }; + }); +} + +function sessionSpecs(context: Context): SessionSpec[] { + const sessions = context.config.dataset.prior_sessions ?? []; + if (sessions.length === 0) throw new Error(`Case ${context.config.case_id} has no prior sessions for notes generation.`); + return sessions.map((session, index) => ({ + index: index + 1, + sessionId: session.session_id, + createdAt: session.created_at ?? "", + checkpointId: session.checkpoint_id ?? "", + title: session.title ?? "", + })); +} + +async function prepareTargetRepo(context: Context): Promise { + rmSync(context.targetRepoDir, { recursive: true, force: true }); + const archivePath = resolve(context.runDir, "base-source.tar.gz"); + const extractDir = resolve(context.runDir, "base-source"); + rmSync(extractDir, { recursive: true, force: true }); + mkdirSync(extractDir, { recursive: true }); + const response = await fetch(`https://codeload.github.com/${context.config.repo.full_name}/tar.gz/${context.config.repo.base_commit}`); + if (!response.ok) throw new Error(`Failed to download base archive: ${response.status} ${response.statusText}`); + writeFileSync(archivePath, Buffer.from(await response.arrayBuffer())); + const extract = run(["tar", "-xzf", archivePath, "-C", extractDir], context.repoRoot, process.env); + if (extract.exit_code !== 0) throw new Error(`Failed to extract base archive: ${extract.stderr ?? extract.stdout ?? ""}`); + const roots = readdirSync(extractDir, { withFileTypes: true }).filter((entry) => entry.isDirectory()); + if (roots.length !== 1) throw new Error(`Expected one archive root in ${extractDir}, found ${roots.length}`); + renameSync(resolve(extractDir, roots[0]?.name ?? ""), context.targetRepoDir); + runOrThrow(["git", "init", "-q"], context.targetRepoDir); + runOrThrow(["git", "remote", "add", "origin", `swechat-eval://${context.config.repo.full_name}`], context.targetRepoDir); +} + +function commitCleanRepoSnapshot(context: Context, message: string): void { + runOrThrow(["git", "add", "-A"], context.targetRepoDir); + runOrThrow( + ["git", "-c", "user.email=swechat-notes@example.invalid", "-c", "user.name=SWE-chat Notes Replay", "commit", "-q", "--allow-empty", "--no-gpg-sign", "-m", message], + context.targetRepoDir, + ); +} + +function prepareSessionOriginalWorkingTree(context: Context, replay: Replay): void { + run(["git", "reset", "--hard"], context.targetRepoDir, process.env); + run(["git", "clean", "-fd"], context.targetRepoDir, process.env); + for (const file of replay.edits.files) writeRepoFile(context.targetRepoDir, file.path, file.original); + commitCleanRepoSnapshot(context, `original session file state ${replay.spec.sessionId}`); +} + +function applyPostSessionWorkingTree(context: Context, replay: Replay): void { + for (const file of replay.edits.files) writeRepoFile(context.targetRepoDir, file.path, file.final); +} + +async function runNotesAgent(context: Context, model: string, prompt: string, label: string): Promise { + mkdirSync(context.codexHomeDir, { recursive: true }); + const result = await runCodexAgent({ + cwd: context.targetRepoDir, + env: { ...process.env, CODEX_HOME: context.codexHomeDir }, + model, + prompt, + transcriptPath: resolve(context.runDir, `${label}-agent-events.jsonl`), + finalMessagePath: resolve(context.runDir, `${label}-final-message.txt`), + }); + if (result.exit_code !== 0) throw new Error(`Notes agent failed for ${label} with exit code ${String(result.exit_code)}.`); + if (readNotes(context).trim().length === 0) throw new Error(`Notes agent did not write ${context.notesRepoPath} for ${label}.`); + return result; +} + +function bootstrapNotesPrompt(context: Context): string { + return `You are preparing repository notes for future coding agents. + +Task: explore this repository and create ${context.notesRepoPath} (create the docs directory if it is missing) - a single Markdown file of durable engineering notes that a future agent should read before starting a task here. + +Guidelines: +- Capture repo identity, the major components and where they live, key workflows and commands, and important constraints or gotchas. +- Prefer concrete facts with file paths over vague prose. +- Keep the file shallow and navigation-focused, not an exhaustive inventory. +- Do not modify any file other than ${context.notesRepoPath}. Do not commit. +- Do not use git history, the network, or any external tool. + +This is a full ${context.config.repo.full_name} checkout at the benchmark base snapshot.`; +} + +function updateNotesPrompt(context: Context, replay: Replay): string { + const changeFact = replay.edits.files.length > 0 + ? "The historical session's code changes have already been applied to this working tree as uncommitted changes." + : "No code edits were reconstructed for this historical session; treat it as transcript-only evidence and do not invent a patch."; + return `You are updating repository notes after a completed coding session. + +${context.notesRepoPath} contains notes from earlier sessions. The transcript of a historical coding session on this repository is at: +${replay.transcriptMarkdownPath} + +${changeFact} + +Task: read the transcript and update ${context.notesRepoPath} with the durable learnings from this session that a future agent should know - decisions, constraints, gotchas, corrected assumptions, and where the relevant code lives. + +Guidelines: +- Edit and reorganize the notes freely, but keep them in this single file. +- Keep existing notes that are still true; correct anything the session proved wrong. +- Prefer concrete facts with file paths over session narration. +- Do not modify any file other than ${context.notesRepoPath}. Do not commit. +- Do not use git history, the network, or any external tool.`; +} + +function readNotes(context: Context): string { + try { + return readFileSync(resolve(context.targetRepoDir, context.notesRepoPath), "utf8"); + } catch { + return ""; + } +} + +function buildManifest(context: Context, model: string, replays: Replay[], steps: GenerationStep[], notes?: string) { + const chars = notes?.length ?? 0; + return { + case_id: context.config.case_id, + notes_path: relativeSeedPath(context), + model, + generated_at: new Date().toISOString(), + sessions: replays.map((replay) => ({ session_id: replay.spec.sessionId, title: replay.spec.title })), + steps: steps.map((step) => ({ + kind: step.kind, + session_id: step.session_id, + notes_chars: step.notes_chars, + total_tokens: step.generation.total_tokens, + tool_calls: step.generation.tool_calls, + })), + stats: { + notes_chars: chars, + notes_estimated_tokens: Math.ceil(chars / 4), + }, + }; +} + +function relativeSeedPath(context: Context): string { + return context.config.notes?.seed_path ?? "notes-seeds/agent-notes.md"; +} + +function repoName(fullName: string): string { + return fullName.split("/").pop() ?? fullName; +} + +function writeRepoFile(repoDir: string, path: string, content: string): void { + const fullPath = resolve(repoDir, path); + mkdirSync(dirname(fullPath), { recursive: true }); + writeFileSync(fullPath, content); +} + +function parseArgs(argv: string[], defaultCaseId?: string): Args { + const caseId = valueAfter(argv, "--case") ?? defaultCaseId; + if (!caseId) throw new Error("Usage: swechat-plan build-notes --case "); + return { + caseId, + agentModel: valueAfter(argv, "--agent-model"), + runRoot: valueAfter(argv, "--run-root"), + fixtureOnly: argv.includes("--fixture-only"), + }; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/evals/swechat-plan/run.ts b/evals/swechat-plan/run.ts index ad35270..eedfc6c 100644 --- a/evals/swechat-plan/run.ts +++ b/evals/swechat-plan/run.ts @@ -25,7 +25,7 @@ import { runCodexAgent } from "../../libs/agent-runner/codex.js"; import type { AgentRunResult } from "../../libs/agent-runner/types.js"; import { loadRepoEnv } from "../../libs/env/load-local-env.js"; -type RunnerName = "baseline" | "greplica" | "docs"; +type RunnerName = "baseline" | "greplica" | "docs" | "notes"; type JudgeKey = | "is_actionable_engineering_plan" | "identifies_relevant_systems" @@ -58,6 +58,9 @@ interface CaseConfig { memory?: { manifest_path?: string; }; + notes?: { + seed_path?: string; + }; } interface Args { @@ -79,6 +82,9 @@ interface RunContext { codexHomeDir: string; docsMemoryDir: string; docsMemoryStatsPath: string; + notesSeedPath: string; + notesMemoryPath: string; + notesMemoryStatsPath: string; transcriptPath: string; finalPlanPath: string; judgeInputPath: string; @@ -100,6 +106,9 @@ interface TranscriptAudit { first_docs_memory_command?: string; docs_memory_commands?: string[]; docs_memory_first_navigation_used?: boolean; + first_notes_memory_command?: string; + notes_memory_commands?: string[]; + notes_memory_first_navigation_used?: boolean; } interface DocsMemoryStats { @@ -112,6 +121,12 @@ interface DocsMemoryStats { largest_files: Array<{ path: string; chars: number }>; } +interface NotesMemoryStats { + path: string; + notes_chars: number; + notes_estimated_tokens: number; +} + interface JudgeChecks extends Record { evidence: Record; explanation: string; @@ -152,6 +167,7 @@ export async function main(argv = process.argv.slice(2), defaultCaseId?: string) const seedCommands = args.runner === "greplica" || args.runner === "docs" ? seedGreplicaMemory(context) : []; const docsMemorySetup = args.runner === "docs" ? setupDocsMemory(context) : null; + const notesMemorySetup = args.runner === "notes" ? setupNotesMemory(context) : null; installToolGuards(context.guardDir, args.runner, context.greplicaCommand); const generation = await runPlanningAgent(context, args); const changedFiles = changedFilesInRepo(context.targetRepoDir); @@ -212,6 +228,9 @@ export async function main(argv = process.argv.slice(2), defaultCaseId?: string) docs_memory_dir: docsMemorySetup?.stats.directory, docs_memory_stats_path: docsMemorySetup === null ? undefined : context.docsMemoryStatsPath, docs_memory_stats: docsMemorySetup?.stats, + notes_memory_path: notesMemorySetup?.path, + notes_memory_stats_path: notesMemorySetup === null ? undefined : context.notesMemoryStatsPath, + notes_memory_stats: notesMemorySetup ?? undefined, fixture_prep: fixturePrep, generation, final_plan_path: context.finalPlanPath, @@ -256,6 +275,9 @@ function prepareRun(args: Args): RunContext { codexHomeDir: resolve(runDir, "greplica-setup-codex-home"), docsMemoryDir: resolve(runDir, "target-repo", "greplica-memory-docs"), docsMemoryStatsPath: resolve(runDir, "docs-memory-stats.json"), + notesSeedPath: resolve(caseDir, config.notes?.seed_path ?? "notes-seeds/agent-notes.md"), + notesMemoryPath: resolve(runDir, "target-repo", "docs", "agent-notes.md"), + notesMemoryStatsPath: resolve(runDir, "notes-memory-stats.json"), transcriptPath: resolve(runDir, "agent-events.jsonl"), finalPlanPath: resolve(runDir, "final-plan.md"), judgeInputPath: resolve(runDir, "judge-input.json"), @@ -358,6 +380,23 @@ function collectDocsMemoryStats(context: RunContext): DocsMemoryStats { }; } +function setupNotesMemory(context: RunContext): NotesMemoryStats { + if (!existsSync(context.notesSeedPath)) { + throw new Error(`Notes seed missing: ${context.notesSeedPath}. Generate it with swechat-plan build-notes --case ${context.config.case_id}.`); + } + const notes = readFileSync(context.notesSeedPath, "utf8"); + if (notes.trim().length === 0) throw new Error(`Notes seed is empty: ${context.notesSeedPath}`); + mkdirSync(dirname(context.notesMemoryPath), { recursive: true }); + writeFileSync(context.notesMemoryPath, notes); + const stats: NotesMemoryStats = { + path: relative(context.runDir, context.notesMemoryPath), + notes_chars: notes.length, + notes_estimated_tokens: Math.ceil(notes.length / 4), + }; + writeJson(context.notesMemoryStatsPath, stats); + return stats; +} + async function runPlanningAgent(context: RunContext, args: Args): Promise { return runCodexAgent({ cwd: context.targetRepoDir, @@ -387,7 +426,12 @@ function agentPrompt(context: RunContext, runner: RunnerName): string { - Treat the docs as navigation context, then verify only the repo files needed for the plan. - Do not use Greplica commands. - Do not use CodeGraph commands.` - : `- Do not use Greplica commands or Greplica memory. + : runner === "notes" + ? `- Use docs/agent-notes.md as your repo memory map: read it before broad manual exploration. +- Treat the notes as remembered navigation context rather than final truth, then verify only the repo files needed for the plan. +- Do not use Greplica commands. +- Do not use CodeGraph commands.` + : `- Do not use Greplica commands or Greplica memory. - Do not use CodeGraph commands.`; return `You are running a local-only planning benchmark. @@ -448,6 +492,8 @@ function auditTranscript(transcriptPath: string, runner: RunnerName): Transcript const greplica_context_commands: string[] = []; let first_docs_memory_command: string | undefined; const docs_memory_commands: string[] = []; + let first_notes_memory_command: string | undefined; + const notes_memory_commands: string[] = []; for (const [index, line] of readOptional(transcriptPath).split("\n").entries()) { if (!line.trim()) continue; let event: unknown; @@ -465,6 +511,10 @@ function auditTranscript(transcriptPath: string, runner: RunnerName): Transcript first_docs_memory_command ??= command; docs_memory_commands.push(command); } + if (commandTouchesNotesMemory(command)) { + first_notes_memory_command ??= command; + notes_memory_commands.push(command); + } const violation = auditCommand(command, runner); if (violation) violations.push(`line ${index + 1}: ${violation}: ${command}`); } @@ -479,6 +529,9 @@ function auditTranscript(transcriptPath: string, runner: RunnerName): Transcript first_docs_memory_command, docs_memory_commands, docs_memory_first_navigation_used: runner === "docs" && first_command !== undefined && commandTouchesDocsMemory(first_command), + first_notes_memory_command, + notes_memory_commands, + notes_memory_first_navigation_used: runner === "notes" && first_command !== undefined && commandTouchesNotesMemory(first_command), }; } @@ -495,6 +548,7 @@ function auditCommand(command: string, runner: RunnerName): string | undefined { return "forbidden_codegraph_command"; } if (normalized.includes("greplica-memory-docs") && runner !== "docs") return "forbidden_docs_memory_access"; + if (normalized.includes("agent-notes.md") && runner !== "notes") return "forbidden_notes_memory_access"; if (/\bnpx\s+.*codegraph\b/.test(normalized) || /\bnpm\s+(exec|install)\b.*codegraph\b/.test(normalized)) return "forbidden_codegraph_registry_or_setup"; if (/\bgit\s+(clone|fetch|pull|log|show|reflog|blame|bisect)\b/.test(normalized)) return "forbidden_git_history_or_network"; if (normalized.includes(".context/swechat") || normalized.includes("judge-input") || normalized.includes("case.json") || normalized.includes("judge.md")) return "hidden_eval_artifact_access"; @@ -506,6 +560,10 @@ function commandTouchesDocsMemory(command: string): boolean { return command.toLowerCase().includes("greplica-memory-docs"); } +function commandTouchesNotesMemory(command: string): boolean { + return command.toLowerCase().includes("agent-notes.md"); +} + function commandInvokesTool(command: string, tool: "greplica" | "codegraph"): boolean { const escaped = tool.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return new RegExp(`(^|[\\s;&|('"\\\\/])${escaped}($|[\\s'"])`).test(command.toLowerCase().replace(/\s+/g, " ")); @@ -640,9 +698,9 @@ function extractOutputText(body: Record): string { function parseArgs(argv: string[], defaultCaseId?: string): Args { const caseId = valueAfter(argv, "--case") ?? defaultCaseId; - if (!caseId) throw new Error("Usage: swechat-plan run --case --runner baseline|greplica|docs"); + if (!caseId) throw new Error("Usage: swechat-plan run --case --runner baseline|greplica|docs|notes"); const runner = valueAfter(argv, "--runner") ?? "baseline"; - if (runner !== "baseline" && runner !== "greplica" && runner !== "docs") throw new Error("Only --runner baseline|greplica|docs is supported."); + if (runner !== "baseline" && runner !== "greplica" && runner !== "docs" && runner !== "notes") throw new Error("Only --runner baseline|greplica|docs|notes is supported."); return { caseId, runner, diff --git a/package.json b/package.json index 6544d30..d42d54d 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "eval:search-current": "npm run build && node dist/evals/cases/search-current-repo-at-8038fe8/run.js", "eval:swechat-plan": "npm run build && node dist/evals/swechat-plan/run.js", "eval:swechat-plan-build-memory": "npm run build && node dist/evals/swechat-plan/build-memory.js", + "eval:swechat-plan-build-notes": "npm run build && node dist/evals/swechat-plan/build-notes.js", "eval:transcript-backfill-insights": "npm run build && node dist/evals/cases/transcript-backfill-insights/run.js --judge openai", "eval:claim-dedupe-threshold-calibration": "npm run build && node dist/evals/cases/claim-dedupe-threshold-calibration/run.js", "eval:update-working-memory-source-evidence": "npm run build && node dist/evals/cases/update-working-memory-source-evidence-at-0438915/run.js", From 25a4344d3f2773a079496ae868dfdacdf7d10716 Mon Sep 17 00:00:00 2001 From: chiragbiradar Date: Thu, 16 Jul 2026 14:14:32 +0530 Subject: [PATCH 2/2] fix: use relative tar paths in build-notes for Windows compatibility Same GNU-tar absolute-Windows-path issue fixed for run.ts and build-memory.ts in a separate PR; build-notes carries the fix for its own copy of the extraction step. --- evals/swechat-plan/build-notes.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/evals/swechat-plan/build-notes.ts b/evals/swechat-plan/build-notes.ts index e838110..3ef7b21 100644 --- a/evals/swechat-plan/build-notes.ts +++ b/evals/swechat-plan/build-notes.ts @@ -7,7 +7,7 @@ import { rmSync, writeFileSync, } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { dirname, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { findRepoRoot, @@ -189,7 +189,9 @@ async function prepareTargetRepo(context: Context): Promise { const response = await fetch(`https://codeload.github.com/${context.config.repo.full_name}/tar.gz/${context.config.repo.base_commit}`); if (!response.ok) throw new Error(`Failed to download base archive: ${response.status} ${response.statusText}`); writeFileSync(archivePath, Buffer.from(await response.arrayBuffer())); - const extract = run(["tar", "-xzf", archivePath, "-C", extractDir], context.repoRoot, process.env); + // Run tar from the run directory with relative paths: GNU tar interprets + // absolute Windows paths ("C:\...") as remote host specs and fails. + const extract = run(["tar", "-xzf", relative(context.runDir, archivePath), "-C", relative(context.runDir, extractDir)], context.runDir, process.env); if (extract.exit_code !== 0) throw new Error(`Failed to extract base archive: ${extract.stderr ?? extract.stdout ?? ""}`); const roots = readdirSync(extractDir, { withFileTypes: true }).filter((entry) => entry.isDirectory()); if (roots.length !== 1) throw new Error(`Expected one archive root in ${extractDir}, found ${roots.length}`);