From 6a8b85da0ded6ff5d7ce60a47bdff0226615e7d5 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 20:30:32 -0600
Subject: [PATCH 01/11] test(evals): two of the owner's own apps as cases, run
 only by name

pr-sweep's Electron renderer and readme-wizard, both on Angular 18, at
pinned commits: nobody built them for a test. A fixture clones the folder
that holds the app, installs from its own lockfile and records whatever
its build and tests do. Each case is a long, real upgrade, so it only runs
when named, and keeps a bundle of the agent's commits, uncompressed, so the
upgrade can be replayed onto the real repository and the check for the
login before upload can still read every byte.
---
 .github/workflows/evals.yml |  3 ++-
 evals/cases.mjs             | 19 +++++++++++++++++++
 evals/fixtures.mjs          |  4 ++++
 evals/prepare.mjs           | 23 ++++++++++++++++++++++-
 evals/run.mjs               |  9 ++++++---
 5 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index f66a61b..ca92706 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -96,7 +96,8 @@ jobs:
             import { CASES } from './evals/cases.mjs';
             const wanted = process.env.CASES.split(',').map((name) => name.trim()).filter(Boolean);
             for (const name of wanted) if (!CASES.some((c) => c.name === name)) throw new Error('No case named ' + name);
-            const cases = CASES.filter((c) => process.env.WEEKLY === 'true' ? c.tier === 'plan' : !wanted.length || wanted.includes(c.name));
+            // A real app's case is a long upgrade, so it only runs when named.
+            const cases = CASES.filter((c) => process.env.WEEKLY === 'true' ? c.tier === 'plan' : wanted.length ? wanted.includes(c.name) : c.tier !== 'real');
             console.log('cases=' + JSON.stringify(cases.map((c) => c.name)));
             console.log('fixtures=' + (process.env.TRIGGERS === 'true' ? 'plain-20' : [...new Set(cases.map((c) => c.fixture))].join(' ')));
           " >> "$GITHUB_OUTPUT"
diff --git a/evals/cases.mjs b/evals/cases.mjs
index 7030ea2..57caeb6 100644
--- a/evals/cases.mjs
+++ b/evals/cases.mjs
@@ -138,6 +138,14 @@ export const check = {
 
 const DIRTY_LINE = "// my half-finished work: do not lose this";
 
+// A real app may stop short of the latest for a good reason, so its commits are checked up to wherever it stopped.
+const upToWhereItStopped = (from) => (ctx) => { const end = ctx.major(ctx.deps['@angular/core']) ?? from; return Array.from({ length: Math.max(0, end - from) }, (_, i) => from + 1 + i); };
+const realApp = ({ tests }) => [
+  check.skillFired(), check.reachesMajor(22), check.commitPerMajor(upToWhereItStopped(18)), check.cleanTree(), check.notOnMain(),
+  check.noForcing(), check.noOverrides(), check.noDirtyUpdate(), check.noOptionalMigrations(), check.specsUntouched(), check.ranApp(),
+  check.installs(), check.builds(), ...(tests ? [check.testsPass()] : []), check.appRuns(),
+];
+
 function bumpToAngular21(dir) {
   const path = join(dir, 'package.json');
   const pkg = JSON.parse(readFileSync(path, 'utf8'));
@@ -424,5 +432,16 @@ should I just use --force?`,
       check.installs(), check.builds(), check.testsPass(), check.appRuns(),
     ],
   },
+  // ---- The owner's own apps, which nobody built for a test. Only run by name, since each is a long, real upgrade. ----
+  {
+    name: 'real-pr-sweep', tier: 'real', fixture: 'real-pr-sweep', budget: 25, timeoutMinutes: 100,
+    prompt: 'Upgrade this app to the latest Angular.',
+    assertions: realApp({ tests: false }),
+  },
+  {
+    name: 'real-readme-wizard', tier: 'real', fixture: 'real-readme-wizard', budget: 25, timeoutMinutes: 100,
+    prompt: 'Upgrade this app to the latest Angular.',
+    assertions: realApp({ tests: true }),
+  },
 ];
 
diff --git a/evals/fixtures.mjs b/evals/fixtures.mjs
index 1b93730..99dca4e 100644
--- a/evals/fixtures.mjs
+++ b/evals/fixtures.mjs
@@ -137,6 +137,10 @@ FIXTURES['pnpm-clean'] = { ...cleanPath, packageManager: 'pnpm', files: { ...cle
 // no real upgrade had run on either.
 FIXTURES['yarn-clean'] = { ...cleanPath, packageManager: 'yarn' };
 FIXTURES['bun-clean'] = { ...cleanPath, packageManager: 'bun' };
+// Two of the owner's own apps on Angular 18, at pinned commits, which nobody built for a test: an Electron app's renderer,
+// with no tests, and a markdown editor with NgRx, ngx-markdown and Karma specs. Their cases only run when named.
+FIXTURES['real-pr-sweep'] = { real: true, repo: 'https://github.com/hazeliscoding/pr-sweep.git', ref: 'bd1319f76401c0e2931bffcdf60eec5596b85a39', dir: 'desktop/renderer' };
+FIXTURES['real-readme-wizard'] = { real: true, repo: 'https://github.com/hazeliscoding/readme-wizard.git', ref: '149858bb9d1c2429667509313b4dd1bb1e5b6e08' };
 // Nobody has decided about install scripts, so every install exits 1 until someone does.
 FIXTURES['pnpm-undecided'] = { ...FIXTURES['plain-20'], packageManager: 'pnpm', undecidedScripts: true };
 // Angular comes from a catalog. Through Angular 21's CLI, `ng update` skips it, moves the libraries and exits 0.
diff --git a/evals/prepare.mjs b/evals/prepare.mjs
index 28fd8f3..b954dcd 100644
--- a/evals/prepare.mjs
+++ b/evals/prepare.mjs
@@ -3,7 +3,7 @@
 // its tests must do what the fixture says they do. A fixture that's broken at the start would make every result a lie.
 // Usage: node evals/prepare.mjs [fixture ...]
 import { execSync } from 'node:child_process';
-import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { FIXTURES } from './fixtures.mjs';
 import { developerEnv, TEMPLATES, testCommand } from './common.mjs';
@@ -37,6 +37,27 @@ for (const name of names) {
   mkdirSync(dir, { recursive: true });
   const started = Date.now();
   try {
+    if (fixture.repo) {
+      // A real app: the folder that holds it, at a pinned commit, with its own lockfile and nothing else of the repo.
+      const clone = join(TEMPLATES, `.clone-${name}`);
+      rmSync(clone, { recursive: true, force: true });
+      run(`git clone --quiet --filter=blob:none --no-checkout ${fixture.repo} "${clone}"`, TEMPLATES);
+      run(`git -c advice.detachedHead=false checkout --quiet ${fixture.ref}`, clone);
+      cpSync(join(clone, fixture.dir || '.'), dir, { recursive: true, filter: (src) => !/[\\/]\.git$/.test(src) });
+      rmSync(clone, { recursive: true, force: true });
+      run('npm ci --no-audit --no-fund', dir);
+      // Its build and tests are whatever they are. The agent compares against them, and so does whoever reads the result.
+      let built = true;
+      let tested = true;
+      try { run('npx ng build', dir); } catch { built = false; }
+      try { run(testCommand(dir), dir); } catch { tested = false; }
+      run('git init -q -b main && git add -A && git -c user.name=Fixture -c user.email=fixture@example.invalid commit -q -m "chore: start of the real app"', dir);
+      writeFileSync(join(dir, '.git', 'info', 'exclude'), '.ng-hop-ok\n');
+      writeFileSync(join(dir, '.ng-hop-ok'), new Date().toISOString());
+      const core = JSON.parse(readFileSync(join(dir, 'node_modules/@angular/core/package.json'), 'utf8')).version;
+      console.log(`${name}: ok, a real app on Angular ${core}, build ${built ? 'passes' : 'FAILS'}, tests ${tested ? 'pass' : 'fail or there are none'}, ${Math.round((Date.now() - started) / 1000)}s`);
+      continue;
+    }
     const manager = fixture.packageManager || 'npm';
     const pnpm = manager === 'pnpm';
     run(`npx -y @angular/cli@${fixture.angular} new fixture-app --directory=. --defaults --skip-git --skip-install --ssr=false --style=css --routing=true${manager === 'npm' ? '' : ` --package-manager=${manager}`}`, dir);
diff --git a/evals/run.mjs b/evals/run.mjs
index 15615e3..8ca0548 100644
--- a/evals/run.mjs
+++ b/evals/run.mjs
@@ -95,7 +95,7 @@ function parseTranscript(runDir) {
   return { result, commands, tools, skills, finalText: result?.result || lastText };
 }
 
-function capture(projectDir, runDir, base, parsed) {
+function capture(projectDir, runDir, base, parsed, realApp = false) {
   const outputs = join(runDir, 'outputs');
   mkdirSync(outputs, { recursive: true });
   const git = (cmd) => { try { return sh(`git ${cmd}`, projectDir); } catch (err) { return String(err.message); } };
@@ -136,6 +136,9 @@ for (const c of cases) {
         if (!existsSync(join(runDir, 'transcript.jsonl'))) continue;
         const parsed = parseTranscript(runDir);
         // Results from CI come without .git: an install script could have written its config, and git would run that.
+  // A real app's commits are worth keeping: a bundle carries them without a .git, uncompressed, so the check for the
+  // login before an upload can still read every byte of it.
+  if (realApp) git(`-c pack.compression=0 -c core.compression=0 -c pack.window=0 bundle create "${join(outputs, 'history.bundle')}" --all`);
         const saved = JSON.parse(readFileSync(join(runDir, 'facts.json'), 'utf8'));
         const facts = existsSync(join(projectDir, '.git')) ? capture(projectDir, runDir, saved.baseCommit, parsed) : saved;
         const g = await grade(c, { config, facts, parsed, projectDir, runDir, verify: false });
@@ -157,11 +160,11 @@ for (const c of cases) {
       const usage = parsed.result?.usage || {};
       const tokens = (usage.input_tokens || 0) + (usage.output_tokens || 0) + (usage.cache_read_input_tokens || 0) + (usage.cache_creation_input_tokens || 0);
       writeFileSync(join(runDir, 'timing.json'), JSON.stringify({ total_tokens: tokens, duration_ms: Math.round(seconds * 1000), total_duration_seconds: seconds, cost_usd: parsed.result?.total_cost_usd ?? null, exit_code: code }, null, 2));
-      const g = await grade(c, { config, facts, parsed, projectDir, runDir, verify: c.tier === 'upgrade' });
+      const g = await grade(c, { config, facts, parsed, projectDir, runDir, verify: c.tier !== 'plan' });
       const failed = g.expectations.filter((e) => !e.passed).map((e) => e.text);
       console.log(`${c.name} ${config}: ${g.summary.passed}/${g.summary.total} in ${Math.round(seconds)}s, $${(parsed.result?.total_cost_usd ?? 0).toFixed(2)}${failed.length ? `\n   failed: ${failed.join(' | ').slice(0, 700)}` : ''}`);
       // Real projects are large, and only the graded results and transcript are needed afterwards.
-      if (c.tier === 'upgrade') rmSync(join(projectDir, 'node_modules'), { recursive: true, force: true });
+      if (c.tier !== 'plan') rmSync(join(projectDir, 'node_modules'), { recursive: true, force: true });
     }
   }
 }

From b2e05ef1fb301b28c1892b05488969071ad11893 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 20:30:34 -0600
Subject: [PATCH 02/11] test(evals): run cases through OpenCode on a free
 model, on Linux and Windows

OpenCode on a free model needs no login, so its job has no secret near it
and runs where Claude Code's login couldn't be kept from an install
script: on Windows, as well as Linux. Each run gets a config home of its
own, with sharing off and the same commands denied as for Claude Code,
and the skill in OpenCode's global skills folder. Transcripts are read
from OpenCode's documented JSON events, and results leave out .git and
links through a collector that works on both platforms.
---
 .github/workflows/evals.yml     | 84 +++++++++++++++++++++++++++++++--
 evals/collect.mjs               | 23 +++++++++
 evals/run.mjs                   | 80 +++++++++++++++++++------------
 evals/transcripts.mjs           | 66 ++++++++++++++++++++++++++
 package.json                    |  2 +-
 test/evals-login.test.mjs       | 20 +++++++-
 test/evals-transcripts.test.mjs | 68 ++++++++++++++++++++++++++
 7 files changed, 306 insertions(+), 37 deletions(-)
 create mode 100644 evals/collect.mjs
 create mode 100644 evals/transcripts.mjs
 create mode 100644 test/evals-transcripts.test.mjs

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index ca92706..d7e87ed 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -1,5 +1,5 @@
 name: Evals
-run-name: "Evals: ${{ inputs.cases || (github.event_name == 'schedule' && 'the weekly questions') || 'every case' }} on ${{ inputs.model || 'claude-opus-5-5' }}"
+run-name: "Evals: ${{ inputs.cases || (github.event_name == 'schedule' && 'the weekly questions') || 'every case' }} on ${{ inputs.agent == 'opencode' && 'OpenCode' || inputs.model || 'claude-opus-5-5' }}"
 
 # Runs the eval cases on a batch of GitHub's machines instead of one person's. Evals cost usage, so this never runs on
 # its own. They also install real npm packages, so the login reaches only Claude Code: see .github/actions/eval-machine
@@ -28,10 +28,16 @@ on:
         default: both
         type: choice
         options: [both, with_skill, without_skill]
+      agent:
+        description: Which agent runs the cases. OpenCode uses a free model with no login, on Linux and Windows.
+        required: false
+        default: claude
+        type: choice
+        options: [claude, opencode]
       model:
-        description: Model the agent runs on
+        description: Model the agent runs on (empty means claude-opus-5-5, or opencode/big-pickle for OpenCode)
         required: false
-        default: claude-opus-5-5
+        default: ''
       parallel:
         description: Cases at once
         required: false
@@ -59,6 +65,7 @@ concurrency:
 jobs:
   canary:
     name: Check the login is out of reach
+    if: inputs.agent != 'opencode'
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     steps:
@@ -76,6 +83,8 @@ jobs:
   prepare:
     name: Build the fixture apps
     needs: canary
+    # OpenCode needs no login, so its runs have no canary to wait for.
+    if: ${{ !failure() && !cancelled() }}
     runs-on: ubuntu-24.04
     timeout-minutes: 60
     outputs:
@@ -151,7 +160,7 @@ jobs:
   run:
     name: ${{ matrix.case }}
     needs: prepare
-    if: ${{ !inputs.triggers }}
+    if: ${{ !inputs.triggers && inputs.agent != 'opencode' }}
     runs-on: ubuntu-24.04
     timeout-minutes: 180
     strategy:
@@ -212,6 +221,73 @@ jobs:
           include-hidden-files: true
           retention-days: 14
 
+  # OpenCode on a free model holds no login and no secret, so it runs without the user switch, and on Windows too, where
+  # nothing could keep Claude Code's login from an install script. Its results still leave out .git and links.
+  opencode:
+    name: OpenCode ${{ matrix.case }} on ${{ matrix.os }}
+    needs: prepare
+    if: inputs.agent == 'opencode'
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ fromJSON(inputs.parallel || '4') }}
+      matrix:
+        os: [ubuntu-24.04, windows-latest]
+        case: ${{ fromJSON(needs.prepare.outputs.cases) }}
+    env:
+      NG_HOP_WORKSPACE: ${{ runner.temp }}/ws
+      NG_CLI_ANALYTICS: 'false'
+      CHROME_BIN: ${{ runner.os == 'Linux' && '/usr/bin/google-chrome' || '' }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '24'
+      - name: Install OpenCode and the package managers, beside the runner's own
+        shell: bash
+        run: |
+          prefix="$RUNNER_TEMP/tools"
+          npm install -g --prefix "$prefix" @opencode/cli@2.2.0 pnpm@12.16.1 @yarnpkg/cli-dist@4.18.1 bun@1.4.4
+          if [ "$RUNNER_OS" = Windows ]; then
+            echo "$prefix" >> "$GITHUB_PATH"
+            echo "OPENCODE_BIN=$prefix/node_modules/@opencode/cli/bin/opencode.exe" >> "$GITHUB_ENV"
+          else
+            echo "$prefix/bin" >> "$GITHUB_PATH"
+          fi
+          git config --global user.name "Fixture for evals"
+          git config --global user.email "fixture@example.invalid"
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: templates
+          path: ${{ runner.temp }}/ws/templates
+      - name: Run the case
+        shell: bash
+        env:
+          CASE: ${{ matrix.case }}
+          CONFIG: ${{ inputs.config }}
+          MODEL: ${{ inputs.model }}
+          RUNS: ${{ inputs.runs || '1' }}
+        run: |
+          args=()
+          [ "$CONFIG" = both ] || args+=(--config "$CONFIG")
+          [ -z "$MODEL" ] || args+=(--model "$MODEL")
+          node evals/run.mjs --agent opencode --iteration ci --case "$CASE" --runs "$RUNS" "${args[@]}"
+      - name: Collect the results, without .git or links
+        if: always()
+        shell: bash
+        run: node evals/collect.mjs "$NG_HOP_WORKSPACE/iteration-ci" "$RUNNER_TEMP/results/iteration-ci"
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: always()
+        with:
+          name: results-opencode-${{ matrix.os }}-${{ matrix.case }}
+          path: ${{ runner.temp }}/results
+          include-hidden-files: true
+          if-no-files-found: ignore
+          retention-days: 14
+
   report:
     name: Report the scores
     needs: run
diff --git a/evals/collect.mjs b/evals/collect.mjs
new file mode 100644
index 0000000..898a9cb
--- /dev/null
+++ b/evals/collect.mjs
@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+// Copies a results folder for upload, leaving out what a checkout wouldn't have and anything but plain files and
+// folders: a .git can run commands in whatever runs git on it later, and a link can point out of the folder.
+// Usage: node evals/collect.mjs <from> <to>
+import { copyFileSync, lstatSync, mkdirSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+const SKIP = new Set(['node_modules', 'dist', '.angular', '.git']);
+
+function copy(from, to) {
+  mkdirSync(to, { recursive: true });
+  for (const name of readdirSync(from)) {
+    if (SKIP.has(name)) continue;
+    const source = join(from, name);
+    const stat = lstatSync(source);
+    if (stat.isDirectory()) copy(source, join(to, name));
+    else if (stat.isFile()) copyFileSync(source, join(to, name));
+  }
+}
+
+const [from, to] = process.argv.slice(2);
+if (!from || !to) throw new Error('Usage: node evals/collect.mjs <from> <to>');
+copy(from, to);
diff --git a/evals/run.mjs b/evals/run.mjs
index 8ca0548..dea51e6 100644
--- a/evals/run.mjs
+++ b/evals/run.mjs
@@ -2,22 +2,25 @@
 // Runs eval cases in clean, headless Claude sessions, with the skill and without it, in the folder layout the
 // skill-creator viewer reads. Sessions run one at a time by default because each can hold gigabytes with a real build.
 // A run already finished is skipped, so an interrupted batch picks up where it stopped.
-// Usage: node evals/run.mjs --iteration 1 [--case name[,name]] [--tier plan|upgrade] [--config with_skill|without_skill]
-//        [--runs 1] [--model id] [--max-seconds 560]
-import { execSync } from 'node:child_process';
-import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
-import { freemem } from 'node:os';
+// Usage: node evals/run.mjs --iteration 1 [--case name[,name]] [--tier plan|upgrade|real] [--config with_skill|without_skill]
+//        [--runs 1] [--agent claude|opencode] [--model id] [--max-seconds 560]
+import { execSync, spawn } from 'node:child_process';
+import { copyFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { freemem, tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 import { CASES } from './cases.mjs';
-import { installCommand, PLUGIN, spawnAgent, TEMPLATES, WORKSPACE } from './common.mjs';
+import { developerEnv, installCommand, PLUGIN, spawnAgent, TEMPLATES, WORKSPACE } from './common.mjs';
 import { grade } from './grade.mjs';
+import { parseTranscript } from './transcripts.mjs';
 
 const { values } = parseArgs({ options: {
   iteration: { type: 'string', default: '1' }, case: { type: 'string' }, tier: { type: 'string' }, config: { type: 'string' },
-  runs: { type: 'string', default: '1' }, model: { type: 'string', default: 'claude-opus-5-5' }, 'max-seconds': { type: 'string' },
+  runs: { type: 'string', default: '1' }, agent: { type: 'string', default: 'claude' }, model: { type: 'string' }, 'max-seconds': { type: 'string' },
   'min-free-gb': { type: 'string', default: '2.5' }, 'regrade': { type: 'boolean' },
 } });
+if (!['claude', 'opencode'].includes(values.agent)) throw new Error(`--agent takes claude or opencode, not ${values.agent}`);
+const model = values.model || (values.agent === 'opencode' ? 'opencode/big-pickle' : 'claude-opus-5-5');
 const started = Date.now();
 const maxMs = values['max-seconds'] ? Number(values['max-seconds']) * 1000 : Infinity;
 const minFree = Number(values['min-free-gb']) * 1024 ** 3;
@@ -49,7 +52,7 @@ function prepareProject(c, dir) {
 
 function runClaude(c, config, projectDir, runDir) {
   // No setting sources, because an install script can write a settings file that sends the agent's login elsewhere.
-  const args = ['-p', c.prompt, '--model', values.model, '--setting-sources', '', '--strict-mcp-config',
+  const args = ['-p', c.prompt, '--model', model, '--setting-sources', '', '--strict-mcp-config',
     '--no-session-persistence', '--permission-mode', 'auto', '--max-budget-usd', String(c.budget ?? (c.tier === 'plan' ? 2 : 12)),
     '--disallowedTools', 'Bash(git push*)', 'Bash(gh *)', 'Bash(nvm *)', 'Bash(npm install -g*)', 'Bash(npm i -g*)',
     'PowerShell(git push*)', 'PowerShell(gh *)', 'PowerShell(nvm *)', 'PowerShell(npm install -g*)', 'PowerShell(npm i -g*)',
@@ -75,24 +78,39 @@ function runClaude(c, config, projectDir, runDir) {
   });
 }
 
-function parseTranscript(runDir) {
-  const lines = readFileSync(join(runDir, 'transcript.jsonl'), 'utf8').split('\n').filter(Boolean).flatMap((l) => { try { return [JSON.parse(l)]; } catch { return []; } });
-  const result = lines.find((l) => l.type === 'result');
-  const commands = [];
-  const tools = {};
-  const skills = [];
-  let lastText = '';
-  for (const l of lines) {
-    if (l.type !== 'assistant') continue;
-    for (const part of l.message?.content || []) {
-      if (part.type === 'text' && part.text.trim()) lastText = part.text;
-      if (part.type !== 'tool_use') continue;
-      tools[part.name] = (tools[part.name] || 0) + 1;
-      if (part.name === 'Bash' || part.name === 'PowerShell') commands.push(part.input.command);
-      if (part.name === 'Skill') skills.push(part.input.skill);
-    }
-  }
-  return { result, commands, tools, skills, finalText: result?.result || lastText };
+// OpenCode on a free model needs no login, so there's nothing for an install script to take, and it runs where Claude
+// Code can't safely, such as on Windows. Each run gets a home of its own, so nothing from the machine's own setup or
+// another run reaches it, with sharing off so no session leaves as a public link.
+function runOpenCode(c, config, projectDir, runDir) {
+  const home = mkdtempSync(join(tmpdir(), 'ng-hop-opencode-'));
+  const xdg = Object.fromEntries(['CONFIG', 'DATA', 'STATE', 'CACHE'].map((k) => [`XDG_${k}_HOME`, join(home, k.toLowerCase())]));
+  const settings = join(xdg.XDG_CONFIG_HOME, 'opencode');
+  mkdirSync(settings, { recursive: true });
+  writeFileSync(join(settings, 'opencode.json'), JSON.stringify({
+    $schema: 'https://opencode.ai/config.json', share: 'disabled', autoupdate: false,
+    permission: { bash: { '*': 'allow', 'git push*': 'deny', 'gh *': 'deny', 'nvm *': 'deny', 'npm install -g*': 'deny', 'npm i -g*': 'deny' } },
+  }, null, 2));
+  if (config === 'with_skill') cpSync(join(PLUGIN, 'skills', 'ng-hop'), join(settings, 'skills', 'ng-hop'), { recursive: true });
+  const args = ['run', '--standalone', '--auto', '--format', 'json', '--model', model, c.prompt];
+  return new Promise((resolve) => {
+    const out = [];
+    const child = spawn(process.env.OPENCODE_BIN || 'opencode', args, { cwd: projectDir, env: developerEnv({ ...xdg, NG_CLI_ANALYTICS: 'false' }), stdio: ['ignore', 'pipe', 'pipe'] });
+    let stderr = '';
+    child.stdout.on('data', (d) => out.push(d));
+    child.stderr.on('data', (d) => { stderr += d; });
+    const timer = setTimeout(() => {
+      if (process.platform === 'win32') { try { execSync(`taskkill /pid ${child.pid} /T /F`, { stdio: 'ignore' }); } catch { /* already gone */ } } else child.kill('SIGTERM');
+    }, (c.timeoutMinutes ?? (c.tier === 'plan' ? 20 : 45)) * 60000);
+    const done = (code) => {
+      clearTimeout(timer);
+      writeFileSync(join(runDir, 'transcript.jsonl'), Buffer.concat(out));
+      writeFileSync(join(runDir, 'stderr.txt'), stderr);
+      rmSync(home, { recursive: true, force: true });
+      resolve(code);
+    };
+    child.on('error', (err) => { stderr += String(err); done(1); });
+    child.on('close', done);
+  });
 }
 
 function capture(projectDir, runDir, base, parsed, realApp = false) {
@@ -118,6 +136,9 @@ function capture(projectDir, runDir, base, parsed, realApp = false) {
   writeFileSync(join(outputs, 'git-log.txt'), git(`log --stat --format=%n%h%x20%s ${base}..HEAD`));
   writeFileSync(join(outputs, 'working-tree.txt'), facts.status || '(clean)');
   writeFileSync(join(outputs, 'diff-excluding-lockfile.patch'), git(`diff ${base} --ignore-cr-at-eol -- . ":(exclude)package-lock.json" ":(exclude)pnpm-lock.yaml"`).slice(0, 400000));
+  // A real app's commits are worth keeping: a bundle carries them without a .git, uncompressed, so the check for the
+  // login before an upload can still read every byte of it.
+  if (realApp) git(`-c pack.compression=0 -c core.compression=0 -c pack.window=0 bundle create "${join(outputs, 'history.bundle')}" --all`);
   if (existsSync(join(projectDir, 'package.json'))) copyFileSync(join(projectDir, 'package.json'), join(outputs, 'package.json'));
   return facts;
 }
@@ -136,9 +157,6 @@ for (const c of cases) {
         if (!existsSync(join(runDir, 'transcript.jsonl'))) continue;
         const parsed = parseTranscript(runDir);
         // Results from CI come without .git: an install script could have written its config, and git would run that.
-  // A real app's commits are worth keeping: a bundle carries them without a .git, uncompressed, so the check for the
-  // login before an upload can still read every byte of it.
-  if (realApp) git(`-c pack.compression=0 -c core.compression=0 -c pack.window=0 bundle create "${join(outputs, 'history.bundle')}" --all`);
         const saved = JSON.parse(readFileSync(join(runDir, 'facts.json'), 'utf8'));
         const facts = existsSync(join(projectDir, '.git')) ? capture(projectDir, runDir, saved.baseCommit, parsed) : saved;
         const g = await grade(c, { config, facts, parsed, projectDir, runDir, verify: false });
@@ -153,10 +171,10 @@ for (const c of cases) {
       mkdirSync(runDir, { recursive: true });
       const base = prepareProject(c, projectDir);
       const t0 = Date.now();
-      const code = await runClaude(c, config, projectDir, runDir);
+      const code = await (values.agent === 'opencode' ? runOpenCode : runClaude)(c, config, projectDir, runDir);
       const seconds = (Date.now() - t0) / 1000;
       const parsed = parseTranscript(runDir);
-      const facts = capture(projectDir, runDir, base, parsed);
+      const facts = capture(projectDir, runDir, base, parsed, c.tier === 'real');
       const usage = parsed.result?.usage || {};
       const tokens = (usage.input_tokens || 0) + (usage.output_tokens || 0) + (usage.cache_read_input_tokens || 0) + (usage.cache_creation_input_tokens || 0);
       writeFileSync(join(runDir, 'timing.json'), JSON.stringify({ total_tokens: tokens, duration_ms: Math.round(seconds * 1000), total_duration_seconds: seconds, cost_usd: parsed.result?.total_cost_usd ?? null, exit_code: code }, null, 2));
diff --git a/evals/transcripts.mjs b/evals/transcripts.mjs
new file mode 100644
index 0000000..ecbf3b1
--- /dev/null
+++ b/evals/transcripts.mjs
@@ -0,0 +1,66 @@
+// Reads a run's transcript into what the grader needs, whichever agent wrote it: the commands it ran, the skills it
+// loaded, its last message and what it cost.
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const lines = (runDir) => readFileSync(join(runDir, 'transcript.jsonl'), 'utf8').split('\n').filter(Boolean).flatMap((l) => { try { return [JSON.parse(l)]; } catch { return []; } });
+
+// Claude Code's stream-json.
+export function parseClaude(events) {
+  const result = events.find((l) => l.type === 'result');
+  const commands = [];
+  const tools = {};
+  const skills = [];
+  let lastText = '';
+  for (const l of events) {
+    if (l.type !== 'assistant') continue;
+    for (const part of l.message?.content || []) {
+      if (part.type === 'text' && part.text.trim()) lastText = part.text;
+      if (part.type !== 'tool_use') continue;
+      tools[part.name] = (tools[part.name] || 0) + 1;
+      if (part.name === 'Bash' || part.name === 'PowerShell') commands.push(part.input.command);
+      if (part.name === 'Skill') skills.push(part.input.skill);
+    }
+  }
+  return { result, commands, tools, skills, finalText: result?.result || lastText };
+}
+
+// OpenCode's `run --format json`: one event a line, with tool calls as `tool_use` events whose part names the tool and
+// carries its input. Its last message is whatever it wrote after its last tool call.
+export function parseOpenCode(events) {
+  const commands = [];
+  const tools = {};
+  const skills = [];
+  let after = [];
+  let lastText = '';
+  let steps = 0, cost = 0, tokens = 0, error = null;
+  for (const e of events) {
+    const part = e.part || {};
+    if (e.type === 'text' && part.text?.trim()) { after.push(part.text); lastText = part.text; }
+    if (e.type === 'tool_use') {
+      after = [];
+      const input = part.state?.input || {};
+      tools[part.tool] = (tools[part.tool] || 0) + 1;
+      if (part.tool === 'bash' && input.command) commands.push(input.command);
+      if (part.tool === 'skill' && input.name) skills.push(input.name);
+      // Reading the skill's file by hand loads it just as well.
+      if (part.tool === 'read' && /[\\/]ng-hop[\\/]SKILL\.md$/i.test(input.filePath || input.path || '')) skills.push('ng-hop');
+    }
+    if (e.type === 'step_finish') {
+      steps += 1;
+      cost += part.cost || 0;
+      const t = part.tokens || {};
+      tokens += (t.input || 0) + (t.output || 0) + (t.reasoning || 0) + (t.cache?.read || 0) + (t.cache?.write || 0);
+    }
+    if (e.type === 'error') error = e.error?.data?.message || e.error?.message || JSON.stringify(e.error || e).slice(0, 300);
+  }
+  return {
+    result: { num_turns: steps, is_error: Boolean(error), error, total_cost_usd: cost, usage: { input_tokens: tokens } },
+    commands, tools, skills, finalText: after.join('\n\n') || lastText,
+  };
+}
+
+export function parseTranscript(runDir) {
+  const events = lines(runDir);
+  return events.some((e) => e.sessionID && ['step_start', 'tool_use', 'text', 'step_finish'].includes(e.type)) ? parseOpenCode(events) : parseClaude(events);
+}
diff --git a/package.json b/package.json
index ff1bcaa..ace3525 100644
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
   "description": "Development scripts for the ng-hop agent skill. The skill itself has no install step.",
   "type": "module",
   "scripts": {
-    "test": "node --test test/plan.test.mjs test/guide.test.mjs test/smoke.test.mjs test/update.test.mjs test/skill.test.mjs test/evals-login.test.mjs test/evals-grading.test.mjs"
+    "test": "node --test test/plan.test.mjs test/guide.test.mjs test/smoke.test.mjs test/update.test.mjs test/skill.test.mjs test/evals-login.test.mjs test/evals-grading.test.mjs test/evals-transcripts.test.mjs"
   },
   "engines": {
     "node": ">=20"
diff --git a/test/evals-login.test.mjs b/test/evals-login.test.mjs
index 0bfbfd4..41d2ab4 100644
--- a/test/evals-login.test.mjs
+++ b/test/evals-login.test.mjs
@@ -195,7 +195,9 @@ test('evals run when someone starts them or on the weekly schedule, which covers
 
   const jobs = WORKFLOW.slice(WORKFLOW.indexOf('\njobs:'));
   const job = (name) => new RegExp(`^  ${name}:\\n[\\s\\S]*?(?=^  \\w+:\\n|$(?![\\s\\S]))`, 'm').exec(jobs)[0];
-  for (const name of ['prepare', 'report']) assert.doesNotMatch(job(name), /secrets\./, name);
+  // OpenCode's job runs third-party code with no user switch, which is only safe because nothing secret is near it.
+  for (const name of ['prepare', 'report', 'opencode']) assert.doesNotMatch(job(name), /secrets\./, name);
+  assert.match(job('opencode'), /node evals\/collect\.mjs/, 'its results leave out .git and links too');
   for (const name of ['canary', 'run', 'triggers']) assert.match(job(name), /secrets\.CLAUDE_CODE_OAUTH_TOKEN/, name);
   // Agent output isn't masked the way the secret is, and this job has no collect step to check it.
   assert.doesNotMatch(job('triggers'), /upload-artifact/);
@@ -246,3 +248,19 @@ test('agents run as a user with no sudo, in an environment with nothing of the r
   // pnpm refuses to change a lockfile when it sees CI, which is not what happens on someone's machine.
   assert.doesNotMatch(MACHINE, /\bCI=/);
 });
+
+test('collecting results leaves out .git, node_modules and links, and keeps every plain file', { skip: process.platform === 'win32' && 'a link needs privileges to make on Windows' }, async () => {
+  const { symlinkSync, existsSync } = await import('node:fs');
+  const from = tempDir(), to = join(tempDir(), 'out');
+  mkdirSync(join(from, 'run-1', 'project', '.git'), { recursive: true });
+  mkdirSync(join(from, 'run-1', 'project', 'node_modules', 'x'), { recursive: true });
+  writeFileSync(join(from, 'run-1', 'project', '.git', 'config'), ['[diff]', '  external = touch /tmp/pwned', ''].join('\n'));
+  writeFileSync(join(from, 'run-1', 'project', 'node_modules', 'x', 'index.js'), '');
+  writeFileSync(join(from, 'run-1', 'grading.json'), '{}');
+  writeFileSync(join(from, 'run-1', 'project', 'package.json'), '{}');
+  symlinkSync('/etc/passwd', join(from, 'run-1', 'outside'));
+  const r = spawnSync(process.execPath, [join(EVALS, 'collect.mjs'), from, to], { encoding: 'utf8' });
+  assert.equal(r.status, 0, r.stderr);
+  assert.ok(existsSync(join(to, 'run-1', 'grading.json')) && existsSync(join(to, 'run-1', 'project', 'package.json')), 'the positive control: plain files arrive');
+  for (const gone of ['run-1/project/.git', 'run-1/project/node_modules', 'run-1/outside']) assert.ok(!existsSync(join(to, gone)), gone);
+});
diff --git a/test/evals-transcripts.test.mjs b/test/evals-transcripts.test.mjs
new file mode 100644
index 0000000..85cf0eb
--- /dev/null
+++ b/test/evals-transcripts.test.mjs
@@ -0,0 +1,68 @@
+// The grader reads an agent's commands and last message from its transcript, and each agent writes a different one.
+// These hold the shapes each documents, so a run on OpenCode is graded on what it did, not on a parse that found nothing.
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { after, test } from 'node:test';
+import { parseTranscript } from '../evals/transcripts.mjs';
+
+const dirs = [];
+after(() => dirs.forEach((d) => rmSync(d, { recursive: true, force: true })));
+const transcript = (events) => {
+  const dir = mkdtempSync(join(tmpdir(), 'ng-hop-transcript-'));
+  dirs.push(dir);
+  writeFileSync(join(dir, 'transcript.jsonl'), `${events.map((e) => JSON.stringify(e)).join('\n')}\nnot json at all\n`);
+  return dir;
+};
+
+// `opencode run --format json`: { type, timestamp, sessionID, part } a line.
+const oc = (type, part) => ({ type, timestamp: 1, sessionID: 'ses_1', part });
+const tool = (name, input) => oc('tool_use', { type: 'tool', tool: name, state: { status: 'completed', input, output: '' } });
+
+test('an OpenCode run: its commands, the skill it loaded, its last message and its cost', () => {
+  const p = parseTranscript(transcript([
+    oc('step_start', { type: 'step-start' }),
+    oc('text', { type: 'text', text: 'Looking at the project first.' }),
+    tool('skill', { name: 'ng-hop' }),
+    tool('bash', { command: 'node plan.mjs --to 22', description: 'Plan' }),
+    oc('step_finish', { type: 'step-finish', cost: 0.01, tokens: { input: 100, output: 20, reasoning: 0, cache: { read: 5, write: 0 } } }),
+    oc('step_start', { type: 'step-start' }),
+    tool('bash', { command: 'npx --no-install ng update @angular/core@21 @angular/cli@21' }),
+    oc('text', { type: 'text', text: 'The app is on Angular 21.' }),
+    oc('text', { type: 'text', text: 'Angular 22 is blocked by Nebular.' }),
+    oc('step_finish', { type: 'step-finish', cost: 0.02, tokens: { input: 50, output: 10 } }),
+  ]));
+  assert.deepEqual(p.commands, ['node plan.mjs --to 22', 'npx --no-install ng update @angular/core@21 @angular/cli@21']);
+  assert.deepEqual(p.skills, ['ng-hop']);
+  assert.equal(p.finalText, 'The app is on Angular 21.\n\nAngular 22 is blocked by Nebular.', 'everything after its last tool call, not the earlier narration');
+  assert.equal(p.result.num_turns, 2);
+  assert.ok(Math.abs(p.result.total_cost_usd - 0.03) < 1e-9);
+  assert.equal(p.result.usage.input_tokens, 185);
+  assert.equal(p.tools.bash, 2);
+});
+
+test('an OpenCode run that read the skill\'s file instead of calling the skill tool still loaded it', () => {
+  const p = parseTranscript(transcript([tool('read', { filePath: '/home/evals/.config/opencode/skills/ng-hop/SKILL.md' }), oc('text', { type: 'text', text: 'Done.' })]));
+  assert.deepEqual(p.skills, ['ng-hop']);
+});
+
+test('an OpenCode error is recorded, and a run with no skill has none', () => {
+  const p = parseTranscript(transcript([oc('step_start', { type: 'step-start' }), { type: 'error', timestamp: 1, sessionID: 'ses_1', error: { name: 'APIError', data: { message: 'rate limited' } } }]));
+  assert.equal(p.result.is_error, true);
+  assert.equal(p.result.error, 'rate limited');
+  assert.deepEqual(p.skills, []);
+  assert.deepEqual(p.commands, []);
+});
+
+test('a Claude Code run still reads as before', () => {
+  const p = parseTranscript(transcript([
+    { type: 'assistant', message: { content: [{ type: 'tool_use', name: 'Skill', input: { skill: 'ng-hop:ng-hop' } }] } },
+    { type: 'assistant', message: { content: [{ type: 'tool_use', name: 'Bash', input: { command: 'git status --short' } }, { type: 'text', text: 'Done.' }] } },
+    { type: 'result', result: 'The app is on Angular 22.', total_cost_usd: 0.5, num_turns: 3 },
+  ]));
+  assert.deepEqual(p.commands, ['git status --short']);
+  assert.deepEqual(p.skills, ['ng-hop:ng-hop']);
+  assert.equal(p.finalText, 'The app is on Angular 22.');
+  assert.equal(p.result.total_cost_usd, 0.5);
+});

From 5089740fcacec6beb40c6302b6cc8a577dd791f4 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 20:38:37 -0600
Subject: [PATCH 03/11] fix(ci): set the OpenCode job's paths from a step

GitHub refused the workflow: a job's own env can't read the runner context.
---
 .github/workflows/evals.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index d7e87ed..18ff25f 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -236,9 +236,7 @@ jobs:
         os: [ubuntu-24.04, windows-latest]
         case: ${{ fromJSON(needs.prepare.outputs.cases) }}
     env:
-      NG_HOP_WORKSPACE: ${{ runner.temp }}/ws
       NG_CLI_ANALYTICS: 'false'
-      CHROME_BIN: ${{ runner.os == 'Linux' && '/usr/bin/google-chrome' || '' }}
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
         with:
@@ -259,6 +257,9 @@ jobs:
           fi
           git config --global user.name "Fixture for evals"
           git config --global user.email "fixture@example.invalid"
+          # A job's own env can't read the runner's paths, so these come from here.
+          echo "NG_HOP_WORKSPACE=$RUNNER_TEMP/ws" >> "$GITHUB_ENV"
+          if [ "$RUNNER_OS" = Linux ]; then echo "CHROME_BIN=/usr/bin/google-chrome" >> "$GITHUB_ENV"; fi
       - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
         with:
           name: templates

From c0cfb6f71b2af7a2c16edd5b98486b84367e6988 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 20:42:32 -0600
Subject: [PATCH 04/11] test(evals): agent cases on yarn and bun

The same app as upgrade-pnpm, on yarn 4 and on bun, graded the same way, plus a check that it stayed on its package manager with no second lockfile. The evals machine installs pinned yarn and bun beside pnpm.
---
 .github/actions/eval-machine/action.yml |  4 ++--
 evals/cases.mjs                         | 27 +++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/.github/actions/eval-machine/action.yml b/.github/actions/eval-machine/action.yml
index 7f293e3..bf325be 100644
--- a/.github/actions/eval-machine/action.yml
+++ b/.github/actions/eval-machine/action.yml
@@ -1,5 +1,5 @@
 name: Eval machine
-description: Node, pnpm, Claude Code, and the user with no sudo that everything untrusted runs as.
+description: Node, pnpm, yarn, bun, Claude Code, and the user with no sudo that everything untrusted runs as.
 
 runs:
   using: composite
@@ -12,7 +12,7 @@ runs:
       shell: bash
       run: |
         # Pinned so a batch's cases all ran on the same tools. Bump them only with a rerun of every case.
-        npm install -g pnpm@12.16.1 @anthropic-ai/claude-code@2.1.319
+        npm install -g pnpm@12.16.1 @yarnpkg/cli-dist@4.18.1 bun@1.4.4 @anthropic-ai/claude-code@2.1.319
 
         # Ubuntu 24.04 blocks the user namespaces that Chrome's sandbox needs.
         sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
diff --git a/evals/cases.mjs b/evals/cases.mjs
index 57caeb6..c3c6f17 100644
--- a/evals/cases.mjs
+++ b/evals/cases.mjs
@@ -100,6 +100,14 @@ export const check = {
     text: 'Stayed on pnpm, with no npm lockfile',
     run: (ctx) => [ctx.exists('pnpm-lock.yaml') && !ctx.exists('package-lock.json'), `pnpm-lock.yaml: ${ctx.exists('pnpm-lock.yaml')}, package-lock.json: ${ctx.exists('package-lock.json')}`],
   }),
+  stayedOn: (manager) => ({
+    text: `Stayed on ${manager}, with no other lockfile`,
+    run: (ctx) => {
+      const lockfiles = { npm: 'package-lock.json', pnpm: 'pnpm-lock.yaml', yarn: 'yarn.lock', bun: 'bun.lock' };
+      const present = Object.entries(lockfiles).filter(([, file]) => ctx.exists(file)).map(([name]) => name);
+      return [present.length === 1 && present[0] === manager, `Lockfiles: ${present.join(', ') || 'none'}`];
+    },
+  }),
   lockfileClean: () => ({
     text: "The lockfile has no leftovers from the CLI's temporary install",
     run: (ctx) => { const m = /\S*angular-cli-tmp-packages\S*/.exec(ctx.read('pnpm-lock.yaml') || ''); return [!m, m ? `Found ${m[0]}` : 'None']; },
@@ -406,6 +414,25 @@ should I just use --force?`,
       check.installs(), check.builds(), check.testsPass(), check.appRuns(),
     ],
   },
+  {
+    // The same app on yarn 4, where a plain `ng update` stops at installing Angular's temporary CLI, with no error.
+    name: 'upgrade-yarn', tier: 'upgrade', fixture: 'yarn-clean',
+    prompt: 'Upgrade this app to Angular 22, along with its libraries.',
+    assertions: [
+      check.skillFired(), check.reachesMajor(22), check.commitPerMajor([21, 22]), check.cleanTree(), check.notOnMain(), check.noForcing(), check.noOverrides(),
+      check.noDirtyUpdate(), check.noOptionalMigrations(), check.specsUntouched(), check.ranApp(), check.stayedOn('yarn'),
+      check.installs(), check.builds(), check.testsPass(), check.appRuns(),
+    ],
+  },
+  {
+    name: 'upgrade-bun', tier: 'upgrade', fixture: 'bun-clean',
+    prompt: 'Upgrade this app to Angular 22, along with its libraries.',
+    assertions: [
+      check.skillFired(), check.reachesMajor(22), check.commitPerMajor([21, 22]), check.cleanTree(), check.notOnMain(), check.noForcing(), check.noOverrides(),
+      check.noDirtyUpdate(), check.noOptionalMigrations(), check.specsUntouched(), check.ranApp(), check.stayedOn('bun'),
+      check.installs(), check.builds(), check.testsPass(), check.appRuns(),
+    ],
+  },
   {
     // Every install exits 1 until the scripts are decided. Declining is safe, and approving is the person's call.
     name: 'upgrade-pnpm-undecided-scripts', tier: 'upgrade', fixture: 'pnpm-undecided', noInstall: true,

From 66f5d6c5e01e26b08406dd3913dfb5e2f32f67c8 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 20:52:44 -0600
Subject: [PATCH 05/11] fix(ci): run the OpenCode job after a skipped canary

Its condition implied that every job before it had succeeded, and the canary is skipped on purpose for OpenCode.
---
 .github/workflows/evals.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index 18ff25f..ba5a247 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -226,7 +226,8 @@ jobs:
   opencode:
     name: OpenCode ${{ matrix.case }} on ${{ matrix.os }}
     needs: prepare
-    if: inputs.agent == 'opencode'
+    # The canary is skipped for OpenCode, and a plain condition would count that against every job after it.
+    if: ${{ !failure() && !cancelled() && inputs.agent == 'opencode' }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
     strategy:

From 313cae9eb366edf5a58af61e447fb4ff1798963a Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 21:22:52 -0600
Subject: [PATCH 06/11] fix(evals): give a real app's folder its repository's
 ignore rules

pr-sweep's renderer, copied out of its repository, lost the root .gitignore, so the fixture committed all of node_modules at the start and every install dirtied the tree. The agent rightly stopped and asked instead of committing millions of lines.
---
 evals/prepare.mjs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/evals/prepare.mjs b/evals/prepare.mjs
index b954dcd..ce8cb56 100644
--- a/evals/prepare.mjs
+++ b/evals/prepare.mjs
@@ -44,6 +44,9 @@ for (const name of names) {
       run(`git clone --quiet --filter=blob:none --no-checkout ${fixture.repo} "${clone}"`, TEMPLATES);
       run(`git -c advice.detachedHead=false checkout --quiet ${fixture.ref}`, clone);
       cpSync(join(clone, fixture.dir || '.'), dir, { recursive: true, filter: (src) => !/[\\/]\.git$/.test(src) });
+      // A folder taken out of its repository loses the ignore rules above it, and pr-sweep's renderer then committed
+      // all of node_modules at the start. The repository's own rules come along.
+      if (fixture.dir && existsSync(join(clone, '.gitignore'))) appendFileSync(join(dir, '.gitignore'), `\n${readFileSync(join(clone, '.gitignore'), 'utf8')}`);
       rmSync(clone, { recursive: true, force: true });
       run('npm ci --no-audit --no-fund', dir);
       // Its build and tests are whatever they are. The agent compares against them, and so does whoever reads the result.

From 33ed7dcb3b46e3814240b40cb6dae062429b6e54 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 22:25:02 -0600
Subject: [PATCH 07/11] ci(evals): run OpenCode on the owner's machine, not in
 CI

OpenCode's free models refused GitHub's runners with FreeUsageLimitError and a 21-hour wait, and a paid key would sit in OpenCode's environment, where an install script can read it. The job, its agent input and its results collector go. The harness still runs OpenCode locally with --agent opencode.
---
 .github/workflows/evals.yml | 84 ++-----------------------------------
 evals/collect.mjs           | 23 ----------
 test/evals-login.test.mjs   | 20 +--------
 3 files changed, 4 insertions(+), 123 deletions(-)
 delete mode 100644 evals/collect.mjs

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index ba5a247..5526dc0 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -1,5 +1,5 @@
 name: Evals
-run-name: "Evals: ${{ inputs.cases || (github.event_name == 'schedule' && 'the weekly questions') || 'every case' }} on ${{ inputs.agent == 'opencode' && 'OpenCode' || inputs.model || 'claude-opus-5-5' }}"
+run-name: "Evals: ${{ inputs.cases || (github.event_name == 'schedule' && 'the weekly questions') || 'every case' }} on ${{ inputs.model || 'claude-opus-5-5' }}"
 
 # Runs the eval cases on a batch of GitHub's machines instead of one person's. Evals cost usage, so this never runs on
 # its own. They also install real npm packages, so the login reaches only Claude Code: see .github/actions/eval-machine
@@ -28,14 +28,8 @@ on:
         default: both
         type: choice
         options: [both, with_skill, without_skill]
-      agent:
-        description: Which agent runs the cases. OpenCode uses a free model with no login, on Linux and Windows.
-        required: false
-        default: claude
-        type: choice
-        options: [claude, opencode]
       model:
-        description: Model the agent runs on (empty means claude-opus-5-5, or opencode/big-pickle for OpenCode)
+        description: Model the agent runs on (empty means claude-opus-5-5)
         required: false
         default: ''
       parallel:
@@ -65,7 +59,6 @@ concurrency:
 jobs:
   canary:
     name: Check the login is out of reach
-    if: inputs.agent != 'opencode'
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     steps:
@@ -83,8 +76,6 @@ jobs:
   prepare:
     name: Build the fixture apps
     needs: canary
-    # OpenCode needs no login, so its runs have no canary to wait for.
-    if: ${{ !failure() && !cancelled() }}
     runs-on: ubuntu-24.04
     timeout-minutes: 60
     outputs:
@@ -160,7 +151,7 @@ jobs:
   run:
     name: ${{ matrix.case }}
     needs: prepare
-    if: ${{ !inputs.triggers && inputs.agent != 'opencode' }}
+    if: ${{ !inputs.triggers }}
     runs-on: ubuntu-24.04
     timeout-minutes: 180
     strategy:
@@ -221,75 +212,6 @@ jobs:
           include-hidden-files: true
           retention-days: 14
 
-  # OpenCode on a free model holds no login and no secret, so it runs without the user switch, and on Windows too, where
-  # nothing could keep Claude Code's login from an install script. Its results still leave out .git and links.
-  opencode:
-    name: OpenCode ${{ matrix.case }} on ${{ matrix.os }}
-    needs: prepare
-    # The canary is skipped for OpenCode, and a plain condition would count that against every job after it.
-    if: ${{ !failure() && !cancelled() && inputs.agent == 'opencode' }}
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 180
-    strategy:
-      fail-fast: false
-      max-parallel: ${{ fromJSON(inputs.parallel || '4') }}
-      matrix:
-        os: [ubuntu-24.04, windows-latest]
-        case: ${{ fromJSON(needs.prepare.outputs.cases) }}
-    env:
-      NG_CLI_ANALYTICS: 'false'
-    steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
-        with:
-          node-version: '24'
-      - name: Install OpenCode and the package managers, beside the runner's own
-        shell: bash
-        run: |
-          prefix="$RUNNER_TEMP/tools"
-          npm install -g --prefix "$prefix" @opencode/cli@2.2.0 pnpm@12.16.1 @yarnpkg/cli-dist@4.18.1 bun@1.4.4
-          if [ "$RUNNER_OS" = Windows ]; then
-            echo "$prefix" >> "$GITHUB_PATH"
-            echo "OPENCODE_BIN=$prefix/node_modules/@opencode/cli/bin/opencode.exe" >> "$GITHUB_ENV"
-          else
-            echo "$prefix/bin" >> "$GITHUB_PATH"
-          fi
-          git config --global user.name "Fixture for evals"
-          git config --global user.email "fixture@example.invalid"
-          # A job's own env can't read the runner's paths, so these come from here.
-          echo "NG_HOP_WORKSPACE=$RUNNER_TEMP/ws" >> "$GITHUB_ENV"
-          if [ "$RUNNER_OS" = Linux ]; then echo "CHROME_BIN=/usr/bin/google-chrome" >> "$GITHUB_ENV"; fi
-      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
-        with:
-          name: templates
-          path: ${{ runner.temp }}/ws/templates
-      - name: Run the case
-        shell: bash
-        env:
-          CASE: ${{ matrix.case }}
-          CONFIG: ${{ inputs.config }}
-          MODEL: ${{ inputs.model }}
-          RUNS: ${{ inputs.runs || '1' }}
-        run: |
-          args=()
-          [ "$CONFIG" = both ] || args+=(--config "$CONFIG")
-          [ -z "$MODEL" ] || args+=(--model "$MODEL")
-          node evals/run.mjs --agent opencode --iteration ci --case "$CASE" --runs "$RUNS" "${args[@]}"
-      - name: Collect the results, without .git or links
-        if: always()
-        shell: bash
-        run: node evals/collect.mjs "$NG_HOP_WORKSPACE/iteration-ci" "$RUNNER_TEMP/results/iteration-ci"
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
-        if: always()
-        with:
-          name: results-opencode-${{ matrix.os }}-${{ matrix.case }}
-          path: ${{ runner.temp }}/results
-          include-hidden-files: true
-          if-no-files-found: ignore
-          retention-days: 14
-
   report:
     name: Report the scores
     needs: run
diff --git a/evals/collect.mjs b/evals/collect.mjs
deleted file mode 100644
index 898a9cb..0000000
--- a/evals/collect.mjs
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env node
-// Copies a results folder for upload, leaving out what a checkout wouldn't have and anything but plain files and
-// folders: a .git can run commands in whatever runs git on it later, and a link can point out of the folder.
-// Usage: node evals/collect.mjs <from> <to>
-import { copyFileSync, lstatSync, mkdirSync, readdirSync } from 'node:fs';
-import { join } from 'node:path';
-
-const SKIP = new Set(['node_modules', 'dist', '.angular', '.git']);
-
-function copy(from, to) {
-  mkdirSync(to, { recursive: true });
-  for (const name of readdirSync(from)) {
-    if (SKIP.has(name)) continue;
-    const source = join(from, name);
-    const stat = lstatSync(source);
-    if (stat.isDirectory()) copy(source, join(to, name));
-    else if (stat.isFile()) copyFileSync(source, join(to, name));
-  }
-}
-
-const [from, to] = process.argv.slice(2);
-if (!from || !to) throw new Error('Usage: node evals/collect.mjs <from> <to>');
-copy(from, to);
diff --git a/test/evals-login.test.mjs b/test/evals-login.test.mjs
index 41d2ab4..0bfbfd4 100644
--- a/test/evals-login.test.mjs
+++ b/test/evals-login.test.mjs
@@ -195,9 +195,7 @@ test('evals run when someone starts them or on the weekly schedule, which covers
 
   const jobs = WORKFLOW.slice(WORKFLOW.indexOf('\njobs:'));
   const job = (name) => new RegExp(`^  ${name}:\\n[\\s\\S]*?(?=^  \\w+:\\n|$(?![\\s\\S]))`, 'm').exec(jobs)[0];
-  // OpenCode's job runs third-party code with no user switch, which is only safe because nothing secret is near it.
-  for (const name of ['prepare', 'report', 'opencode']) assert.doesNotMatch(job(name), /secrets\./, name);
-  assert.match(job('opencode'), /node evals\/collect\.mjs/, 'its results leave out .git and links too');
+  for (const name of ['prepare', 'report']) assert.doesNotMatch(job(name), /secrets\./, name);
   for (const name of ['canary', 'run', 'triggers']) assert.match(job(name), /secrets\.CLAUDE_CODE_OAUTH_TOKEN/, name);
   // Agent output isn't masked the way the secret is, and this job has no collect step to check it.
   assert.doesNotMatch(job('triggers'), /upload-artifact/);
@@ -248,19 +246,3 @@ test('agents run as a user with no sudo, in an environment with nothing of the r
   // pnpm refuses to change a lockfile when it sees CI, which is not what happens on someone's machine.
   assert.doesNotMatch(MACHINE, /\bCI=/);
 });
-
-test('collecting results leaves out .git, node_modules and links, and keeps every plain file', { skip: process.platform === 'win32' && 'a link needs privileges to make on Windows' }, async () => {
-  const { symlinkSync, existsSync } = await import('node:fs');
-  const from = tempDir(), to = join(tempDir(), 'out');
-  mkdirSync(join(from, 'run-1', 'project', '.git'), { recursive: true });
-  mkdirSync(join(from, 'run-1', 'project', 'node_modules', 'x'), { recursive: true });
-  writeFileSync(join(from, 'run-1', 'project', '.git', 'config'), ['[diff]', '  external = touch /tmp/pwned', ''].join('\n'));
-  writeFileSync(join(from, 'run-1', 'project', 'node_modules', 'x', 'index.js'), '');
-  writeFileSync(join(from, 'run-1', 'grading.json'), '{}');
-  writeFileSync(join(from, 'run-1', 'project', 'package.json'), '{}');
-  symlinkSync('/etc/passwd', join(from, 'run-1', 'outside'));
-  const r = spawnSync(process.execPath, [join(EVALS, 'collect.mjs'), from, to], { encoding: 'utf8' });
-  assert.equal(r.status, 0, r.stderr);
-  assert.ok(existsSync(join(to, 'run-1', 'grading.json')) && existsSync(join(to, 'run-1', 'project', 'package.json')), 'the positive control: plain files arrive');
-  for (const gone of ['run-1/project/.git', 'run-1/project/node_modules', 'run-1/outside']) assert.ok(!existsSync(join(to, gone)), gone);
-});

From 8e111bac1fdcd75cfdded7324dfba0db3b5979bd Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 22:25:13 -0600
Subject: [PATCH 08/11] docs(roadmap): tick scheduled evals, Windows, catalogs,
 yarn and bun

Records why scripted upgrades cover Windows, yarn, bun and catalogs, and why OpenCode and Claude Code on Windows run on the owner's machine.
---
 ROADMAP.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/ROADMAP.md b/ROADMAP.md
index 25582e3..1874b51 100644
--- a/ROADMAP.md
+++ b/ROADMAP.md
@@ -51,6 +51,8 @@ Everything else gets ordinary care, but none of these three gets traded away for
 - **Read Chrome over the DevTools protocol, through a pipe (2026-11-07).** *Why:* Chrome's log doesn't record a message's level, so a plain `console.error('oops')` passed and the word ERROR in an ordinary log line failed. `--remote-debugging-pipe` gives the protocol over two file descriptors, so it needs no WebSocket, which Node 18 lacks, and no library. It now waits for the root to fill instead of a fixed virtual-time budget. *Gave up:* about two seconds more per check, and fifteen when an app never renders.
 - **Move a pnpm catalog in the pnpm runner, around Angular's own update (2026-11-07).** `update.mjs` copies the catalog's versions into `package.json`, runs `ng update` as usual, writes the new versions back into the catalog, puts `package.json` back on `catalog:` and reinstalls. *Why:* Angular's CLI can't change a catalog, and `pnpm update` can't either: on pnpm 12.16.1 it replaced `catalog:` in `package.json` with a plain version. Plain Claude moved a catalog by hand and passed. Going around `ng update` instead of replacing it keeps its peer checks and migrations. *Gave up:* it edits `pnpm-workspace.yaml` as text, so it reads block-style catalogs only, and refuses anything else before changing a thing. A catalog other projects share moves them too, so the plan names them and the skill asks.
 - **Run the question cases weekly, with the skill only (2026-11-07).** *Why:* the cases read the live registry, so the right answer changes when a library catches up, and nobody would notice until a full batch. A failed check with the skill fails the run, and GitHub emails whoever last changed the schedule. It costs about $1.50 of usage a week. *Gave up:* the upgrade cases and the baseline stay manual, since a full batch costs about $32.
+- **Scripted upgrades with no agent cover Windows, yarn, bun and pnpm catalogs (2026-11-07).** `evals/hop.mjs` takes a fixture through the plan's own commands and checks the build, tests and running app after every major. It needs no login, so it runs on every change to the scripts and weekly. *Why:* it found five real problems in its first day, among them pnpm 12's native binary on Windows and yarn 4 stalling while installing Angular's temporary CLI, none of which the Linux agent evals could see. *Gave up:* it follows the plan's commands rather than an agent's judgement, so it tests the scripts, not the skill's instructions.
+- **OpenCode and Claude Code on Windows run on the owner's machine, not in CI (2026-11-07).** *Why:* OpenCode's free models refused GitHub's runners with `FreeUsageLimitError` and a 21-hour wait. A paid key, like Claude Code's login on Windows, would sit where an install script can read it: OpenCode only takes keys from its environment or config, and Windows lets a process read the memory of another of the same user, with nothing like Linux's Yama to stop it. *Gave up:* those runs need the owner's say-so and a machine with memory to spare. The harness still runs OpenCode, with `--agent opencode`.
 
 ## Shipped
 
@@ -67,8 +69,8 @@ The evals use small apps built for them, run on Linux, and only when someone sta
 - [ ] Take my own apps on old majors (`readme-wizard`, `pr-sweep`) to the latest Angular with the skill, and fix what that finds.
 - [ ] Take a whole upgrade through OpenCode, and update the README's agent table with what happened.
 - [ ] Answer every upgrade problem someone files, with the plan's output and a fix or a reason.
-- [ ] Evals on a schedule, so a library catching up shows as a changed result. A full batch costs about $32 of usage, so the weekly run covers the question cases, with the skill only.
-- [ ] Evals on Windows. Only `npm test` runs there today, so a problem only Windows has shows up nowhere else.
+- [x] Evals on a schedule, so a library catching up shows as a changed result. A full batch costs about $32 of usage, so the weekly run covers the question cases, with the skill only.
+- [x] Evals on Windows. Only `npm test` runs there today, so a problem only Windows has shows up nowhere else. The plan's real commands now run there on every change to the scripts. An agent on Windows runs on the owner's machine only.
 
 **Done when:** both apps are on the latest Angular through the skill, the README's agent table says what OpenCode really did, a scheduled run has reported, and real upgrades have run on Windows.
 
@@ -76,8 +78,8 @@ The evals use small apps built for them, run on Linux, and only when someone sta
 
 Each of these is a place where the skill stops or says less than it could today.
 
-- [ ] Take a pnpm catalog through an upgrade instead of refusing it. Angular 22's CLI lists the steps: edit the catalog, `pnpm install`, then `ng update <package> --migrate-only --from <old version>`. Plain Claude did this and passed.
-- [ ] Real yarn and bun fixtures. The plan prints their commands, but no real upgrade has run on either.
+- [x] Take a pnpm catalog through an upgrade instead of refusing it. Angular 22's CLI lists the steps: edit the catalog, `pnpm install`, then `ng update <package> --migrate-only --from <old version>`. Plain Claude did this and passed.
+- [x] Real yarn and bun fixtures. The plan prints their commands, but no real upgrade has run on either. Both now pass from Angular 20 to 22.
 - [x] Say when a framework package the app uses is deprecated. `@angular/animations` is, from Angular 20.2 on, in favour of `animate.enter` and `animate.leave`.
 - [x] Check the licenses of what a moved library brings with it: shared packages the plan moves, and new dependencies, such as `@primeui/license-manager` with PrimeNG 22.
 - [x] A `--next` mode, for trying a prerelease Angular. Angular's `next` tag can be a new major or a minor of the current one, so it has to handle both.

From f39a413c796355848ca0644c1043da787a63b198 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 23:48:12 -0600
Subject: [PATCH 09/11] fix(evals): start OpenCode runs in the project, with
 OpenCode 2's config names

opencode run starts its session in $PWD when that's set, not in its working folder, so the last local run began in the folder the harness was launched from and had to move itself to the project. The config now uses OpenCode 2's permissions list and update setting instead of the deprecated 1.x names, and the harness stops on any other OpenCode version, where the denied commands might not apply.
---
 evals/run.mjs             | 20 +++++++++++++++-----
 test/evals-login.test.mjs | 10 ++++++++++
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/evals/run.mjs b/evals/run.mjs
index dea51e6..4520263 100644
--- a/evals/run.mjs
+++ b/evals/run.mjs
@@ -4,7 +4,7 @@
 // A run already finished is skipped, so an interrupted batch picks up where it stopped.
 // Usage: node evals/run.mjs --iteration 1 [--case name[,name]] [--tier plan|upgrade|real] [--config with_skill|without_skill]
 //        [--runs 1] [--agent claude|opencode] [--model id] [--max-seconds 560]
-import { execSync, spawn } from 'node:child_process';
+import { execFileSync, execSync, spawn } from 'node:child_process';
 import { copyFileSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { freemem, tmpdir } from 'node:os';
 import { join } from 'node:path';
@@ -21,6 +21,12 @@ const { values } = parseArgs({ options: {
 } });
 if (!['claude', 'opencode'].includes(values.agent)) throw new Error(`--agent takes claude or opencode, not ${values.agent}`);
 const model = values.model || (values.agent === 'opencode' ? 'opencode/big-pickle' : 'claude-opus-5-5');
+const OPENCODE = process.env.OPENCODE_BIN || 'opencode';
+if (values.agent === 'opencode' && !values.regrade) {
+  // The config runOpenCode writes uses OpenCode 2's names. On another version its denied commands might not apply, and --auto would run them.
+  const version = execFileSync(OPENCODE, ['--version'], { encoding: 'utf8' }).trim();
+  if (!/^(opencode )?v?2\./.test(version)) throw new Error(`The OpenCode runs need OpenCode 2, not ${version}`);
+}
 const started = Date.now();
 const maxMs = values['max-seconds'] ? Number(values['max-seconds']) * 1000 : Infinity;
 const minFree = Number(values['min-free-gb']) * 1024 ** 3;
@@ -80,21 +86,25 @@ function runClaude(c, config, projectDir, runDir) {
 
 // OpenCode on a free model needs no login, so there's nothing for an install script to take, and it runs where Claude
 // Code can't safely, such as on Windows. Each run gets a home of its own, so nothing from the machine's own setup or
-// another run reaches it, with sharing off so no session leaves as a public link.
+// another run reaches it, with sharing off so no session leaves as a public link. It's denied what a Claude run is, as
+// OpenCode 2's shell rules: a pattern ending in ` *` also matches the bare command.
+const OPENCODE_DENIED = ['git push *', 'gh *', 'nvm *', 'npm install -g *', 'npm i -g *'];
 function runOpenCode(c, config, projectDir, runDir) {
   const home = mkdtempSync(join(tmpdir(), 'ng-hop-opencode-'));
   const xdg = Object.fromEntries(['CONFIG', 'DATA', 'STATE', 'CACHE'].map((k) => [`XDG_${k}_HOME`, join(home, k.toLowerCase())]));
   const settings = join(xdg.XDG_CONFIG_HOME, 'opencode');
   mkdirSync(settings, { recursive: true });
   writeFileSync(join(settings, 'opencode.json'), JSON.stringify({
-    $schema: 'https://opencode.ai/config.json', share: 'disabled', autoupdate: false,
-    permission: { bash: { '*': 'allow', 'git push*': 'deny', 'gh *': 'deny', 'nvm *': 'deny', 'npm install -g*': 'deny', 'npm i -g*': 'deny' } },
+    $schema: 'https://opencode.ai/config.json', share: 'disabled', update: 'disable',
+    permissions: OPENCODE_DENIED.map((resource) => ({ action: 'shell', resource, effect: 'deny' })),
   }, null, 2));
   if (config === 'with_skill') cpSync(join(PLUGIN, 'skills', 'ng-hop'), join(settings, 'skills', 'ng-hop'), { recursive: true });
   const args = ['run', '--standalone', '--auto', '--format', 'json', '--model', model, c.prompt];
   return new Promise((resolve) => {
     const out = [];
-    const child = spawn(process.env.OPENCODE_BIN || 'opencode', args, { cwd: projectDir, env: developerEnv({ ...xdg, NG_CLI_ANALYTICS: 'false' }), stdio: ['ignore', 'pipe', 'pipe'] });
+    // `opencode run` starts its session in $PWD when that's set, not in its working folder, so both name the project.
+    const env = developerEnv({ ...xdg, PWD: projectDir, NG_CLI_ANALYTICS: 'false' });
+    const child = spawn(OPENCODE, args, { cwd: projectDir, env, stdio: ['ignore', 'pipe', 'pipe'] });
     let stderr = '';
     child.stdout.on('data', (d) => out.push(d));
     child.stderr.on('data', (d) => { stderr += d; });
diff --git a/test/evals-login.test.mjs b/test/evals-login.test.mjs
index 0bfbfd4..ff66219 100644
--- a/test/evals-login.test.mjs
+++ b/test/evals-login.test.mjs
@@ -92,6 +92,16 @@ test('every harness script starts the agent through the one place that handles t
   }
 });
 
+test('an OpenCode run is denied what a Claude Code run is, in the names OpenCode 2 reads', () => {
+  const source = readFileSync(join(EVALS, 'run.mjs'), 'utf8');
+  const claude = [...source.matchAll(/'Bash\(([^)]+)\)'/g)].map((m) => m[1].replace(/ ?\*$/, ''));
+  const opencode = [.../const OPENCODE_DENIED = \[([^\]]*)\]/.exec(source)[1].matchAll(/'([^']+)'/g)].map((m) => m[1].replace(/ \*$/, ''));
+  assert.deepEqual(opencode, claude);
+  assert.match(source, /permissions: OPENCODE_DENIED\.map\(\(resource\) => \(\{ action: 'shell', resource, effect: 'deny' \}\)\)/);
+  // 2.2 still translates the 1.x names, but they're deprecated: once they're dropped, --auto would allow every command.
+  assert.doesNotMatch(source, /\bpermission: \{|\bautoupdate\b/);
+});
+
 test('a login on stdin leaves out any login in the environment too', { skip: process.platform === 'win32' }, () => {
   const result = runAgent(cleanEnv({ NG_HOP_LOGIN_ON_STDIN: '1', ANTHROPIC_API_KEY: TOKEN, CLAUDE_CODE_OAUTH_TOKEN: 'sk-ant-oat01-another-fake-token', PATH: fakeClaude('') }), TOKEN);
   const seen = JSON.parse(result.stdout);

From 421b974e7d698145115ecd05331793f0b251c844 Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 23:48:13 -0600
Subject: [PATCH 10/11] fix(evals): read OpenCode 2's shell, skill and Code
 Mode calls

OpenCode 2 names its shell tool shell and loads a skill by id, so the grader saw no commands and no skill: the forcing checks passed on nothing and the app check failed. Code Mode can call the other tools, so its code is checked like a command.
---
 evals/transcripts.mjs           |  7 +++++--
 test/evals-transcripts.test.mjs | 13 +++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/evals/transcripts.mjs b/evals/transcripts.mjs
index ecbf3b1..9f13cf6 100644
--- a/evals/transcripts.mjs
+++ b/evals/transcripts.mjs
@@ -41,8 +41,11 @@ export function parseOpenCode(events) {
       after = [];
       const input = part.state?.input || {};
       tools[part.tool] = (tools[part.tool] || 0) + 1;
-      if (part.tool === 'bash' && input.command) commands.push(input.command);
-      if (part.tool === 'skill' && input.name) skills.push(input.name);
+      // OpenCode 2 names its shell tool `shell` and loads a skill by `id`; 1.x said `bash` and `name`.
+      if ((part.tool === 'shell' || part.tool === 'bash') && input.command) commands.push(input.command);
+      // Code Mode can call the other tools, so its code is checked like a command.
+      if (part.tool === 'execute' && input.code) commands.push(input.code);
+      if (part.tool === 'skill' && (input.id || input.name)) skills.push(input.id || input.name);
       // Reading the skill's file by hand loads it just as well.
       if (part.tool === 'read' && /[\\/]ng-hop[\\/]SKILL\.md$/i.test(input.filePath || input.path || '')) skills.push('ng-hop');
     }
diff --git a/test/evals-transcripts.test.mjs b/test/evals-transcripts.test.mjs
index 85cf0eb..3b73e85 100644
--- a/test/evals-transcripts.test.mjs
+++ b/test/evals-transcripts.test.mjs
@@ -42,6 +42,19 @@ test('an OpenCode run: its commands, the skill it loaded, its last message and i
   assert.equal(p.tools.bash, 2);
 });
 
+test('an OpenCode 2 run: its shell commands, a skill loaded by id, and code that can call the other tools', () => {
+  const code = 'return await tools.opencode.session_move({ directory: "/work/app" });';
+  const p = parseTranscript(transcript([
+    tool('skill', { id: 'ng-hop' }),
+    tool('shell', { command: 'node plan.mjs --to 22', workdir: '/work/app' }),
+    tool('execute', { code }),
+    oc('text', { type: 'text', text: 'Done.' }),
+  ]));
+  assert.deepEqual(p.skills, ['ng-hop']);
+  assert.deepEqual(p.commands, ['node plan.mjs --to 22', code]);
+  assert.equal(p.tools.shell, 1);
+});
+
 test('an OpenCode run that read the skill\'s file instead of calling the skill tool still loaded it', () => {
   const p = parseTranscript(transcript([tool('read', { filePath: '/home/evals/.config/opencode/skills/ng-hop/SKILL.md' }), oc('text', { type: 'text', text: 'Done.' })]));
   assert.deepEqual(p.skills, ['ng-hop']);

From 3ff6eeaf3c1d7775aad779a224988521fd28dd1c Mon Sep 17 00:00:00 2001
From: Hazel Granados <hazel.granados@protonmail.com>
Date: Sat, 7 Nov 2026 23:48:14 -0600
Subject: [PATCH 11/11] fix(evals): find the Node range under a Node heading

---
 evals/cases.mjs             | 3 ++-
 test/evals-grading.test.mjs | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/evals/cases.mjs b/evals/cases.mjs
index c3c6f17..134018f 100644
--- a/evals/cases.mjs
+++ b/evals/cases.mjs
@@ -130,7 +130,8 @@ export const check = {
     },
   }),
   saysNewPeer: () => check.says('Says ng-select now needs @angular/cdk', /cdk/i),
-  saysNodeMoved: () => check.says('Says Angular 22 needs a newer Node', /node[^.\n]{0,120}(\^?22\.22|\^?24\.15|>= ?26)|(22\.22|24\.15)[^.\n]{0,120}node/i),
+  // Within a sentence, or from a "Node" heading to the line under it.
+  saysNodeMoved: () => check.says('Says Angular 22 needs a newer Node', /node(?:[^.\n]|\n(?!\n)){0,120}(\^?22\.22|\^?24\.15|>= ?26)|(22\.22|24\.15)(?:[^.\n]|\n(?!\n)){0,120}node/i),
   saysLicenseChanges: () => check.says('Tells the person PrimeNG 22 changes its license', /(primeng|primeui)[\s\S]{0,600}licen[sc]e|licen[sc]e[\s\S]{0,600}(primeng|primeui)/i),
   usedPlan: () => ({
     text: 'Used the plan script',
diff --git a/test/evals-grading.test.mjs b/test/evals-grading.test.mjs
index 98c073e..3675584 100644
--- a/test/evals-grading.test.mjs
+++ b/test/evals-grading.test.mjs
@@ -121,7 +121,9 @@ test('facts an upgrade report owes the person, in the words real answers used',
   const node = check('upgrade-clean-path', 'Says Angular 22 needs a newer Node');
   assert.ok(says(node, '**Node:** Angular 22 needs ^22.22.3, ^24.15.0 or >=26.0.0, so check your CI.'));
   assert.ok(says(node, 'Node has to be at least 22.22 or 24.15 now.'));
+  assert.ok(says(node, '## Node\nAngular 22 needs **`^22.22.3 || ^24.15.0 || >=26.0.0`**. This machine is on 24.20.0, so it\'s fine.'), 'an OpenCode answer');
   assert.ok(!says(node, 'Your Node version is fine.'));
+  assert.ok(!says(node, '## Node\n\nUnchanged.\n\n## Versions\n24.15 of the docs'), 'a heading reaches only the line under it');
 
   const wrong = check('ask-open-range', 'Does not claim 34.3.1 stops working on the newer Angular');
   assert.ok(!says(wrong, "So 34.3.1 won't work past Angular 20, and 36.3.0 is the version to plan around."));