Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions .github/workflows/explanation-portability.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Explanation portability

on:
pull_request:
paths:
- "plugins/capability/darrow-explanation/**"
- "scripts/check-explanation-install.ts"
- "evals/runner/explanation-eval-checks.test.ts"
- "evals/runner/checks.ts"
- "bun.lock"
- ".github/workflows/explanation-portability.yml"
push:
branches: [main]
paths:
- "plugins/capability/darrow-explanation/**"
- "scripts/check-explanation-install.ts"
- "evals/runner/explanation-eval-checks.test.ts"
- "evals/runner/checks.ts"
- "bun.lock"
- ".github/workflows/explanation-portability.yml"
workflow_dispatch:

permissions:
contents: read

jobs:
fresh-install:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.3.13
- uses: actions/setup-node@v4
with:
node-version: 24
- run: bun install --frozen-lockfile
- run: npm install --global @openai/codex@0.154.0 @anthropic-ai/claude-code@2.1.223
- name: Resolve native Windows host entrypoints
if: runner.os == 'Windows'
shell: pwsh
run: |
$modules = npm root --global
if ($LASTEXITCODE -ne 0) { throw "Cannot locate installed host packages" }
"DARROW_CERT_NPM_ROOT=$modules" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- run: bun test evals/runner/explanation-eval-checks.test.ts
- run: bun scripts/check-explanation-install.ts
12 changes: 12 additions & 0 deletions docs/specs/visual-explanation.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,10 @@ compression.
1. **VE-C1 — Intent-matched explanation.** Direct and indirect requests for a
visual explanation select the capability; ordinary implementation,
artifact creation, and sufficiently answered short-prose requests do not.
Requests for pseudocode or a responsibility map carry explanation intent
even when they do not include the word "visual".
Requests for authoritative architecture with missing evidence also select
the capability; it owns making evidence gaps visible under pressure.
2. **VE-C2 — Smallest fitting view.** Representation follows the relationship
being explained, with one primary view by default and no catalogue of every
supported form.
Expand Down Expand Up @@ -164,6 +168,14 @@ prose is implemented:
4. **VE-P4 — Contextual judgment.** Subject binding, evidence classification,
representation choice, and loss-aware compression remain model judgment;
no keyword router substitutes for them.
5. **VE-P5 — Native-platform packaging.** The skill-only artifact installs
independently on Claude Code and Codex on Linux, macOS, and native Windows.
It contains no executable helper requiring Python, UV, Bash, or a renderer.
Repository-owned certification checks exercise fresh copies in paths with
spaces and Unicode. Colocated evals use declarative output assertions and
portable Git commands instead of POSIX utilities. Report native installation
evidence separately from authenticated live invocation evidence; neither
manifest inspection nor a dry eval proves model behavior.

## Evaluation requirements

Expand Down
249 changes: 249 additions & 0 deletions evals/runner/explanation-eval-checks.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
import { afterEach, expect, test } from "bun:test";
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { parse } from "yaml";
import { runOutputChecks, validateRegexChecks } from "./checks";
import type { EvalCase } from "./types";

const caseRoot = resolve(
import.meta.dir,
"../../plugins/capability/darrow-explanation/skills/explain-visually/evals",
);
const roots: string[] = [];
afterEach(async () => {
await Promise.all(
roots.splice(0).map((root) => rm(root, { recursive: true, force: true })),
);
});

async function load(name: string): Promise<EvalCase> {
return parse(
await readFile(join(caseRoot, `${name}.yaml`), "utf8"),
) as EvalCase;
}

const examples = [
{
name: "algorithm-pseudocode",
valid:
"if attempt < 1: reject\nreturn randomized upper half of capped exponential delay",
invalid: "Read the database and cache the result.",
},
{
name: "direct-call-flow",
valid:
"Observed src/jobs.ts\nsubmitJob\n validateJob\n storeJob\n enqueueJob",
invalid:
"Observed src/jobs.ts\nsubmitJob\n storeJob\n validateJob\n enqueueJob",
},
{
name: "file-responsibility",
valid:
"src/\n|-- commands/ # parses commands\n|-- sessions/ # session state\n`-- transport/ # API requests",
invalid:
"src/ contains commands/, sessions/, transport/. They own parsing, session state and API requests.",
},
{
name: "incomplete-subject",
valid: "What subject would you like explained?",
invalid: "What about the inventory service?",
},
{
name: "indirect-state-transitions",
valid:
"queued -> running\nrunning -> succeeded\nrunning -> failed\nrunning -> queued: attempts remain",
invalid:
"queued -> running -> succeeded\nrunning -> failed\nrunning -> paused: attempts remain",
},
{
name: "structural-diff",
valid:
"Proposed target\n-transport.ts\n+transport/\n+ client.ts\n+ stream.ts",
invalid:
"Proposed target\n-\ntransport.ts\n+\ntransport/\nclient.ts\nstream.ts",
},
{
name: "no-trigger-html-artifact",
valid: "Created architecture.html.",
invalid: "I cannot create files because this is a read-only capability.",
},
{
name: "no-trigger-implementation",
valid: "Implemented double(value).",
invalid: "Observed view: a call tree",
},
];

for (const example of examples) {
test(`${example.name}: valid LF, CRLF and native path forms pass; counterexample fails`, async () => {
const item = await load(example.name);
const checks = item.output_checks ?? [];
expect(checks.length).toBeGreaterThan(0);
expect(validateRegexChecks(checks, item.id)).toEqual([]);
for (const text of [
example.valid,
example.valid.replaceAll("\n", "\r\n"),
example.valid.replaceAll("/", "\\"),
]) {
expect(
(await runOutputChecks(text, checks)).every((check) => check.passed),
).toBe(true);
}
expect(
(await runOutputChecks(example.invalid, checks)).some(
(check) => !check.passed,
),
).toBe(true);
});
}

test("compactness checks reject one line beyond their declared bounds", async () => {
for (const [name, limit] of [
["algorithm-pseudocode", 24],
["direct-call-flow", 24],
["file-responsibility", 22],
["incomplete-subject", 8],
["indirect-state-transitions", 28],
["structural-diff", 24],
] as const) {
const checks = (await load(name)).output_checks!.filter(
(check) => check.name === "response stays compact",
);
expect(checks).toHaveLength(1);
for (const newline of ["\n", "\r\n"]) {
const message = Array<string>(limit).fill("line").join(newline);
expect(
(await runOutputChecks(message + newline, checks))[0]!.passed,
).toBe(true);
expect(
(await runOutputChecks(message + newline + "extra", checks))[0]!.passed,
).toBe(false);
}
}
});

async function git(root: string, args: string[]): Promise<string> {
const proc = Bun.spawn(["git", ...args], {
cwd: root,
stdout: "pipe",
stderr: "pipe",
});
const [out, err, code] = await Promise.all([
new Response(proc.stdout).text(),
new Response(proc.stderr).text(),
proc.exited,
]);
expect(code, err).toBe(0);
return out;
}

async function fixture(item: EvalCase) {
const root = await mkdtemp(join(tmpdir(), "explanation ü "));
roots.push(root);
await git(root, ["init"]);
await git(root, ["config", "user.name", "Fixture"]);
await git(root, ["config", "user.email", "fixture@example.invalid"]);
await git(root, ["config", "core.autocrlf", "false"]);
for (const commit of item.fixture.commits ?? []) {
await files(root, commit.files);
await git(root, ["add", "."]);
await git(root, [
"-c",
"commit.gpgsign=false",
"commit",
"-m",
commit.message,
]);
}
await files(root, item.fixture.files ?? {});
return root;
}

async function files(root: string, entries: Record<string, string>) {
for (const [path, content] of Object.entries(entries)) {
await mkdir(dirname(join(root, path)), { recursive: true });
await writeFile(join(root, path), content);
}
}

// The shared live runner currently launches checks through /bin/sh. Bun's
// shell lets CI exercise these exact Git-only checks on native Windows too.
async function passes(root: string, item: EvalCase) {
const outcomes = [];
for (const check of item.checks) {
expect(check.run).toMatch(/^git /);
expect(check.run).not.toMatch(/[;\n]|&&|\|\||\$\(/);
const result = await Bun.$`${{ raw: check.run }}`
.cwd(root)
.quiet()
.nothrow();
const out = result.text().replace(/\r?\n$/, "");
outcomes.push(
result.exitCode === (check.exit_code ?? 0) &&
(check.expect_exact === undefined || out === check.expect_exact) &&
(check.expect_regex === undefined ||
new RegExp(check.expect_regex, "m").test(out)),
);
}
return outcomes.every(Boolean);
}

test("read-only cases reject edits and created artifacts", async () => {
for (const name of examples
.filter((item) => !item.name.startsWith("no-trigger"))
.map((item) => item.name)) {
const item = await load(name);
expect(item.fixture.setup).toBeUndefined();
const root = await fixture(item);
expect(await passes(root, item)).toBe(true);
await writeFile(join(root, "unrequested.html"), "artifact");
expect(await passes(root, item)).toBe(false);
await rm(join(root, "unrequested.html"));
const tracked = Object.keys(item.fixture.commits![0]!.files)[0]!;
await writeFile(join(root, tracked), "modified");
expect(await passes(root, item)).toBe(false);
}
});

test("pressure preserves untracked bytes as well as status", async () => {
const item = await load("pressure-insufficient-evidence");
expect(item.fixture.setup).toBeUndefined();
const root = await fixture(item);
expect(await passes(root, item)).toBe(true);
await writeFile(join(root, "local-note.txt"), "different content\n");
expect(await passes(root, item)).toBe(false);
await rm(join(root, "local-note.txt"));
expect(await passes(root, item)).toBe(false);
});

test("HTML case rejects missing or non-HTML files", async () => {
const item = await load("no-trigger-html-artifact");
const root = await fixture(item);
expect(await passes(root, item)).toBe(false);
await writeFile(join(root, "architecture.html"), "not HTML");
expect(await passes(root, item)).toBe(false);
await writeFile(
join(root, "architecture.html"),
"<!doctype html><html></html>",
);
expect(await passes(root, item)).toBe(true);
});

test("implementation case requires a changed doubling expression", async () => {
const item = await load("no-trigger-implementation");
const root = await fixture(item);
expect(await passes(root, item)).toBe(false);
await writeFile(
join(root, "src/math.js"),
"export const double = (value) => value * 3;\n",
);
expect(await passes(root, item)).toBe(false);
await writeFile(
join(root, "src/math.js"),
"export const double = (value) => value * 2;\n",
);
expect(await passes(root, item)).toBe(true);
await git(root, ["add", "src/math.js"]);
expect(await passes(root, item)).toBe(true);
});
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "darrow-explanation",
"description": "Compact, grounded visual explanations for technical subjects",
"version": "0.1.2",
"version": "0.1.4",
"license": "BUSL-1.1",
"author": {
"name": "Björn Rochel",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "darrow-explanation",
"version": "0.1.2",
"version": "0.1.4",
"description": "Compact, grounded visual explanations for technical subjects",
"author": {
"name": "Björn Rochel",
Expand Down
Loading
Loading