refactor: move plugin validation mechanics to Python

.github/workflows/python-quality.yml

@@ -184,8 +184,7 @@ jobs:
         with:
           python-version: "3.13"
       - uses: astral-sh/setup-uv@v6
-      - if: runner.os != 'Windows'
-        run: bash plugins/capability/darrow-git/tests/fresh-install.test.sh
+      - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-git/backend python plugins/capability/darrow-git/backend/tests/fresh_install.py
       - name: Verify Git CLI regression scenarios
         if: runner.os != 'Windows'
         shell: bash
@@ -194,9 +193,6 @@ jobs:
           for test in plugins/capability/darrow-git/backend/tests/shell/*.test.sh; do
             /bin/bash "$test"
           done
-      - if: runner.os == 'Windows'
-        shell: pwsh
-        run: "& plugins/capability/darrow-git/tests/fresh-install.test.ps1"
 
   review-windows:
     name: Review Python ${{ matrix.python-version }} on windows-latest
@@ -464,28 +460,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-            shell: bash
-            test: plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh
-          - os: macos-latest
-            shell: bash
-            test: plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh
-          - os: windows-latest
-            shell: pwsh
-            test: plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.ps1
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: "3.13"
       - uses: astral-sh/setup-uv@v6
-      - if: matrix.shell == 'bash'
-        run: bash "${{ matrix.test }}"
-      - if: matrix.shell == 'pwsh'
-        shell: pwsh
-        run: "& '${{ matrix.test }}'"
+      - run: uv run --quiet --frozen --no-dev --project plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend python plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/fresh_install.py
 
   observability-fresh-install:
     name: Fresh install observability-langfuse on ubuntu-latest
@@ -498,7 +481,7 @@ jobs:
         with:
           python-version: "3.13"
       - uses: astral-sh/setup-uv@v6
-      - run: bash plugins/capability/darrow-observability-langfuse/tests/fresh-install.test.sh
+      - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-observability-langfuse/backend python plugins/capability/darrow-observability-langfuse/backend/tests/fresh_install.py
 
   discovery-fresh-install:
     name: Fresh install discovery on ${{ matrix.os }}
@@ -507,28 +490,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-            shell: bash
-            test: plugins/capability/darrow-discovery/tests/fresh-install.test.sh
-          - os: macos-latest
-            shell: bash
-            test: plugins/capability/darrow-discovery/tests/fresh-install.test.sh
-          - os: windows-latest
-            shell: pwsh
-            test: plugins/capability/darrow-discovery/tests/fresh-install.test.ps1
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: "3.13"
       - uses: astral-sh/setup-uv@v6
-      - if: matrix.shell == 'bash'
-        run: bash "${{ matrix.test }}"
-      - if: matrix.shell == 'pwsh'
-        shell: pwsh
-        run: "& '${{ matrix.test }}'"
+      - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-discovery/skills/plan-implementation/backend python plugins/capability/darrow-discovery/skills/plan-implementation/backend/tests/fresh_install.py
 
   verification-fresh-install:
     name: Fresh install verification on ${{ matrix.os }}
@@ -537,28 +507,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: ubuntu-latest
-            shell: bash
-            test: plugins/capability/darrow-verification/tests/fresh-install.test.sh
-          - os: macos-latest
-            shell: bash
-            test: plugins/capability/darrow-verification/tests/fresh-install.test.sh
-          - os: windows-latest
-            shell: pwsh
-            test: plugins/capability/darrow-verification/tests/fresh-install.test.ps1
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
           python-version: "3.13"
       - uses: astral-sh/setup-uv@v6
-      - if: matrix.shell == 'bash'
-        run: bash "${{ matrix.test }}"
-      - if: matrix.shell == 'pwsh'
-        shell: pwsh
-        run: "& '${{ matrix.test }}'"
+      - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-verification/skills/verify-change/backend python plugins/capability/darrow-verification/skills/verify-change/backend/tests/fresh_install.py
 
   performance:
     name: Python performance invariants

docs/eval-development.md

@@ -27,6 +27,9 @@ constraints for producing and interpreting that evidence.
   require one exact tool sequence when several safe implementations yield the
   same observable result.
 - Keep participant prompts visible and pass criteria hidden.
+- Keep packaged Python oracles under `backend/tests/evals/` so participant mounts
+  omit them. Copy them into `.git/eval-checks/` during fixture setup for hidden
+  grading; agents cannot read or write that reserved subtree.
 - Keep authoritative fixture input and its executable acceptance oracle aligned,
   including whether representation details such as trailing newlines matter.
 - Keep fixture skills inert in the source tree: never name an eval fixture

docs/specs/python-quality.md

@@ -54,6 +54,18 @@ precedence, serialized bytes, and resource lifetimes. Standard-library replaceme
 must preserve those contracts across the package's declared Python versions;
 resource cleanup must not implicitly change transaction or publication semantics.
 
+The same implementation boundary applies to validation mechanics: substantial
+eval fixture providers, structured JSON/TSV evidence checks, and copied-artifact
+installation checks belong in the owning plugin's Python package. Keep eval
+oracles independent of the implementation they assess. Share fixture mechanics
+within that plugin rather than duplicating them across YAML cases; case-specific
+input and expectations remain explicit in each case. Use one portable Python
+installation check for Unix and Windows, preserving platform-specific probes.
+Small host launchers and tests whose subject is shell behavior may remain shell.
+Invoking a shell launcher does not by itself make a test shell-specific: keep
+configuration matrices, protocol assertions, and fixture construction in Python,
+and invoke each supported shell from those tests when launcher coverage matters.
+
 ## Tests and coverage
 
 Each package keeps fast unit tests for local behavior and integration tests for

docs/specs/skill-evaluation.md

@@ -257,6 +257,12 @@ credential-free environment and cannot access source worktrees, peer fixtures,
 harness credentials, or modify retained evidence. An unavailable isolation
 boundary is an explicit error.
 
+Packaged Python eval oracles remain hidden from participants. Plugin mounting
+excludes every nested `evals/` directory, including `backend/tests/evals/`.
+Fixture setup may copy grading helpers to `.git/eval-checks/`; the outer agent
+sandbox denies reading and writing that subtree, while isolated grading may
+execute it. Declared external isolation must preserve this boundary too.
+
 Codex trials may import the repository's explicit agent-concurrency limit into
 their isolated configuration. They must not inherit unrelated repository or
 user settings. Capture the limit once per runner process, include it in the

evals/runner/fixture.test.ts

@@ -239,6 +239,7 @@ describe("eval fixture skill mounts", () => {
       await mkdir(skill, { recursive: true });
       await mkdir(join(backend, ".venv"), { recursive: true });
       await mkdir(join(backend, "evals"), { recursive: true });
+      await mkdir(join(backend, "tests", "evals"), { recursive: true });
       await mkdir(join(plugin, ".claude-plugin"), { recursive: true });
       await mkdir(join(plugin, ".codex-plugin"), { recursive: true });
       await writeFile(
@@ -248,6 +249,7 @@ describe("eval fixture skill mounts", () => {
       await writeFile(join(backend, "pyproject.toml"), "[project]\n");
       await writeFile(join(backend, ".venv", "generated"), "private\n");
       await writeFile(join(backend, "evals", "secret.yaml"), "hidden\n");
+      await writeFile(join(backend, "tests", "evals", "oracle.py"), "hidden\n");
       await writeFile(
         join(plugin, ".claude-plugin", "hooks.json"),
         '{"hooks":{"SessionStart":[]}}\n',
@@ -275,6 +277,7 @@ describe("eval fixture skill mounts", () => {
       expect(existsSync(join(destination, "pyproject.toml"))).toBe(true);
       expect(existsSync(join(destination, ".venv"))).toBe(false);
       expect(existsSync(join(destination, "evals"))).toBe(false);
+      expect(existsSync(join(destination, "tests", "evals"))).toBe(false);
       expect(
         existsSync(join(dirname(destination), ".claude-plugin", "hooks.json")),
       ).toBe(host !== "project");

evals/runner/fixture.ts

@@ -216,7 +216,6 @@ async function copySkillWithoutEvals(
   mountedSkillDir: string,
   destination: string,
 ): Promise<void> {
-  const evalsDir = join(mountedSkillDir, "evals");
   const generatedEntries = new Set([
     ".coverage",
     ".hypothesis",
@@ -226,13 +225,12 @@ async function copySkillWithoutEvals(
     ".venv",
     "__pycache__",
     "coverage.json",
+    "evals",
   ]);
   await cp(mountedSkillDir, destination, {
     recursive: true,
     // Never expose any skill's colocated pass criteria to the model.
     filter: (src) =>
-      src !== evalsDir &&
-      !src.startsWith(evalsDir + "/") &&
       relative(mountedSkillDir, src)
         .split(sep)
         .every((entry) => !generatedEntries.has(entry)),

evals/runner/review-route-eval-checks.test.ts

@@ -1,6 +1,6 @@
 import { afterEach, describe, expect, test } from "bun:test";
 import { spawnSync } from "node:child_process";
-import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { cp, mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { parse as parseYaml } from "yaml";
@@ -78,6 +78,23 @@ function launchEvent({
   };
 }
 
+async function copyOracle(root: string) {
+  const backend = resolve(
+    import.meta.dir,
+    "../../plugins/capability/darrow-review/backend",
+  );
+  const copied = join(root, ".git/eval-checks/review");
+  await mkdir(join(copied, "tests", "evals"), { recursive: true });
+  for (const name of [
+    "src",
+    "pyproject.toml",
+    "uv.lock",
+    "tests/evals/eval_routes.py",
+  ]) {
+    await cp(join(backend, name), join(copied, name), { recursive: true });
+  }
+}
+
 async function runGate(
   entry: (typeof cases)[number],
   host: "claude" | "codex",
@@ -87,6 +104,7 @@ async function runGate(
   roots.push(root);
   const artifacts = join(root, ".git", "darrow-review.fixture");
   await mkdir(artifacts, { recursive: true });
+  await copyOracle(root);
   const [model, effort] = entry[host];
   const route = `${host}\t${host === "claude" ? "anthropic" : "openai"}\t${model}\t${effort}`;
   await writeFile(

evals/runner/sandbox.test.ts

@@ -1,8 +1,19 @@
 import { afterEach, describe, expect, test } from "bun:test";
-import { mkdtemp, mkdir, readFile, realpath, rm } from "node:fs/promises";
+import {
+  mkdtemp,
+  mkdir,
+  readFile,
+  realpath,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { sandboxedAgentCommand, sandboxProfile } from "./sandbox";
+import {
+  sandboxedAgentCommand,
+  sandboxedCommand,
+  sandboxProfile,
+} from "./sandbox";
 
 const cleanup: string[] = [];
 
@@ -14,6 +25,34 @@ afterEach(async () => {
 });
 
 describe("eval outer sandbox", () => {
+  test.skipIf(process.platform !== "darwin")(
+    "keeps packaged oracles hidden from agents and available to grading",
+    async () => {
+      const repo = await mkdtemp(join(tmpdir(), "darrow-eval-oracle-"));
+      cleanup.push(repo);
+      const directory = join(repo, ".git", "eval-checks");
+      await mkdir(directory, { recursive: true });
+      const oracle = join(directory, "oracle.py");
+      await writeFile(oracle, "hidden acceptance check\n");
+      for (const argv of [
+        ["/bin/cat", oracle],
+        ["/usr/bin/touch", oracle],
+      ]) {
+        const command = await sandboxedAgentCommand(argv, repo);
+        expect(
+          await Bun.spawn(command, { stdout: "ignore", stderr: "ignore" })
+            .exited,
+        ).not.toBe(0);
+      }
+      const command = await sandboxedCommand(["/bin/cat", oracle], repo);
+      const grader = Bun.spawn(command, { stdout: "pipe", stderr: "pipe" });
+      expect(await new Response(grader.stdout).text()).toBe(
+        "hidden acceptance check\n",
+      );
+      expect(await grader.exited).toBe(0);
+    },
+  );
+
   test("builds deterministic deny rules and escapes literals", () => {
     const profile = sandboxProfile(
       ['/tmp/z"q', "/tmp/a\\b", "/tmp/a\\b"],

evals/runner/sandbox.ts

@@ -101,13 +101,14 @@ async function repositoryWorktrees(): Promise<string[]> {
  * CLI flags. The native agent sandbox is bypassed only inside this boundary so
  * fixture mutations remain representative while source cases stay secret.
  */
-export function sandboxedAgentCommand(
+export async function sandboxedAgentCommand(
   argv: string[],
   repoDir: string,
   writeDeniedPaths: string[] = [],
   executeOnlyPaths: string[] = [],
 ): Promise<string[]> {
   return sandboxedCommand(argv, repoDir, {
+    deniedPaths: [join(await realpath(repoDir), ".git", "eval-checks")],
     writeDeniedPaths,
     executeOnlyPaths,
   });

plugins/capability/darrow-discovery/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "darrow-discovery",
   "description": "Explicit grilling, feature discovery, and implementation planning",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "license": "BUSL-1.1",
   "author": {
     "name": "Björn Rochel",

plugins/capability/darrow-discovery/.codex-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "darrow-discovery",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "Explicit grilling, feature discovery, and implementation planning",
   "author": {
     "name": "Björn Rochel",