diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml index ebd29c9c..556eee9c 100644 --- a/.github/workflows/python-quality.yml +++ b/.github/workflows/python-quality.yml @@ -184,8 +184,7 @@ jobs: with: python-version: "3.13" - uses: astral-sh/setup-uv@v6 - - if: runner.os != 'Windows' - run: bash plugins/capability/darrow-git/tests/fresh-install.test.sh + - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-git/backend python plugins/capability/darrow-git/backend/tests/fresh_install.py - name: Verify Git CLI regression scenarios if: runner.os != 'Windows' shell: bash @@ -194,9 +193,6 @@ jobs: for test in plugins/capability/darrow-git/backend/tests/shell/*.test.sh; do /bin/bash "$test" done - - if: runner.os == 'Windows' - shell: pwsh - run: "& plugins/capability/darrow-git/tests/fresh-install.test.ps1" review-windows: name: Review Python ${{ matrix.python-version }} on windows-latest @@ -464,16 +460,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - shell: bash - test: plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh - - os: macos-latest - shell: bash - test: plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh - - os: windows-latest - shell: pwsh - test: plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.ps1 + os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -481,11 +468,7 @@ jobs: with: python-version: "3.13" - uses: astral-sh/setup-uv@v6 - - if: matrix.shell == 'bash' - run: bash "${{ matrix.test }}" - - if: matrix.shell == 'pwsh' - shell: pwsh - run: "& '${{ matrix.test }}'" + - run: uv run --quiet --frozen --no-dev --project plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend python plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/fresh_install.py observability-fresh-install: name: Fresh install observability-langfuse on ubuntu-latest @@ -498,7 +481,7 @@ jobs: with: python-version: "3.13" - uses: astral-sh/setup-uv@v6 - - run: bash plugins/capability/darrow-observability-langfuse/tests/fresh-install.test.sh + - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-observability-langfuse/backend python plugins/capability/darrow-observability-langfuse/backend/tests/fresh_install.py discovery-fresh-install: name: Fresh install discovery on ${{ matrix.os }} @@ -507,16 +490,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - shell: bash - test: plugins/capability/darrow-discovery/tests/fresh-install.test.sh - - os: macos-latest - shell: bash - test: plugins/capability/darrow-discovery/tests/fresh-install.test.sh - - os: windows-latest - shell: pwsh - test: plugins/capability/darrow-discovery/tests/fresh-install.test.ps1 + os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -524,11 +498,7 @@ jobs: with: python-version: "3.13" - uses: astral-sh/setup-uv@v6 - - if: matrix.shell == 'bash' - run: bash "${{ matrix.test }}" - - if: matrix.shell == 'pwsh' - shell: pwsh - run: "& '${{ matrix.test }}'" + - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-discovery/skills/plan-implementation/backend python plugins/capability/darrow-discovery/skills/plan-implementation/backend/tests/fresh_install.py verification-fresh-install: name: Fresh install verification on ${{ matrix.os }} @@ -537,16 +507,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - shell: bash - test: plugins/capability/darrow-verification/tests/fresh-install.test.sh - - os: macos-latest - shell: bash - test: plugins/capability/darrow-verification/tests/fresh-install.test.sh - - os: windows-latest - shell: pwsh - test: plugins/capability/darrow-verification/tests/fresh-install.test.ps1 + os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -554,11 +515,7 @@ jobs: with: python-version: "3.13" - uses: astral-sh/setup-uv@v6 - - if: matrix.shell == 'bash' - run: bash "${{ matrix.test }}" - - if: matrix.shell == 'pwsh' - shell: pwsh - run: "& '${{ matrix.test }}'" + - run: uv run --quiet --frozen --no-dev --project plugins/capability/darrow-verification/skills/verify-change/backend python plugins/capability/darrow-verification/skills/verify-change/backend/tests/fresh_install.py performance: name: Python performance invariants diff --git a/docs/eval-development.md b/docs/eval-development.md index facca76f..cc716043 100644 --- a/docs/eval-development.md +++ b/docs/eval-development.md @@ -27,6 +27,9 @@ constraints for producing and interpreting that evidence. require one exact tool sequence when several safe implementations yield the same observable result. - Keep participant prompts visible and pass criteria hidden. +- Keep packaged Python oracles under `backend/tests/evals/` so participant mounts + omit them. Copy them into `.git/eval-checks/` during fixture setup for hidden + grading; agents cannot read or write that reserved subtree. - Keep authoritative fixture input and its executable acceptance oracle aligned, including whether representation details such as trailing newlines matter. - Keep fixture skills inert in the source tree: never name an eval fixture diff --git a/docs/specs/python-quality.md b/docs/specs/python-quality.md index 0711ea8f..17aa2fea 100644 --- a/docs/specs/python-quality.md +++ b/docs/specs/python-quality.md @@ -54,6 +54,18 @@ precedence, serialized bytes, and resource lifetimes. Standard-library replaceme must preserve those contracts across the package's declared Python versions; resource cleanup must not implicitly change transaction or publication semantics. +The same implementation boundary applies to validation mechanics: substantial +eval fixture providers, structured JSON/TSV evidence checks, and copied-artifact +installation checks belong in the owning plugin's Python package. Keep eval +oracles independent of the implementation they assess. Share fixture mechanics +within that plugin rather than duplicating them across YAML cases; case-specific +input and expectations remain explicit in each case. Use one portable Python +installation check for Unix and Windows, preserving platform-specific probes. +Small host launchers and tests whose subject is shell behavior may remain shell. +Invoking a shell launcher does not by itself make a test shell-specific: keep +configuration matrices, protocol assertions, and fixture construction in Python, +and invoke each supported shell from those tests when launcher coverage matters. + ## Tests and coverage Each package keeps fast unit tests for local behavior and integration tests for diff --git a/docs/specs/skill-evaluation.md b/docs/specs/skill-evaluation.md index 8ddd9844..a20bc40a 100644 --- a/docs/specs/skill-evaluation.md +++ b/docs/specs/skill-evaluation.md @@ -257,6 +257,12 @@ credential-free environment and cannot access source worktrees, peer fixtures, harness credentials, or modify retained evidence. An unavailable isolation boundary is an explicit error. +Packaged Python eval oracles remain hidden from participants. Plugin mounting +excludes every nested `evals/` directory, including `backend/tests/evals/`. +Fixture setup may copy grading helpers to `.git/eval-checks/`; the outer agent +sandbox denies reading and writing that subtree, while isolated grading may +execute it. Declared external isolation must preserve this boundary too. + Codex trials may import the repository's explicit agent-concurrency limit into their isolated configuration. They must not inherit unrelated repository or user settings. Capture the limit once per runner process, include it in the diff --git a/evals/runner/fixture.test.ts b/evals/runner/fixture.test.ts index 6460d0b0..06c4b727 100644 --- a/evals/runner/fixture.test.ts +++ b/evals/runner/fixture.test.ts @@ -239,6 +239,7 @@ describe("eval fixture skill mounts", () => { await mkdir(skill, { recursive: true }); await mkdir(join(backend, ".venv"), { recursive: true }); await mkdir(join(backend, "evals"), { recursive: true }); + await mkdir(join(backend, "tests", "evals"), { recursive: true }); await mkdir(join(plugin, ".claude-plugin"), { recursive: true }); await mkdir(join(plugin, ".codex-plugin"), { recursive: true }); await writeFile( @@ -248,6 +249,7 @@ describe("eval fixture skill mounts", () => { await writeFile(join(backend, "pyproject.toml"), "[project]\n"); await writeFile(join(backend, ".venv", "generated"), "private\n"); await writeFile(join(backend, "evals", "secret.yaml"), "hidden\n"); + await writeFile(join(backend, "tests", "evals", "oracle.py"), "hidden\n"); await writeFile( join(plugin, ".claude-plugin", "hooks.json"), '{"hooks":{"SessionStart":[]}}\n', @@ -275,6 +277,7 @@ describe("eval fixture skill mounts", () => { expect(existsSync(join(destination, "pyproject.toml"))).toBe(true); expect(existsSync(join(destination, ".venv"))).toBe(false); expect(existsSync(join(destination, "evals"))).toBe(false); + expect(existsSync(join(destination, "tests", "evals"))).toBe(false); expect( existsSync(join(dirname(destination), ".claude-plugin", "hooks.json")), ).toBe(host !== "project"); diff --git a/evals/runner/fixture.ts b/evals/runner/fixture.ts index df58f7e6..41c507f3 100644 --- a/evals/runner/fixture.ts +++ b/evals/runner/fixture.ts @@ -216,7 +216,6 @@ async function copySkillWithoutEvals( mountedSkillDir: string, destination: string, ): Promise { - const evalsDir = join(mountedSkillDir, "evals"); const generatedEntries = new Set([ ".coverage", ".hypothesis", @@ -226,13 +225,12 @@ async function copySkillWithoutEvals( ".venv", "__pycache__", "coverage.json", + "evals", ]); await cp(mountedSkillDir, destination, { recursive: true, // Never expose any skill's colocated pass criteria to the model. filter: (src) => - src !== evalsDir && - !src.startsWith(evalsDir + "/") && relative(mountedSkillDir, src) .split(sep) .every((entry) => !generatedEntries.has(entry)), diff --git a/evals/runner/review-route-eval-checks.test.ts b/evals/runner/review-route-eval-checks.test.ts index 39c99517..70ea0a82 100644 --- a/evals/runner/review-route-eval-checks.test.ts +++ b/evals/runner/review-route-eval-checks.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from "bun:test"; import { spawnSync } from "node:child_process"; -import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { cp, mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { parse as parseYaml } from "yaml"; @@ -78,6 +78,23 @@ function launchEvent({ }; } +async function copyOracle(root: string) { + const backend = resolve( + import.meta.dir, + "../../plugins/capability/darrow-review/backend", + ); + const copied = join(root, ".git/eval-checks/review"); + await mkdir(join(copied, "tests", "evals"), { recursive: true }); + for (const name of [ + "src", + "pyproject.toml", + "uv.lock", + "tests/evals/eval_routes.py", + ]) { + await cp(join(backend, name), join(copied, name), { recursive: true }); + } +} + async function runGate( entry: (typeof cases)[number], host: "claude" | "codex", @@ -87,6 +104,7 @@ async function runGate( roots.push(root); const artifacts = join(root, ".git", "darrow-review.fixture"); await mkdir(artifacts, { recursive: true }); + await copyOracle(root); const [model, effort] = entry[host]; const route = `${host}\t${host === "claude" ? "anthropic" : "openai"}\t${model}\t${effort}`; await writeFile( diff --git a/evals/runner/sandbox.test.ts b/evals/runner/sandbox.test.ts index 42887eb6..be152f7e 100644 --- a/evals/runner/sandbox.test.ts +++ b/evals/runner/sandbox.test.ts @@ -1,8 +1,19 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, mkdir, readFile, realpath, rm } from "node:fs/promises"; +import { + mkdtemp, + mkdir, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { sandboxedAgentCommand, sandboxProfile } from "./sandbox"; +import { + sandboxedAgentCommand, + sandboxedCommand, + sandboxProfile, +} from "./sandbox"; const cleanup: string[] = []; @@ -14,6 +25,34 @@ afterEach(async () => { }); describe("eval outer sandbox", () => { + test.skipIf(process.platform !== "darwin")( + "keeps packaged oracles hidden from agents and available to grading", + async () => { + const repo = await mkdtemp(join(tmpdir(), "darrow-eval-oracle-")); + cleanup.push(repo); + const directory = join(repo, ".git", "eval-checks"); + await mkdir(directory, { recursive: true }); + const oracle = join(directory, "oracle.py"); + await writeFile(oracle, "hidden acceptance check\n"); + for (const argv of [ + ["/bin/cat", oracle], + ["/usr/bin/touch", oracle], + ]) { + const command = await sandboxedAgentCommand(argv, repo); + expect( + await Bun.spawn(command, { stdout: "ignore", stderr: "ignore" }) + .exited, + ).not.toBe(0); + } + const command = await sandboxedCommand(["/bin/cat", oracle], repo); + const grader = Bun.spawn(command, { stdout: "pipe", stderr: "pipe" }); + expect(await new Response(grader.stdout).text()).toBe( + "hidden acceptance check\n", + ); + expect(await grader.exited).toBe(0); + }, + ); + test("builds deterministic deny rules and escapes literals", () => { const profile = sandboxProfile( ['/tmp/z"q', "/tmp/a\\b", "/tmp/a\\b"], diff --git a/evals/runner/sandbox.ts b/evals/runner/sandbox.ts index 02bbec50..af653935 100644 --- a/evals/runner/sandbox.ts +++ b/evals/runner/sandbox.ts @@ -101,13 +101,14 @@ async function repositoryWorktrees(): Promise { * CLI flags. The native agent sandbox is bypassed only inside this boundary so * fixture mutations remain representative while source cases stay secret. */ -export function sandboxedAgentCommand( +export async function sandboxedAgentCommand( argv: string[], repoDir: string, writeDeniedPaths: string[] = [], executeOnlyPaths: string[] = [], ): Promise { return sandboxedCommand(argv, repoDir, { + deniedPaths: [join(await realpath(repoDir), ".git", "eval-checks")], writeDeniedPaths, executeOnlyPaths, }); diff --git a/plugins/capability/darrow-discovery/.claude-plugin/plugin.json b/plugins/capability/darrow-discovery/.claude-plugin/plugin.json index 6e994549..73eb843c 100644 --- a/plugins/capability/darrow-discovery/.claude-plugin/plugin.json +++ b/plugins/capability/darrow-discovery/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "darrow-discovery", "description": "Explicit grilling, feature discovery, and implementation planning", - "version": "0.2.3", + "version": "0.2.4", "license": "BUSL-1.1", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-discovery/.codex-plugin/plugin.json b/plugins/capability/darrow-discovery/.codex-plugin/plugin.json index 7bd88e9c..20189829 100644 --- a/plugins/capability/darrow-discovery/.codex-plugin/plugin.json +++ b/plugins/capability/darrow-discovery/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-discovery", - "version": "0.2.3", + "version": "0.2.4", "description": "Explicit grilling, feature discovery, and implementation planning", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-discovery/skills/plan-implementation/backend/tests/fresh_install.py b/plugins/capability/darrow-discovery/skills/plan-implementation/backend/tests/fresh_install.py new file mode 100644 index 00000000..71add3ab --- /dev/null +++ b/plugins/capability/darrow-discovery/skills/plan-implementation/backend/tests/fresh_install.py @@ -0,0 +1,149 @@ +"""Validate a fresh copied plugin using only locked runtime dependencies.""" + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + + +def command(cwd: Path, *args: str) -> bytes: + return subprocess.run(args, cwd=cwd, check=True, capture_output=True).stdout + + +def installed_runtime(backend: Path, fixture: Path) -> None: + command(fixture, "uv", "sync", "--frozen", "--no-dev", "--project", str(backend)) + tree = command( + fixture, "uv", "tree", "--frozen", "--no-dev", "--project", str(backend) + ).decode() + assert not re.search(r"\b(coverage|hypothesis|mypy|pytest|ruff)\b", tree), tree + + +def probe( + backend: Path, + cwd: Path, + args: list[str], + *, + output: str = "", + error: str = "", + status: int = 0, + encoding: str = "utf-8", +) -> None: + result = subprocess.run( + [ + "uv", + "run", + "--quiet", + "--frozen", + "--no-dev", + "--project", + str(backend), + "darrow-render-plan-frontier", + *args, + ], + cwd=cwd, + env={**os.environ, "PYTHONIOENCODING": encoding}, + capture_output=True, + check=False, + ) + assert (result.returncode, result.stdout, result.stderr) == ( + status, + output.encode("utf-8"), + error.encode("utf-8"), + ), result + + +def validate(backend: Path, fixture: Path) -> None: + probe( + backend, + fixture, + [ + "--evidence", + "The repository stores records in one region today.", + "--question", + "Data region: single-region or multi-region?", + "--option", + "single-region", + "--option", + "multi-region", + "--choice", + "single-region", + "--rationale", + "it limits operational coupling", + "--deferred", + "storage vendor and migration path", + ], + output=( + "Evidence: The repository stores records in one region today.\n\n" + "Q1 — Data region: single-region or multi-region?\n\n" + "Recommendation: Choose single-region because it limits operational coupling.\n\n" + "Deferred: storage vendor and migration path. After your answer, " + "I will recompute the next frontier.\n" + ), + ) + probe( + backend, + fixture, + [ + "--evidence", + "Fact", + "--question", + "Mode: ä or b?", + "--option", + "Ä", + "--option", + "b", + "--choice", + "ä", + "--rationale", + "reason", + "--deferred", + "category", + ], + encoding="cp1252", + output=( + "Evidence: Fact.\n\nQ1 — Mode: ä or b?\n\n" + "Recommendation: Choose ä because reason.\n\n" + "Deferred: category. After your answer, I will recompute the next frontier.\n" + ), + ) + probe( + backend, fixture, [], status=2, error="every frontier field must be non-empty\n" + ) + usage = ( + "usage: darrow-render-plan-frontier --evidence TEXT --question TEXT " + "--option LABEL --option LABEL [--option LABEL ...] --choice LABEL " + "--rationale TEXT --deferred TEXT\n" + ) + for arguments in (["--evidence"], ["--unknown", "value"]): + probe(backend, fixture, arguments, status=2, error=usage) + + +def main() -> None: + plugin = Path(__file__).resolve().parents[4] + with tempfile.TemporaryDirectory(prefix="darrow plan-implementation ") as temporary: + fixture = Path(temporary).resolve() + copy = fixture / "plugin copy" + shutil.copytree( + plugin, + copy, + ignore=shutil.ignore_patterns( + ".venv", + "__pycache__", + ".hypothesis", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".coverage*", + "coverage.json", + ), + ) + backend = copy / "skills" / "plan-implementation" / "backend" + installed_runtime(backend, fixture) + validate(backend, fixture) + print("plan-implementation fresh install passed") + + +if __name__ == "__main__": + main() diff --git a/plugins/capability/darrow-discovery/tests/fresh-install.test.ps1 b/plugins/capability/darrow-discovery/tests/fresh-install.test.ps1 deleted file mode 100644 index b8c0c737..00000000 --- a/plugins/capability/darrow-discovery/tests/fresh-install.test.ps1 +++ /dev/null @@ -1,157 +0,0 @@ -$ErrorActionPreference = "Stop" - -$pluginRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -$fixture = Join-Path ([System.IO.Path]::GetTempPath()) ("darrow discovery " + [guid]::NewGuid()) -$copy = Join-Path $fixture "plugin copy" -$utf8 = [System.Text.UTF8Encoding]::new($false) - -function Assert-Bytes([string]$Path, [byte[]]$Expected, [string]$Message) { - $actual = [System.IO.File]::ReadAllBytes($Path) - if ([Convert]::ToBase64String($actual) -ne [Convert]::ToBase64String($Expected)) { - throw $Message - } -} - -function Invoke-Renderer( - [string]$Backend, - [string]$WorkingDirectory, - [string[]]$Arguments, - [string]$StdoutPath, - [string]$StderrPath -) { - $startInfo = [System.Diagnostics.ProcessStartInfo]::new() - $startInfo.FileName = (Get-Command uv).Source - $startInfo.UseShellExecute = $false - $startInfo.RedirectStandardOutput = $true - $startInfo.RedirectStandardError = $true - $startInfo.WorkingDirectory = $WorkingDirectory - foreach ($argument in @( - "run", "--quiet", "--frozen", "--no-dev", "--project", $Backend, - "darrow-render-plan-frontier" - ) + $Arguments) { - [void]$startInfo.ArgumentList.Add($argument) - } - - $process = [System.Diagnostics.Process]::new() - $process.StartInfo = $startInfo - $stdoutStream = $null - $stderrStream = $null - try { - if (-not $process.Start()) { throw "renderer process did not start" } - $stdoutStream = [System.IO.File]::Open( - $StdoutPath, - [System.IO.FileMode]::Create, - [System.IO.FileAccess]::Write, - [System.IO.FileShare]::None - ) - $stderrStream = [System.IO.File]::Open( - $StderrPath, - [System.IO.FileMode]::Create, - [System.IO.FileAccess]::Write, - [System.IO.FileShare]::None - ) - $stdoutCopy = $process.StandardOutput.BaseStream.CopyToAsync($stdoutStream) - $stderrCopy = $process.StandardError.BaseStream.CopyToAsync($stderrStream) - $process.WaitForExit() - [System.Threading.Tasks.Task]::WaitAll( - [System.Threading.Tasks.Task[]]@($stdoutCopy, $stderrCopy) - ) - return $process.ExitCode - } - finally { - if ($null -ne $stdoutStream) { $stdoutStream.Dispose() } - if ($null -ne $stderrStream) { $stderrStream.Dispose() } - $process.Dispose() - } -} - -try { - New-Item -ItemType Directory -Path $fixture | Out-Null - Copy-Item -LiteralPath $pluginRoot -Destination $copy -Recurse - $skillDir = Join-Path $copy "skills/plan-implementation" - $backend = Join-Path $skillDir "backend" - foreach ($generatedDirectory in @(".venv", ".hypothesis", ".mypy_cache", ".pytest_cache", ".ruff_cache")) { - Remove-Item -LiteralPath (Join-Path $backend $generatedDirectory) -Recurse -Force -ErrorAction SilentlyContinue - } - Get-ChildItem -LiteralPath $backend -Directory -Recurse -Filter "__pycache__" | - Remove-Item -Recurse -Force - Get-ChildItem -LiteralPath $backend -File -Filter ".coverage*" | - Remove-Item -Force - Remove-Item -LiteralPath (Join-Path $backend "coverage.json") -Force -ErrorAction SilentlyContinue - - & uv sync --locked --no-dev --project $backend - if ($LASTEXITCODE -ne 0) { throw "runtime sync failed" } - $runtimeTree = (& uv tree --locked --no-dev --project $backend) -join "`n" - if ($LASTEXITCODE -ne 0) { throw "runtime tree failed" } - foreach ($tool in @("coverage", "hypothesis", "mypy", "pytest", "ruff")) { - if ($runtimeTree -match "(?m)(^|\s)$tool(\s|$)") { - throw "development dependency installed at runtime: $tool" - } - } - - $stdout = Join-Path $fixture "stdout" - $stderr = Join-Path $fixture "stderr" - $expected = @( - "Evidence: The repository stores records in one region today.", - "", - "Q1 — Data region: single-region or multi-region?", - "", - "Recommendation: Choose single-region because it limits operational coupling.", - "", - "Deferred: storage vendor and migration path. After your answer, I will recompute the next frontier." - ) -join "`n" - - $exitCode = Invoke-Renderer $backend $fixture @( - "--evidence", "The repository stores records in one region today.", - "--question", "Data region: single-region or multi-region?", - "--option", "single-region", "--option", "multi-region", "--choice", "single-region", - "--rationale", "it limits operational coupling", - "--deferred", "storage vendor and migration path" - ) $stdout $stderr - if ($exitCode -ne 0) { throw "renderer failed" } - Assert-Bytes $stdout ($utf8.GetBytes("$expected`n")) "renderer output bytes differed from the public contract" - Assert-Bytes $stderr ([byte[]]@()) "valid renderer invocation wrote stderr" - - $previousPythonIoEncoding = $env:PYTHONIOENCODING - $env:PYTHONIOENCODING = "cp1252" - try { - $exitCode = Invoke-Renderer $backend $fixture @( - "--evidence", "Fact", "--question", "Mode: ä or b?", - "--option", "Ä", "--option", "b", "--choice", "ä", - "--rationale", "reason", "--deferred", "category" - ) $stdout $stderr - if ($exitCode -ne 0) { throw "Unicode renderer invocation failed" } - } - finally { - $env:PYTHONIOENCODING = $previousPythonIoEncoding - } - $unicodeExpected = @( - "Evidence: Fact.", - "", - "Q1 — Mode: ä or b?", - "", - "Recommendation: Choose ä because reason.", - "", - "Deferred: category. After your answer, I will recompute the next frontier." - ) -join "`n" - Assert-Bytes $stdout ($utf8.GetBytes("$unicodeExpected`n")) "Unicode renderer output bytes differed" - Assert-Bytes $stderr ([byte[]]@()) "Unicode renderer invocation wrote stderr" - - $exitCode = Invoke-Renderer $backend $fixture @() $stdout $stderr - if ($exitCode -ne 2) { throw "invalid renderer invocation did not exit 2" } - Assert-Bytes $stdout ([byte[]]@()) "invalid renderer invocation wrote stdout" - Assert-Bytes $stderr ($utf8.GetBytes("every frontier field must be non-empty`n")) "invalid renderer diagnostic bytes differed" - - $usage = "usage: darrow-render-plan-frontier --evidence TEXT --question TEXT --option LABEL --option LABEL [--option LABEL ...] --choice LABEL --rationale TEXT --deferred TEXT`n" - function Assert-Usage([string[]]$Arguments) { - $exitCode = Invoke-Renderer $backend $fixture $Arguments $stdout $stderr - if ($exitCode -ne 2) { throw "malformed renderer invocation did not exit 2" } - Assert-Bytes $stdout ([byte[]]@()) "malformed renderer invocation wrote stdout" - Assert-Bytes $stderr ($utf8.GetBytes($usage)) "renderer usage bytes differed" - } - Assert-Usage -Arguments @("--evidence") - Assert-Usage -Arguments @("--unknown", "value") -} -finally { - Remove-Item -LiteralPath $fixture -Recurse -Force -ErrorAction SilentlyContinue -} diff --git a/plugins/capability/darrow-discovery/tests/fresh-install.test.sh b/plugins/capability/darrow-discovery/tests/fresh-install.test.sh deleted file mode 100644 index 8eab1e87..00000000 --- a/plugins/capability/darrow-discovery/tests/fresh-install.test.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/sh -set -eu - -plugin_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -fixture=$(mktemp -d "${TMPDIR:-/tmp}/darrow-discovery-install.XXXXXX") -trap 'rm -rf "$fixture"' EXIT HUP INT TERM - -copy="$fixture/plugin copy" -mkdir -p "$copy" -cp -R "$plugin_root/." "$copy/" -skill_dir="$copy/skills/plan-implementation" -backend="$skill_dir/backend" -rm -rf \ - "$backend/.venv" \ - "$backend/.hypothesis" \ - "$backend/.mypy_cache" \ - "$backend/.pytest_cache" \ - "$backend/.ruff_cache" -rm -f "$backend"/.coverage* "$backend/coverage.json" -find "$backend" -type d -name __pycache__ -prune -exec rm -rf {} \; - -uv sync --locked --no-dev --project "$backend" -runtime_tree=$(uv tree --locked --no-dev --project "$backend") -for development_tool in coverage hypothesis mypy pytest ruff; do - if printf '%s\n' "$runtime_tree" | grep -E "(^|[[:space:]])${development_tool}([[:space:]]|$)" >/dev/null; then - echo "error: development dependency installed at runtime: $development_tool" >&2 - exit 1 - fi -done - -actual="$fixture/actual" -errors="$fixture/errors" -expected="$fixture/expected" -cd "$fixture" -uv run --quiet --frozen --no-dev --project "$backend" \ - darrow-render-plan-frontier \ - --evidence 'The repository stores records in one region today.' \ - --question 'Data region: single-region or multi-region?' \ - --option single-region --option multi-region --choice single-region \ - --rationale 'it limits operational coupling' \ - --deferred 'storage vendor and migration path' >"$actual" 2>"$errors" - -cat >"$expected" <<'EOF' -Evidence: The repository stores records in one region today. - -Q1 — Data region: single-region or multi-region? - -Recommendation: Choose single-region because it limits operational coupling. - -Deferred: storage vendor and migration path. After your answer, I will recompute the next frontier. -EOF -diff -u "$expected" "$actual" -test ! -s "$errors" - -PYTHONIOENCODING=cp1252 uv run --quiet --frozen --no-dev --project "$backend" \ - darrow-render-plan-frontier \ - --evidence Fact --question 'Mode: ä or b?' \ - --option Ä --option b --choice ä \ - --rationale reason --deferred category >"$actual" 2>"$errors" -cat >"$expected" <<'EOF' -Evidence: Fact. - -Q1 — Mode: ä or b? - -Recommendation: Choose ä because reason. - -Deferred: category. After your answer, I will recompute the next frontier. -EOF -cmp "$expected" "$actual" -test ! -s "$errors" - -status=0 -uv run --quiet --frozen --no-dev --project "$backend" \ - darrow-render-plan-frontier >"$actual" 2>"$errors" || status=$? -test "$status" -eq 2 -test ! -s "$actual" -test "$(cat "$errors")" = 'every frontier field must be non-empty' - -usage='usage: darrow-render-plan-frontier --evidence TEXT --question TEXT --option LABEL --option LABEL [--option LABEL ...] --choice LABEL --rationale TEXT --deferred TEXT' -for malformed in odd unknown; do - status=0 - case "$malformed" in - odd) - uv run --quiet --frozen --no-dev --project "$backend" \ - darrow-render-plan-frontier --evidence \ - >"$actual" 2>"$errors" || status=$? - ;; - unknown) - uv run --quiet --frozen --no-dev --project "$backend" \ - darrow-render-plan-frontier --unknown value \ - >"$actual" 2>"$errors" || status=$? - ;; - esac - test "$status" -eq 2 - test ! -s "$actual" - test "$(cat "$errors")" = "$usage" -done diff --git a/plugins/capability/darrow-git/.claude-plugin/plugin.json b/plugins/capability/darrow-git/.claude-plugin/plugin.json index 13ddf8ed..412fa770 100644 --- a/plugins/capability/darrow-git/.claude-plugin/plugin.json +++ b/plugins/capability/darrow-git/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "darrow-git", "description": "Intent-triggered Git skills for branches, commits, PRs, and candidate-bound PR evidence", - "version": "0.8.3", + "version": "0.8.4", "hooks": "./.claude-plugin/hooks.json", "license": "BUSL-1.1" } diff --git a/plugins/capability/darrow-git/.codex-plugin/plugin.json b/plugins/capability/darrow-git/.codex-plugin/plugin.json index 686deabc..17040b40 100644 --- a/plugins/capability/darrow-git/.codex-plugin/plugin.json +++ b/plugins/capability/darrow-git/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-git", - "version": "0.8.3", + "version": "0.8.4", "description": "Intent-triggered Git skills for branches, commits, PRs, and candidate-bound PR evidence", "author": { "name": "Björn Rochel" diff --git a/plugins/capability/darrow-git/README.md b/plugins/capability/darrow-git/README.md index c58eb231..ffb4d22a 100644 --- a/plugins/capability/darrow-git/README.md +++ b/plugins/capability/darrow-git/README.md @@ -132,10 +132,12 @@ apart, without repeating a push or PR creation. property tests, and separate 95% statement/branch coverage gates. The package has no runtime dependencies. Fresh-install checks copy the whole plugin and exercise all five workflows with real local Git repositories and mocked GitHub -boundaries, using `tests/fresh-install.test.sh` on POSIX and -`tests/fresh-install.test.ps1` in native PowerShell. CI covers Python 3.10–3.13 +boundaries, using `uv run --quiet --frozen --no-dev --project backend python backend/tests/fresh_install.py` +on every supported platform. CI covers Python 3.10–3.13 on Linux, macOS, and Windows. Additional POSIX regression scenarios under `backend/tests/shell/` invoke the same UV entrypoints directly. +Publication evidence regressions run in pytest through the console API, real Git, +and an independent fake GitHub process, without a shell test driver. ## Installation diff --git a/plugins/capability/darrow-git/backend/tests/evidence_gh.py b/plugins/capability/darrow-git/backend/tests/evidence_gh.py new file mode 100644 index 00000000..675afde0 --- /dev/null +++ b/plugins/capability/darrow-git/backend/tests/evidence_gh.py @@ -0,0 +1,143 @@ +"""Independent fake GitHub CLI for publication regression tests.""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +def git(repo: Path, *arguments: str) -> str: + return subprocess.check_output( + ["git", "-C", str(repo), *arguments], text=True + ).strip() + + +def append(path: Path, value: str) -> None: + with path.open("a", encoding="utf-8") as stream: + stream.write(value + "\n") + + +def comment_help() -> str: + if os.environ.get("NO_ATTACH") == "1": + return "flags: --body-file" + if os.environ.get("DISTRACTING_LIMIT") == "1": + return ( + "global maximum 80\nflags: --body-file --attach file\n" + " Uploads attachments (Maximum: 10)\n\nflags: --body-file" + ) + maximum = 10 if os.environ.get("STRICT_ATTACH") == "1" else 50 + return f"flags: --body-file --attach file (maximum {maximum})" + + +def attachments(arguments: list[str], directory: Path) -> tuple[Path, list[str]]: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--body-file", required=True, type=Path) + parser.add_argument("--attach", action="append", default=[]) + parsed, _ = parser.parse_known_args(arguments[2:]) + files = [] + for attached in parsed.attach: + append(directory / "attach-order", attached) + files.append(attached.split("#", 1)[0]) + (directory / "attach-files").write_text( + "".join(path + "\n" for path in files), encoding="utf-8" + ) + return parsed.body_file, files + + +def render(body: Path, files: list[str], mode: str) -> str: + text = body.read_bytes().decode("utf-8") + for index, attached in enumerate(files, 1): + number = 1 if mode == "duplicate-url" else index + text = text.replace( + Path(attached).name, + f"https://github.com/user-attachments/assets/mock-{number}", + ) + if mode == "extra": + text += "\nextra remote text\n" + return text + + +def comment_record(body: str) -> str: + escaped = ( + body.removesuffix("\n") + .replace("\\", "\\\\") + .replace("\r", "\\r") + .replace("\t", "\\t") + .replace("\n", "\\n") + ) + return ( + "9\thttps://github.com/fixture/repo/pull/42#issuecomment-9\t" + escaped + "\n" + ) + + +def publish(repo: Path, directory: Path, arguments: list[str]) -> None: + append(directory / "comment-calls", " ".join(arguments)) + body, files = attachments(arguments, directory) + mode = os.environ.get("COMMENT_MODE", "") + if mode == "fail": + raise SystemExit(1) + if mode == "partial": + record = comment_record( + f"" + ) + (directory / "comments").write_text(record, encoding="utf-8") + raise SystemExit(1) + text = render(body, files, mode) + (directory / "rendered-body").write_bytes(text.encode("utf-8")) + (directory / "comments").write_text(comment_record(text), encoding="utf-8") + if mode == "head-change": + append(repo / "tracked", "changed") + git(repo, "add", "tracked") + git(repo, "commit", "-qm", "test: concurrent head change") + + +def query(repo: Path, arguments: list[str]) -> str: + if arguments[:2] == ["repo", "view"]: + host = ( + "github.example" + if os.environ.get("HOST_MODE") == "enterprise" + else "github.com" + ) + return f"fixture/repo\thttps://{host}/fixture/repo" + ref = "main" if os.environ.get("FORGE_HEAD_MODE") == "wrong" else "HEAD" + return f"42\thttps://github.com/fixture/repo/pull/42\t{git(repo, 'rev-parse', ref)}\tfalse" + + +def main() -> None: + repo = Path( + os.environ.get("DARROW_EVIDENCE_REPOSITORY") + or git(Path.cwd(), "rev-parse", "--show-toplevel") + ) + directory = Path(git(repo, "rev-parse", "--absolute-git-dir")) + arguments = sys.argv[1:] + pair = arguments[:2] + if pair in (["repo", "view"], ["pr", "list"]): + print(query(repo, arguments)) + elif pair == ["pr", "comment"]: + comment(repo, directory, arguments) + elif arguments[0] == "api": + api(directory, arguments) + else: + print("unsupported gh call: " + " ".join(arguments), file=sys.stderr) + raise SystemExit(80) + + +def comment(repo: Path, directory: Path, arguments: list[str]) -> None: + if arguments[2:] == ["--help"]: + print(comment_help()) + else: + publish(repo, directory, arguments) + + +def api(directory: Path, arguments: list[str]) -> None: + if arguments[:2] == ["api", "meta"]: + return + append(directory / "api-calls", " ".join(arguments)) + comments = directory / "comments" + if comments.exists(): + sys.stdout.write(comments.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + main() diff --git a/plugins/capability/darrow-git/backend/tests/shell/publish-pr-evidence-publish.test.sh b/plugins/capability/darrow-git/backend/tests/shell/publish-pr-evidence-publish.test.sh deleted file mode 100644 index 708bd234..00000000 --- a/plugins/capability/darrow-git/backend/tests/shell/publish-pr-evidence-publish.test.sh +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -backend=$(cd "$(dirname "$0")/../.." && pwd -P) -ENTRYPOINT=(uv run --quiet --frozen --no-dev --project "$backend" darrow-publish-pr-evidence) -test_parent=${TMPDIR:-/tmp}; case "$test_parent" in /) template=/publish-pr-evidence-test.XXXXXX ;; */) template=${test_parent}publish-pr-evidence-test.XXXXXX ;; *) template=$test_parent/publish-pr-evidence-test.XXXXXX ;; esac -work=$(mktemp -d "$template"); work=$(cd "$work" && pwd -P) -trap '[[ "${KEEP_TEST_FILES:-}" == 1 ]] || rm -rf "$work"' EXIT -base_path=$PATH -mkdir "$work/bin" "$work/evidence" - -cat > "$work/bin/gh" <<'EOF' -#!/usr/bin/env bash -set -euo pipefail -repo_top=${DARROW_EVIDENCE_REPOSITORY:-$(git rev-parse --show-toplevel)} -git_dir=$(git -C "$repo_top" rev-parse --absolute-git-dir) -case "$1 $2" in - 'repo view') - if [[ "${HOST_MODE:-}" == enterprise ]]; then printf 'fixture/repo\thttps://github.example/fixture/repo\n'; else printf 'fixture/repo\thttps://github.com/fixture/repo\n'; fi - ;; - 'pr list') - tip=$(git -C "$repo_top" rev-parse HEAD) - [[ "${FORGE_HEAD_MODE:-}" != wrong ]] || tip=$(git -C "$repo_top" rev-parse main) - printf '42\thttps://github.com/fixture/repo/pull/42\t%s\tfalse\n' "$tip" - ;; - 'pr comment') - if [[ "${NO_ATTACH:-}" == 1 && "${3:-}" == --help ]]; then printf '%s\n' 'flags: --body-file'; exit 0; fi - if [[ "${3:-}" == --help ]]; then - if [[ "${DISTRACTING_LIMIT:-}" == 1 ]]; then printf '%s\n' 'global maximum 80' 'flags: --body-file --attach file' ' Uploads attachments (Maximum: 10)' '' 'flags: --body-file' - elif [[ "${STRICT_ATTACH:-}" == 1 ]]; then printf '%s\n' 'flags: --body-file --attach file (maximum 10)' - else printf '%s\n' 'flags: --body-file --attach file (maximum 50)'; fi - exit 0 - fi - printf '%s\n' "$*" >> "$git_dir/comment-calls" - body=''; attach_count=0; rendered="$git_dir/rendered-body"; : > "$git_dir/attach-files" - shift 2 - while [[ $# -gt 0 ]]; do case "$1" in --body-file) body=$2; shift 2 ;; --attach) attach_count=$((attach_count + 1)); printf '%s\n' "$2" >> "$git_dir/attach-order"; printf '%s\n' "${2%%#*}" >> "$git_dir/attach-files"; shift 2 ;; *) shift ;; esac; done - [[ "${COMMENT_MODE:-}" != fail ]] || exit 1 - if [[ "${COMMENT_MODE:-}" == partial ]]; then printf '9\thttps://github.com/fixture/repo/pull/42#issuecomment-9\t\n' "$(git -C "$repo_top" rev-parse HEAD)" > "$git_dir/comments"; exit 1; fi - cp "$body" "$rendered" - i=1 - while IFS= read -r attached; do - old=${attached##*/}; url="https://github.com/user-attachments/assets/mock-$i" - [[ "${COMMENT_MODE:-}" != duplicate-url || "$i" -eq 1 ]] || url=https://github.com/user-attachments/assets/mock-1 - awk -v old="$old" -v new="$url" '{s=$0; out=""; while ((p=index(s,old))>0) {out=out substr(s,1,p-1) new; s=substr(s,p+length(old))} print out s}' "$rendered" > "$rendered.next" - mv "$rendered.next" "$rendered"; i=$((i+1)) - done < "$git_dir/attach-files" - [[ "${COMMENT_MODE:-}" != extra ]] || printf '\nextra remote text\n' >> "$rendered" - { printf '9\thttps://github.com/fixture/repo/pull/42#issuecomment-9\t'; awk '{gsub(/\\/, "\\\\"); gsub(/\r/, "\\r"); gsub(/\t/, "\\t"); if (NR>1) printf "\\n"; printf "%s",$0}' "$rendered"; printf '\n'; } > "$git_dir/comments" - if [[ "${COMMENT_MODE:-}" == head-change ]]; then printf 'changed\n' >>"$repo_top/tracked"; git -C "$repo_top" add tracked; git -C "$repo_top" commit -qm 'test: concurrent head change'; fi - ;; - 'api meta') exit 0 ;; - api*) printf '%s\n' "$*" >> "$git_dir/api-calls"; [[ -f "$git_dir/comments" ]] && cat "$git_dir/comments" || true ;; - *) echo "unsupported gh call: $*" >&2; exit 80 ;; -esac -EOF -chmod +x "$work/bin/gh" -export PATH="$work/bin:$base_path" - -fresh() { - repo="$work/repo-${1}" - mkdir "$repo"; cd "$repo" - git init -qb main; git config user.name Fixture; git config user.email fixture@example.invalid; git config commit.gpgsign false - printf 'base\n' > tracked; git add tracked; git commit -qm 'chore: base' - git switch -qc feat/evidence; printf 'feature\n' >> tracked; git commit -qam 'feat: evidence' - expected=$(git rev-parse HEAD) - body="$work/evidence/body-$1.txt"; printf 'Verification passed for candidate %s.\n' "$expected" > "$body" - unset NO_ATTACH STRICT_ATTACH DISTRACTING_LIMIT HOST_MODE FORGE_HEAD_MODE COMMENT_MODE -} -refuses() { if "${ENTRYPOINT[@]}" "$@" >"$work/out" 2>&1; then echo "expected refusal: $*" >&2; exit 1; fi; grep -Eq '^outcome: (refused|partial|ambiguous)$' "$work/out"; } - -fresh 1 -if ! "${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" >"$work/out"; then cat "$work/out"; exit 1; fi -grep -Fx 'outcome: published' "$work/out" -grep -Fx 'attachment-count: 0' "$work/out" -grep -Fx 'head-preserved: true' "$work/out"; grep -Fx 'index-preserved: true' "$work/out" -grep -Fq '.body] | @tsv' .git/api-calls -if grep -Fq 'gsub' .git/api-calls; then echo 'comment observation must use one TSV escape layer' >&2; exit 1; fi -test -f "$(sed -n 's/^prepared-body: //p' "$work/out")" -test "$(tail -c 1 "$(sed -n 's/^prepared-body: //p' "$work/out")" | od -An -tx1 | tr -d '[:space:]')" = 3e -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 -"${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" >"$work/out" -grep -Fx 'outcome: existing' "$work/out" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 -cat .git/comments >>.git/comments.copy; cat .git/comments >>.git/comments.copy; mv .git/comments.copy .git/comments -refuses publish --expected-head "$expected" --body-file "$body" -grep -Fx 'outcome: ambiguous' "$work/out" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 - -fresh 2 -png="$work/evidence/view.png"; jpg="$work/evidence/detail.svg"; video="$work/evidence/run.mp4" -printf '\211PNG\r\n\032\n\000\000\000\015IHDR\000\000\000\001\000\000\000\001\010\006\000\000\000\037\025\304\211\000\000\000\015IDAT\010\327c\370\317\300\360\037\000\005\000\001\377\211\231\075\035\000\000\000\000IEND\256B\140\202' > "$png" -printf '\n' > "$jpg" -printf '\000\000\000\030ftypmp42\000\000\000\000mp42isom\000\000\000\010mdat' > "$video" -"${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" --image "$png" --alt 'Overview screenshot' --video "$video" --explanation 'Recording of the verified flow' --image "$jpg" --alt 'Detailed result' >"$work/out" -grep -Fx 'outcome: published' "$work/out"; grep -Fx 'attachment-count: 3' "$work/out" -grep -F 'Image 1 alt text: Overview screenshot' "$(sed -n 's/^prepared-body: //p' "$work/out")" -grep -F "content identity $(git hash-object "$png")" "$(sed -n 's/^prepared-body: //p' "$work/out")" -grep -F 'Video 2 explanation: Recording of the verified flow' "$(sed -n 's/^prepared-body: //p' "$work/out")" -test 'darrow-evidence-attachment-1.png#Overview screenshot' = "$(sed -n '1p' .git/attach-order)" -test 'darrow-evidence-attachment-2.mp4' = "$(sed -n '2p' .git/attach-order)" -test 'darrow-evidence-attachment-3.svg#Detailed result' = "$(sed -n '3p' .git/attach-order)" -test -f "$png" && test -f "$jpg" && test -f "$video" - -fresh 3 -png_a="$work/evidence/a.svg"; png_b="$work/evidence/b.svg"; printf '\n' > "$png_a"; cp "$png_a" "$png_b" -refuses publish --expected-head "$expected" --body-file "$body" --image "$png_a" --alt one --image "$png_b" --alt two -empty="$work/evidence/empty.svg"; : > "$empty"; refuses publish --expected-head "$expected" --body-file "$body" --image "$empty" --alt empty -missing="$work/evidence/missing.svg"; refuses publish --expected-head "$expected" --body-file "$body" --image "$missing" --alt missing -unsupported="$work/evidence/data.txt"; printf x > "$unsupported"; refuses publish --expected-head "$expected" --body-file "$body" --image "$unsupported" --alt data -renamed="$work/evidence/not-image.png"; printf 'plain text\n' > "$renamed"; refuses publish --expected-head "$expected" --body-file "$body" --image "$renamed" --alt invalid -truncated="$work/evidence/truncated.png"; printf '\211PNG\r\n\032\n' > "$truncated"; refuses publish --expected-head "$expected" --body-file "$body" --image "$truncated" --alt truncated -no_ihdr="$work/evidence/no-ihdr.png"; printf '\211PNG\r\n\032\n\000\000\000\025tEXt000000000000000000000\000\000\000\000IEND\256B\140\202' > "$no_ihdr"; refuses publish --expected-head "$expected" --body-file "$body" --image "$no_ihdr" --alt malformed -zero_width="$work/evidence/zero-width.png"; printf '\211PNG\r\n\032\n\000\000\000\015IHDR\000\000\000\000\000\000\000\001\010\006\000\000\000\037\025\304\211\000\000\000\015IDAT\010\327c\370\317\300\360\037\000\005\000\001\377\211\231\075\035\000\000\000\000IEND\256B\140\202' > "$zero_width"; refuses publish --expected-head "$expected" --body-file "$body" --image "$zero_width" --alt invalid-width -zero_height="$work/evidence/zero-height.png"; printf '\211PNG\r\n\032\n\000\000\000\015IHDR\000\000\000\001\000\000\000\000\010\006\000\000\000\037\025\304\211\000\000\000\015IDAT\010\327c\370\317\300\360\037\000\005\000\001\377\211\231\075\035\000\000\000\000IEND\256B\140\202' > "$zero_height"; refuses publish --expected-head "$expected" --body-file "$body" --image "$zero_height" --alt invalid-height -refuses publish --expected-head "$expected" --body-file "$body" --image "$png_a" --alt ' ' -refuses publish --expected-head "$expected" --body-file "$body" --image "$png_a" --alt $'one\nattachment-2=forged' -oversize="$work/evidence/large.svg"; printf '' > "$oversize"; dd if=/dev/zero bs=1048577 count=10 >>"$oversize" 2>/dev/null -refuses publish --expected-head "$expected" --body-file "$body" --image "$oversize" --alt large - -fresh 4 -NO_ATTACH=1; export NO_ATTACH -refuses publish --expected-head "$expected" --body-file "$body" -unset NO_ATTACH -STRICT_ATTACH=1; export STRICT_ATTACH -refuses publish --expected-head "$expected" --body-file "$body" -unset STRICT_ATTACH -DISTRACTING_LIMIT=1; export DISTRACTING_LIMIT -refuses publish --expected-head "$expected" --body-file "$body" -unset DISTRACTING_LIMIT -HOST_MODE=enterprise; export HOST_MODE -refuses publish --expected-head "$expected" --body-file "$body" -unset HOST_MODE -FORGE_HEAD_MODE=wrong; export FORGE_HEAD_MODE -refuses publish --expected-head "$expected" --body-file "$body" -unset FORGE_HEAD_MODE - -fresh 5 -p1="$work/evidence/path-one.svg"; p2="$work/evidence/path-two.svg"; printf 'identity\n' > "$p1"; cp "$p1" "$p2" -"${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" --image "$p1" --alt 'Same presentation' >"$work/one" -rm -f .git/comments .git/comment-calls .git/attach-order -"${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" --image "$p2" --alt 'Same presentation' >"$work/two" -test "$(sed -n 's/^identity: //p' "$work/one")" = "$(sed -n 's/^identity: //p' "$work/two")" - -fresh 6 -COMMENT_MODE=partial; export COMMENT_MODE -refuses publish --expected-head "$expected" --body-file "$body" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 - -fresh 11 -COMMENT_MODE=extra; export COMMENT_MODE -refuses publish --expected-head "$expected" --body-file "$body" -grep -Fx 'outcome: partial' "$work/out" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 - -fresh 12 -COMMENT_MODE=duplicate-url; export COMMENT_MODE -dup_one="$work/evidence/unique-one.svg"; dup_two="$work/evidence/unique-two.svg" -printf 'one\n' > "$dup_one" -printf 'two\n' > "$dup_two" -refuses publish --expected-head "$expected" --body-file "$body" --image "$dup_one" --alt one --image "$dup_two" --alt two -grep -Fx 'outcome: partial' "$work/out" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 - -fresh 13 -printf 'Evidence and https://github.com/user-attachments/assets/stale\n' > "$body" -refuses publish --expected-head "$expected" --body-file "$body" -test ! -f .git/comment-calls - -fresh 14 -printf 'Windows path C:\\tmp\r\nSecond line.\n' > "$body" -"${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" >"$work/out" -grep -Fx 'outcome: published' "$work/out" -"${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" >"$work/out" -grep -Fx 'outcome: existing' "$work/out" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 - -fresh 8 -COMMENT_MODE=head-change; export COMMENT_MODE -refuses publish --expected-head "$expected" --body-file "$body" -grep -Fx 'outcome: partial' "$work/out" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 -unset COMMENT_MODE -unset COMMENT_MODE -refuses publish --expected-head "$expected" --body-file "$body" -test "$(wc -l < .git/comment-calls | tr -d ' ')" -eq 1 - -fresh 7 -many=(); i=1 -while [[ "$i" -le 51 ]]; do f="$work/evidence/count-$i.svg"; printf '%s\n' "$i" > "$f"; many+=(--image "$f" --alt "image $i"); i=$((i+1)); done -refuses publish --expected-head "$expected" --body-file "$body" "${many[@]}" - -mkdir "$work/tmp-parent" -fresh 9 -TMPDIR="$work/tmp-parent" "${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" >"$work/out" -grep -Fx 'outcome: published' "$work/out" -fresh 10 -TMPDIR="$work/tmp-parent/" "${ENTRYPOINT[@]}" publish --expected-head "$expected" --body-file "$body" >"$work/out" -grep -Fx 'outcome: published' "$work/out" - -echo 'all publish-pr-evidence checks passed' diff --git a/plugins/capability/darrow-git/backend/tests/test_evidence_fixture.py b/plugins/capability/darrow-git/backend/tests/test_evidence_fixture.py new file mode 100644 index 00000000..f9f39c4c --- /dev/null +++ b/plugins/capability/darrow-git/backend/tests/test_evidence_fixture.py @@ -0,0 +1,31 @@ +"""Serialization contracts for the independent publication fixture.""" + +import re +from pathlib import Path + +from hypothesis import given, settings +from hypothesis import strategies as st + +from evidence_gh import comment_record, render + + +@given(st.text()) +@settings(derandomize=True) +def test_comment_tsv_round_trip(body: str) -> None: + identifier, url, field = comment_record(body).removesuffix("\n").split("\t") + assert identifier == "9" + assert url.endswith("#issuecomment-9") + decoded = re.sub( + r"\\([\\rtn])", + lambda match: {"\\": "\\", "r": "\r", "t": "\t", "n": "\n"}[match[1]], + field, + ) + assert decoded == body.removesuffix("\n") + + +def test_attachment_names_are_literal_and_content_bytes_survive(tmp_path: Path) -> None: + path = tmp_path / "body" + path.write_bytes(b"[image](file[1].svg)\r\nC:\\tmp\n") + assert render(path, ["file[1].svg"], "") == ( + "[image](https://github.com/user-attachments/assets/mock-1)\r\nC:\\tmp\n" + ) diff --git a/plugins/capability/darrow-git/backend/tests/test_evidence_publication.py b/plugins/capability/darrow-git/backend/tests/test_evidence_publication.py new file mode 100644 index 00000000..a6aff316 --- /dev/null +++ b/plugins/capability/darrow-git/backend/tests/test_evidence_publication.py @@ -0,0 +1,347 @@ +"""Publication contracts through the console API, real Git, and a fake gh process.""" + +import importlib +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from darrow_git import cli, process +from darrow_git.process import git + +INVOKE = process.invoke +FAKE_GH = Path(__file__).with_name("evidence_gh.py") +PNG = bytes.fromhex( + "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489" + "0000000d4944415408d763f8cfc0f01f00050001ff89993d1d0000000049454e44ae426082" +) +SVG = b'\n' + + +def provider( + args: Sequence[str], + *, + cwd: Path | None = None, + input_bytes: bytes | None = None, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + command = [sys.executable, str(FAKE_GH), *args[1:]] if args[0] == "gh" else args + return INVOKE(command, cwd=cwd, input_bytes=input_bytes, env=env) + + +@dataclass +class Publication: + repository: Path + body: Path + head: str + monkeypatch: pytest.MonkeyPatch + capture: pytest.CaptureFixture[str] + + def publish(self, *args: str, outcome: str = "published") -> dict[str, str]: + self.monkeypatch.setattr( + sys, + "argv", + [ + "darrow-publish-pr-evidence", + "publish", + "--expected-head", + self.head, + "--body-file", + str(self.body), + *args, + ], + ) + with pytest.raises(SystemExit) as result: + cli.publish_evidence() + output = self.capture.readouterr() + assert (result.value.code == 0) == (outcome in {"published", "existing"}), ( + output + ) + fields = dict( + line.split(": ", 1) for line in output.out.splitlines() if ": " in line + ) + assert fields["outcome"] == outcome, output + return fields + + def record(self, name: str) -> Path: + return self.repository / ".git" / name + + def calls(self) -> list[str]: + return self.record("comment-calls").read_text(encoding="utf-8").splitlines() + + def image(self, name: str, content: bytes = SVG) -> Path: + path = self.body.parent / name + path.write_bytes(content) + return path + + +@pytest.fixture +def publication( + repository: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> Publication: + git("switch", "-qc", "feat/evidence") + (repository / "tracked.txt").write_text("feature\n", encoding="utf-8") + git("commit", "-qam", "feat: evidence") + head = git("rev-parse", "HEAD") + body = tmp_path / "evidence body.txt" + body.write_text(f"Verification passed for candidate {head}.\n", encoding="utf-8") + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) + for name in ( + "NO_ATTACH", + "STRICT_ATTACH", + "DISTRACTING_LIMIT", + "HOST_MODE", + "FORGE_HEAD_MODE", + "COMMENT_MODE", + "DARROW_EVIDENCE_REPOSITORY", + ): + monkeypatch.delenv(name, raising=False) + for path in Path(process.__file__).parent.glob("*.py"): + module = importlib.import_module(f"darrow_git.{path.stem}") + if hasattr(module, "invoke"): + monkeypatch.setattr(module, "invoke", provider) + which = shutil.which + monkeypatch.setattr( + "shutil.which", lambda name: sys.executable if name == "gh" else which(name) + ) + return Publication(repository, body, head, monkeypatch, capsys) + + +def test_publish_deduplicate_and_refuse_ambiguous_comments( + publication: Publication, +) -> None: + result = publication.publish() + assert result["attachment-count"] == "0" + assert result["head-preserved"] == result["index-preserved"] == "true" + assert Path(result["prepared-body"]).read_bytes().endswith(b">") + calls = publication.record("api-calls").read_text(encoding="utf-8") + assert ".body] | @tsv" in calls and "gsub" not in calls + publication.publish(outcome="existing") + assert len(publication.calls()) == 1 + comments = publication.record("comments") + comments.write_bytes(comments.read_bytes() * 2) + publication.publish(outcome="ambiguous") + assert len(publication.calls()) == 1 + + +def test_ordered_attachments_preserve_sources(publication: Publication) -> None: + image = publication.image("view.png", PNG) + detail = publication.image("detail.svg") + video = publication.image( + "run.mp4", + bytes.fromhex( + "00000018667479706d703432000000006d70343269736f6d000000086d646174" + ), + ) + sources = {path: path.read_bytes() for path in (image, video, detail)} + result = publication.publish( + "--image", + str(image), + "--alt", + "Overview screenshot", + "--video", + str(video), + "--explanation", + "Recording of the verified flow", + "--image", + str(detail), + "--alt", + "Detailed result", + ) + assert result["attachment-count"] == "3" + body = Path(result["prepared-body"]).read_text(encoding="utf-8") + assert "Image 1 alt text: Overview screenshot" in body + assert f"content identity {git('hash-object', str(image))}" in body + assert "Video 2 explanation: Recording of the verified flow" in body + assert publication.record("attach-order").read_text( + encoding="utf-8" + ).splitlines() == [ + "darrow-evidence-attachment-1.png#Overview screenshot", + "darrow-evidence-attachment-2.mp4", + "darrow-evidence-attachment-3.svg#Detailed result", + ] + assert {path: path.read_bytes() for path in sources} == sources + + +@pytest.mark.parametrize( + "name,content", + [ + ("empty.svg", b""), + ("unsupported.txt", b"x"), + ("not-image.png", b"plain text\n"), + ("truncated.png", PNG[:8]), + ( + "no-ihdr.png", + PNG[:8] + bytes.fromhex("0000001574455874") + b"0" * 21 + PNG[-12:], + ), + ("zero-width.png", PNG[:16] + b"\0" * 4 + PNG[20:]), + ("zero-height.png", PNG[:20] + b"\0" * 4 + PNG[24:]), + ], + ids=[ + "empty", + "unsupported", + "renamed", + "truncated", + "no-ihdr", + "zero-width", + "zero-height", + ], +) +def test_invalid_media_refused_before_comment( + publication: Publication, name: str, content: bytes +) -> None: + path = publication.image(name, content) + publication.publish("--image", str(path), "--alt", "invalid", outcome="refused") + assert not publication.record("comment-calls").exists() + + +def test_oversized_media_refused(publication: Publication) -> None: + path = publication.image("large.svg") + with path.open("r+b") as stream: + stream.truncate(1048577 * 10) + publication.publish("--image", str(path), "--alt", "large", outcome="refused") + assert not publication.record("comment-calls").exists() + + +def test_missing_media_refused(publication: Publication) -> None: + publication.publish( + "--image", + str(publication.body.parent / "missing.svg"), + "--alt", + "missing", + outcome="refused", + ) + assert not publication.record("comment-calls").exists() + + +def test_duplicate_media_refused(publication: Publication) -> None: + one = publication.image("one.svg") + two = publication.image("two.svg") + publication.publish( + "--image", + str(one), + "--alt", + "one", + "--image", + str(two), + "--alt", + "two", + outcome="refused", + ) + + +@pytest.mark.parametrize("alt", [" ", "one\nattachment-2=forged"]) +def test_invalid_alt_refused(publication: Publication, alt: str) -> None: + publication.publish( + "--image", str(publication.image("image.svg")), "--alt", alt, outcome="refused" + ) + + +@pytest.mark.parametrize( + "name,value", + [ + ("NO_ATTACH", "1"), + ("STRICT_ATTACH", "1"), + ("DISTRACTING_LIMIT", "1"), + ("HOST_MODE", "enterprise"), + ("FORGE_HEAD_MODE", "wrong"), + ], +) +def test_provider_refusals(publication: Publication, name: str, value: str) -> None: + publication.monkeypatch.setenv(name, value) + publication.publish(outcome="refused") + assert not publication.record("comment-calls").exists() + + +def test_identity_independent_of_source_path(publication: Publication) -> None: + one = publication.image("path-one.svg") + two = publication.image("path-two.svg") + first = publication.publish("--image", str(one), "--alt", "Same presentation") + for name in ("comments", "comment-calls", "attach-order"): + publication.record(name).unlink() + second = publication.publish("--image", str(two), "--alt", "Same presentation") + assert first["identity"] == second["identity"] + + +@pytest.mark.parametrize( + "mode,outcome", + [("partial", "ambiguous"), ("extra", "partial"), ("duplicate-url", "partial")], +) +def test_uncertain_remote_body_prevents_retry( + publication: Publication, mode: str, outcome: str +) -> None: + publication.monkeypatch.setenv("COMMENT_MODE", mode) + arguments = [] + if mode == "duplicate-url": + one = publication.image("one.svg", SVG.replace(b"M0 0", b"M1 1")) + two = publication.image("two.svg") + arguments = [ + "--image", + str(one), + "--alt", + "one", + "--image", + str(two), + "--alt", + "two", + ] + publication.publish(*arguments, outcome=outcome) + assert len(publication.calls()) == 1 + + +def test_stale_attachment_url_refused(publication: Publication) -> None: + publication.body.write_text( + "Evidence and https://github.com/user-attachments/assets/stale\n", + encoding="utf-8", + ) + publication.publish(outcome="refused") + assert not publication.record("comment-calls").exists() + + +def test_crlf_and_backslashes_survive_deduplication(publication: Publication) -> None: + publication.body.write_bytes(b"Windows path C:\\tmp\r\nSecond line.\n") + publication.publish() + publication.publish(outcome="existing") + assert len(publication.calls()) == 1 + + +def test_concurrent_head_change_prevents_republication( + publication: Publication, +) -> None: + publication.monkeypatch.setenv("COMMENT_MODE", "head-change") + publication.publish(outcome="partial") + assert len(publication.calls()) == 1 + publication.monkeypatch.delenv("COMMENT_MODE") + publication.publish(outcome="refused") + assert len(publication.calls()) == 1 + + +def test_attachment_count_limit(publication: Publication) -> None: + arguments = [] + for number in range(51): + path = publication.image( + f"count-{number}.svg", SVG.replace(b"M0 0", f"M{number} 0".encode()) + ) + arguments.extend(["--image", str(path), "--alt", f"image {number}"]) + publication.publish(*arguments, outcome="refused") + + +@pytest.mark.parametrize("suffix", ["", os.sep]) +def test_temporary_parent_with_optional_separator( + publication: Publication, suffix: str +) -> None: + parent = publication.body.parent / "temporary parent" + parent.mkdir() + publication.monkeypatch.setenv("TMPDIR", str(parent) + suffix) + publication.monkeypatch.setattr(tempfile, "tempdir", None) + result = publication.publish() + assert Path(result["prepared-body"]).parent == parent.resolve() diff --git a/plugins/capability/darrow-git/tests/fresh-install.test.ps1 b/plugins/capability/darrow-git/tests/fresh-install.test.ps1 deleted file mode 100644 index 8d25f14e..00000000 --- a/plugins/capability/darrow-git/tests/fresh-install.test.ps1 +++ /dev/null @@ -1,4 +0,0 @@ -$ErrorActionPreference = "Stop" -$pluginRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -& uv run --quiet --frozen --no-dev --project (Join-Path $pluginRoot "backend") python (Join-Path $pluginRoot "backend/tests/fresh_install.py") -if ($LASTEXITCODE -ne 0) { throw "Fresh Git plugin validation failed: $LASTEXITCODE" } diff --git a/plugins/capability/darrow-git/tests/fresh-install.test.sh b/plugins/capability/darrow-git/tests/fresh-install.test.sh deleted file mode 100644 index b30113a5..00000000 --- a/plugins/capability/darrow-git/tests/fresh-install.test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -plugin_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) -exec uv run --quiet --frozen --no-dev --project "$plugin_root/backend" \ - python "$plugin_root/backend/tests/fresh_install.py" diff --git a/plugins/capability/darrow-observability-langfuse/.claude-plugin/plugin.json b/plugins/capability/darrow-observability-langfuse/.claude-plugin/plugin.json index 2a53f91c..0010a168 100644 --- a/plugins/capability/darrow-observability-langfuse/.claude-plugin/plugin.json +++ b/plugins/capability/darrow-observability-langfuse/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "darrow-observability-langfuse", "description": "Export Codex rollout turns to Langfuse with attribution epochs", - "version": "0.5.2", + "version": "0.5.3", "license": "BUSL-1.1", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-observability-langfuse/.codex-plugin/plugin.json b/plugins/capability/darrow-observability-langfuse/.codex-plugin/plugin.json index b47cbcff..11e866b5 100644 --- a/plugins/capability/darrow-observability-langfuse/.codex-plugin/plugin.json +++ b/plugins/capability/darrow-observability-langfuse/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-observability-langfuse", - "version": "0.5.2", + "version": "0.5.3", "description": "Export Codex rollout turns to Langfuse with attribution epochs", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-observability-langfuse/README.md b/plugins/capability/darrow-observability-langfuse/README.md index 797ca52a..1e478b6f 100644 --- a/plugins/capability/darrow-observability-langfuse/README.md +++ b/plugins/capability/darrow-observability-langfuse/README.md @@ -15,7 +15,7 @@ plugin. The hook never contacts or mutates a tracker. ## Runtime -`hooks/stop.sh` is a portable launcher, while +`hooks/stop.sh` is a Unix Bash launcher, while transcript reconstruction and Langfuse export run in Python managed by UV. The plugin commits `backend/pyproject.toml` and `backend/uv.lock`; no sibling plugin or repository runtime is required. @@ -256,15 +256,12 @@ legacy trace-list API. ## Development checks ```sh -bash tests/package.test.sh -bash hooks/stop.test.sh -bash hooks/export-failure.test.sh -bash hooks/refusal.test.sh -bash hooks/strict.test.sh +uv run --quiet --frozen --all-groups --project backend pytest backend/tests/test_packaged_hooks.py backend/tests/test_hook_failures.py +uv run --quiet --frozen --no-dev --project backend python backend/tests/fresh_install.py bun run check:python ``` -Run each shell test with both `bash` and `/bin/bash` in repository development. +The hook tests invoke both `bash` and `/bin/bash` from pytest. The repository command verifies the UV lock, formatting, lint, strict typing, tests, property tests, and separate statement and branch coverage gates. @@ -277,6 +274,9 @@ Configure or explain Codex turn telemetry, privacy, and attribution. Do not use Codex is the observed runtime. Export requires Codex async hooks, [UV and Python](https://github.com/BjRo/darrow/blob/main/docs/installing-plugins.md#uv-and-python-for-plugin-helpers), and a compatible Langfuse v4 server or Langfuse Cloud. Claude Code turns are not exported. +The hook requires a Unix environment with Bash; the backend uses Unix file +locking. Native Windows support is tracked separately in +[#205](https://github.com/BjRo/darrow/issues/205). Claude installation and guidance invocation are unverified: Claude Code 2.1.223 rejects this package's Codex-specific `Interrupt` hook during native validation. The presence of a Claude manifest is not a compatibility guarantee. diff --git a/plugins/capability/darrow-observability-langfuse/backend/tests/fresh_install.py b/plugins/capability/darrow-observability-langfuse/backend/tests/fresh_install.py new file mode 100644 index 00000000..b255a30c --- /dev/null +++ b/plugins/capability/darrow-observability-langfuse/backend/tests/fresh_install.py @@ -0,0 +1,81 @@ +"""Validate a copied Unix plugin using only its locked runtime dependencies.""" + +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +def command(*args: str) -> str: + return subprocess.check_output(args, text=True, encoding="utf-8") + + +def validate(plugin: Path, fixture: Path) -> None: + backend = plugin / "backend" + command("uv", "sync", "--frozen", "--no-dev", "--project", str(backend)) + tree = command("uv", "tree", "--frozen", "--no-dev", "--project", str(backend)) + assert not any( + tool in tree for tool in ("coverage", "hypothesis", "mypy", "pytest", "ruff") + ) + command( + "uv", + "run", + "--quiet", + "--frozen", + "--no-dev", + "--project", + str(backend), + "python", + "-c", + "import darrow_observability_langfuse", + ) + hooks = json.loads((plugin / "hooks/hooks.json").read_text(encoding="utf-8")) + capture = hooks["hooks"]["Stop"][0]["hooks"][0]["command"] + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith(("DARROW_LANGFUSE_", "LANGFUSE_", "UV_")) + } + environment.update( + { + "DARROW_LANGFUSE_ENABLED": "false", + "PLUGIN_ROOT": str(plugin), + "HOME": str(fixture), + } + ) + subprocess.run( + ["bash", "-c", capture], + input="{}\n", + text=True, + env=environment, + cwd=fixture, + check=True, + ) + + +def main() -> None: + plugin = Path(__file__).resolve().parents[2] + ignored = shutil.ignore_patterns( + ".venv", + "__pycache__", + ".hypothesis", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".coverage*", + "coverage.json", + ) + with tempfile.TemporaryDirectory(prefix="darrow langfuse fresh ") as temporary: + fixture = Path(temporary).resolve() + copied = fixture / "plugin copy" + shutil.copytree(plugin, copied, ignore=ignored) + validate(copied, fixture) + print( + "fresh copied Langfuse plugin: runtime dependencies and registered hook passed" + ) + + +if __name__ == "__main__": + main() diff --git a/plugins/capability/darrow-observability-langfuse/backend/tests/test_hook_failures.py b/plugins/capability/darrow-observability-langfuse/backend/tests/test_hook_failures.py new file mode 100644 index 00000000..fd114810 --- /dev/null +++ b/plugins/capability/darrow-observability-langfuse/backend/tests/test_hook_failures.py @@ -0,0 +1,174 @@ +"""Failure and configuration contracts through both supported Unix launchers.""" + +import json +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +PLUGIN = Path(__file__).resolve().parents[2] +pytestmark = pytest.mark.skipif( + os.name == "nt", reason="native Windows support is tracked in #205" +) + + +@dataclass +class Hook: + shell: str + root: Path + project: Path + home: Path + environment: dict[str, str] + + def run( + self, *args: str, payload: str | None = None, **environment: str + ) -> subprocess.CompletedProcess[str]: + document = { + "session_id": "test", + "turn_id": "0", + "hook_event_name": "Stop", + "cwd": str(self.project), + "transcript_path": str(self.root / "missing.jsonl"), + } + return subprocess.run( + [self.shell, str(PLUGIN / "hooks/stop.sh"), *args], + input=json.dumps(document) if payload is None else payload, + text=True, + capture_output=True, + check=False, + cwd=self.project, + env={**self.environment, **environment}, + timeout=60, + ) + + def configure(self, directory: Path) -> None: + (directory / ".codex/darrow-langfuse.json").write_text( + json.dumps({"enabled": True, "strict": True}), + encoding="utf-8", + ) + + def path_without_uv(self) -> str: + directory = self.root / "without uv" + directory.mkdir(exist_ok=True) + for name in ("dirname", "tr", "sed"): + executable = shutil.which(name) + assert executable is not None + destination = directory / name + if not destination.exists(): + destination.symlink_to(executable) + return str(directory) + + +@pytest.fixture(params=["bash", "/bin/bash"]) +def hook(request: pytest.FixtureRequest, tmp_path: Path) -> Hook: + shell = shutil.which(str(request.param)) + assert shell is not None + project, home = tmp_path / "project", tmp_path / "home" + for directory in (project, home): + (directory / ".codex").mkdir(parents=True) + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith( + ("DARROW_LANGFUSE_", "LANGFUSE_", "OTEL_", "UV_", "CODEX_") + ) + } + environment.update( + {"HOME": str(home), "UV_OFFLINE": "1", "CODEX_PLUGIN_ROOT": str(PLUGIN)} + ) + return Hook(shell, tmp_path, project, home, environment) + + +@pytest.mark.parametrize("strict", [False, True]) +def test_missing_uv_respects_strictness(hook: Hook, strict: bool) -> None: + result = hook.run( + payload="", + PATH=hook.path_without_uv(), + DARROW_LANGFUSE_DEBUG="true", + DARROW_LANGFUSE_STRICT=str(strict).lower(), + ) + assert (result.returncode != 0) == strict + assert "uv is required but was not found" in result.stderr + + +def test_broken_uv_fails_open(hook: Hook) -> None: + result = hook.run( + payload="", UV_PROJECT_ENVIRONMENT=os.devnull, DARROW_LANGFUSE_ENABLED="false" + ) + assert result.returncode == 0, result.stderr + + +def test_malformed_input_refused_in_strict_mode(hook: Hook) -> None: + result = hook.run(payload="not-json", DARROW_LANGFUSE_STRICT="true") + assert result.returncode != 0 + assert "hook input is not valid JSON" in result.stderr + + +@pytest.mark.parametrize("location", ["home", "project"]) +def test_file_strict_configuration(hook: Hook, location: str) -> None: + hook.configure(getattr(hook, location)) + result = hook.run() + assert result.returncode != 0 + assert "Langfuse credentials are missing" in result.stderr + + +@pytest.mark.parametrize("strict", ["1", "true", "yes", "on", " TRUE ", " Yes "]) +def test_environment_strict_values_reach_backend_and_launcher( + hook: Hook, strict: str +) -> None: + result = hook.run(DARROW_LANGFUSE_ENABLED="true", DARROW_LANGFUSE_STRICT=strict) + assert result.returncode != 0 + assert "Langfuse credentials are missing" in result.stderr + missing_uv = hook.run( + payload="", PATH=hook.path_without_uv(), DARROW_LANGFUSE_STRICT=strict + ) + assert missing_uv.returncode != 0 + assert "uv is required but was not found" in missing_uv.stderr + + +def test_environment_false_overrides_file_strictness(hook: Hook) -> None: + hook.configure(hook.project) + result = hook.run(DARROW_LANGFUSE_STRICT="off") + assert result.returncode == 0, result.stderr + + +def test_pre_python_failure_cannot_resolve_file_strictness(hook: Hook) -> None: + hook.configure(hook.project) + result = hook.run(UV_PROJECT_ENVIRONMENT=os.devnull) + assert result.returncode == 0, result.stderr + + +def test_capture_succeeds_and_strict_export_failure_redacts_secret(hook: Hook) -> None: + fixtures = PLUGIN / "tests/fixtures" + rollout = hook.project / "rollout.jsonl" + shutil.copyfile(fixtures / "main-rollout.jsonl", rollout) + shutil.copyfile( + fixtures / "rollout-2026-08-24-child-1.jsonl", + hook.project / "rollout-2026-08-24-child-1.jsonl", + ) + hook.configure(hook.project) + payload = json.dumps( + { + "session_id": "session-main", + "turn_id": "turn-1", + "cwd": str(hook.project), + "transcript_path": str(rollout), + "hook_event_name": "Stop", + } + ) + environment = { + "DARROW_LANGFUSE_ENABLED": "true", + "DARROW_LANGFUSE_CAPTURE_CONTENT": "false", + "LANGFUSE_PUBLIC_KEY": "pk-test-not-secret", + "LANGFUSE_SECRET_KEY": "sk-must-never-appear", + "LANGFUSE_BASE_URL": "http://127.0.0.1:1", + } + capture = hook.run(payload=payload, **environment) + assert capture.returncode == 0, capture.stderr + drain = hook.run("--drain", payload=payload, **environment) + assert drain.returncode != 0 + assert "darrow-langfuse:" in drain.stderr + assert environment["LANGFUSE_SECRET_KEY"] not in drain.stderr diff --git a/plugins/capability/darrow-observability-langfuse/backend/tests/test_packaged_hooks.py b/plugins/capability/darrow-observability-langfuse/backend/tests/test_packaged_hooks.py new file mode 100644 index 00000000..1c32e644 --- /dev/null +++ b/plugins/capability/darrow-observability-langfuse/backend/tests/test_packaged_hooks.py @@ -0,0 +1,149 @@ +"""Exercise packaged hook registration and the real host launcher.""" + +import json +import os +import re +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +PLUGIN = Path(__file__).resolve().parents[2] + + +def manifest(path: str) -> Any: + return json.loads((PLUGIN / path).read_text(encoding="utf-8")) + + +def handlers(event: str) -> list[dict[str, Any]]: + return [ + handler + for registration in manifest("hooks/hooks.json")["hooks"][event] + for handler in registration["hooks"] + ] + + +def test_manifest_identity() -> None: + claude = manifest(".claude-plugin/plugin.json") + codex = manifest(".codex-plugin/plugin.json") + assert claude["name"] == codex["name"] == "darrow-observability-langfuse" + assert re.fullmatch(r"\d+\.\d+\.\d+", claude["version"]) + assert claude["version"] == codex["version"] + assert codex["skills"] == "./skills/" + assert codex["hooks"] == "./hooks/hooks.json" + assert (PLUGIN / "hooks/stop.sh").is_file() + marketplace = json.loads( + (PLUGIN.parents[2] / ".claude-plugin/marketplace.json").read_text() + ) + entry = next(p for p in marketplace["plugins"] if p["name"] == claude["name"]) + assert entry["source"] == "./plugins/capability/darrow-observability-langfuse" + + +@pytest.mark.parametrize("event", ["UserPromptSubmit", "Stop"]) +def test_capture_and_delivery_registration(event: str) -> None: + registered = handlers(event) + first = registered[0] + assert first["type"] == "command" + assert "${PLUGIN_ROOT}" in first["command"] + assert "hooks/stop.sh" in first["command"] + assert any(not h.get("async") and "--drain" not in h["command"] for h in registered) + assert any(h.get("async") is True and "--drain" in h["command"] for h in registered) + + +def test_session_start_recovers_delivery() -> None: + assert any( + h.get("async") is True and "--drain" in h["command"] + for h in handlers("SessionStart") + ) + + +@pytest.mark.parametrize("event", ["Interrupt", "SessionEnd"]) +def test_terminal_receipt_registration(event: str) -> None: + registered = handlers(event) + assert registered + for handler in registered: + assert not handler.get("async") + assert "--drain" not in handler["command"] + assert handler["timeout"] <= 3 + + +def launcher_environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if not key.startswith("DARROW_LANGFUSE_") + } + + +@pytest.mark.skipif(os.name == "nt", reason="packaged hook uses the Unix host shell") +@pytest.mark.parametrize("shell", ["bash", "/bin/bash"]) +def test_disabled_packaged_command(shell: str) -> None: + subprocess.run( + [shell, "-c", handlers("Stop")[0]["command"]], + input="{}\n", + text=True, + check=True, + capture_output=True, + env={ + **launcher_environment(), + "PLUGIN_ROOT": str(PLUGIN), + "DARROW_LANGFUSE_ENABLED": "false", + }, + ) + + +@pytest.mark.skipif(os.name == "nt", reason="packaged hook uses the Unix host shell") +@pytest.mark.parametrize("shell", ["bash", "/bin/bash"]) +def test_stop_reconstructs_trace(shell: str) -> None: + payload = { + "session_id": "session-main", + "turn_id": "turn-1", + "cwd": str(PLUGIN.parents[2]), + "transcript_path": str(PLUGIN / "tests/fixtures/main-rollout.jsonl"), + "hook_event_name": "Stop", + } + result = subprocess.run( + [shell, str(PLUGIN / "hooks/stop.sh")], + input=json.dumps(payload), + text=True, + capture_output=True, + check=True, + env={ + **launcher_environment(), + "CODEX_PLUGIN_ROOT": str(PLUGIN), + "DARROW_LANGFUSE_ENABLED": "true", + "DARROW_LANGFUSE_CAPTURE_CONTENT": "true", + "DARROW_LANGFUSE_DRY_RUN": "true", + "DARROW_LANGFUSE_STRICT": "true", + "DARROW_LANGFUSE_WORK_ITEM_ID": "EXT-7", + }, + ) + document = json.loads(result.stdout) + assert document["status"] == "dry-run" + assert len(document["traces"]) == 1 + trace = document["traces"][0] + assert trace["name"] == "Codex Turn" + assert trace["session_id"] == "session-main:attribution:0" + assert trace["metadata"]["codex.thread_id"] == "session-main" + assert trace["metadata"]["darrow.work_item_id"] == "EXT-7" + assert trace["metadata"]["darrow.attribution_source"] == "configuration" + assert trace["metadata"]["darrow.attribution_epoch"] == trace["session_id"] + assert trace["input"] == "Inspect the repository" + assert "README.md" in trace["output"] + generation = next(o for o in trace["observations"] if o["type"] == "generation") + assert generation["model"] == "gpt-5.6-sol" + assert generation["usage_details"]["total_tokens"] == 120 + tool = next(o for o in generation["children"] if o["type"] == "tool") + assert tool["name"] == "rg --files" + assert tool["output"] == "README.md\npackage.json" + agent = next( + o + for o in trace["observations"] + if o["type"] == "agent" and o["name"] == "Codex Subagent Turn" + ) + assert agent["session_id"] == "child-1" + assert any( + o["type"] == "generation" and o["model"] == "gpt-5.6-luna" + for o in agent["children"] + ) diff --git a/plugins/capability/darrow-observability-langfuse/hooks/export-failure.test.sh b/plugins/capability/darrow-observability-langfuse/hooks/export-failure.test.sh deleted file mode 100644 index fc27ba61..00000000 --- a/plugins/capability/darrow-observability-langfuse/hooks/export-failure.test.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -plugin_dir=$(CDPATH='' cd -- "$script_dir/.." && pwd) -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/darrow-langfuse-failure.XXXXXX") -trap 'rm -rf "$work_dir"' EXIT HUP INT TERM -rollout="$work_dir/rollout.jsonl" -stderr_file="$work_dir/stderr" -cp "$plugin_dir/tests/fixtures/main-rollout.jsonl" "$rollout" -cp "$plugin_dir/tests/fixtures/rollout-2026-08-24-child-1.jsonl" "$work_dir/" -payload=$(printf '{"session_id":"session-main","turn_id":"turn-1","cwd":"%s","transcript_path":"%s","hook_event_name":"Stop"}' "$work_dir" "$rollout") -mkdir "$work_dir/.codex" -printf '%s\n' '{"strict":true}' > "$work_dir/.codex/darrow-langfuse.json" -unset DARROW_LANGFUSE_STRICT - -# Foreground capture must succeed even when the endpoint is unavailable. -printf '%s\n' "$payload" | - DARROW_LANGFUSE_ENABLED=true \ - LANGFUSE_PUBLIC_KEY=pk-test-not-secret \ - LANGFUSE_SECRET_KEY=sk-must-never-appear \ - LANGFUSE_BASE_URL=http://127.0.0.1:1 \ - "$BASH" "$script_dir/stop.sh" - -set +e -printf '%s\n' "$payload" | - CODEX_PLUGIN_ROOT="$plugin_dir" \ - DARROW_LANGFUSE_ENABLED=true \ - DARROW_LANGFUSE_CAPTURE_CONTENT=false \ - LANGFUSE_PUBLIC_KEY=pk-test-not-secret \ - LANGFUSE_SECRET_KEY=sk-must-never-appear \ - LANGFUSE_BASE_URL=http://127.0.0.1:1 \ - "$BASH" "$script_dir/stop.sh" --drain 2>"$stderr_file" -status=$? -set -e - -test "$status" -ne 0 || { - printf 'FAIL: strict mode accepted an exporter failure\n' >&2 - exit 1 -} -grep -F 'darrow-langfuse:' "$stderr_file" >/dev/null || { - printf 'FAIL: strict exporter failure had no bounded diagnostic\n' >&2 - exit 1 -} -if grep -F 'sk-must-never-appear' "$stderr_file" >/dev/null; then - printf 'FAIL: exporter failure leaked the Langfuse secret key\n' >&2 - exit 1 -fi - -printf 'observability exporter failure test passed\n' diff --git a/plugins/capability/darrow-observability-langfuse/hooks/refusal.test.sh b/plugins/capability/darrow-observability-langfuse/hooks/refusal.test.sh deleted file mode 100644 index ec2567df..00000000 --- a/plugins/capability/darrow-observability-langfuse/hooks/refusal.test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/darrow-langfuse-refusal.XXXXXX") -trap 'rm -rf "$work_dir"' EXIT HUP INT TERM - -set +e -PATH=/usr/bin:/bin \ - DARROW_LANGFUSE_DEBUG=true \ - /bin/bash "$script_dir/stop.sh" "$work_dir/missing-uv-default" -default_status=$? -PATH=/usr/bin:/bin \ - DARROW_LANGFUSE_STRICT=true \ - /bin/bash "$script_dir/stop.sh" "$work_dir/missing-uv-strict" -strict_status=$? -UV_PROJECT_ENVIRONMENT=/dev/null \ - DARROW_LANGFUSE_ENABLED=false \ - /bin/bash "$script_dir/stop.sh" "$work_dir/broken-uv-default" -broken_uv_default_status=$? -printf '%s' 'not-json' | - DARROW_LANGFUSE_STRICT=true \ - bash "$script_dir/stop.sh" 2>"$work_dir/malformed" -malformed_status=$? -set -e - -test "$default_status" -eq 0 || { - printf 'FAIL: missing UV did not fail open by default\n' >&2 - exit 1 -} -test "$strict_status" -ne 0 || { - printf 'FAIL: strict missing-UV refusal returned success\n' >&2 - exit 1 -} -test "$broken_uv_default_status" -eq 0 || { - printf 'FAIL: pre-Python UV failure did not fail open by default\n' >&2 - exit 1 -} -test "$malformed_status" -ne 0 || { - printf 'FAIL: strict malformed hook input returned success\n' >&2 - exit 1 -} -grep -F 'uv is required but was not found' "$work_dir/missing-uv-default" >/dev/null -grep -F 'uv is required but was not found' "$work_dir/missing-uv-strict" >/dev/null -grep -F 'hook input is not valid JSON' "$work_dir/malformed" >/dev/null - -printf 'observability refusal tests passed\n' diff --git a/plugins/capability/darrow-observability-langfuse/hooks/stop.test.sh b/plugins/capability/darrow-observability-langfuse/hooks/stop.test.sh deleted file mode 100644 index f73702c2..00000000 --- a/plugins/capability/darrow-observability-langfuse/hooks/stop.test.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -plugin_dir=$(CDPATH='' cd -- "$script_dir/.." && pwd) -repo_dir=$(CDPATH='' cd -- "$plugin_dir/../../.." && pwd) -rollout="$plugin_dir/tests/fixtures/main-rollout.jsonl" -output=$(mktemp "${TMPDIR:-/tmp}/darrow-langfuse-hook.XXXXXX") -trap 'rm -f "$output"' EXIT HUP INT TERM - -payload=$(printf '{"session_id":"session-main","turn_id":"turn-1","cwd":"%s","transcript_path":"%s","hook_event_name":"Stop"}' "$repo_dir" "$rollout") - -printf '%s\n' "$payload" | - CODEX_PLUGIN_ROOT="$plugin_dir" \ - DARROW_LANGFUSE_ENABLED=true \ - DARROW_LANGFUSE_CAPTURE_CONTENT=true \ - DARROW_LANGFUSE_DRY_RUN=true \ - DARROW_LANGFUSE_STRICT=true \ - DARROW_LANGFUSE_WORK_ITEM_ID=EXT-7 \ - bash "$script_dir/stop.sh" >"$output" - -node --input-type=module - "$output" <<'NODE' -import fs from "node:fs"; - -const result = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -if (result.status !== "dry-run" || result.traces.length !== 1) { - throw new Error("Stop hook did not reconstruct exactly one dry-run trace"); -} -const trace = result.traces[0]; -if ( - trace.name !== "Codex Turn" || - trace.session_id !== "session-main:attribution:0" || - trace.metadata["codex.thread_id"] !== "session-main" -) { - throw new Error("turn trace identity is incorrect"); -} -if (trace.metadata["darrow.work_item_id"] !== "EXT-7") { - throw new Error("explicit work-item configuration did not win"); -} -if ( - trace.metadata["darrow.attribution_source"] !== "configuration" || - trace.metadata["darrow.attribution_epoch"] !== trace.session_id -) { - throw new Error("attribution epoch metadata is incorrect"); -} -if (trace.input !== "Inspect the repository" || !trace.output.includes("README.md")) { - throw new Error("turn content was not reconstructed"); -} -const generation = trace.observations.find((item) => item.type === "generation"); -if (!generation || generation.model !== "gpt-5.6-sol") { - throw new Error("model generation is missing"); -} -if (generation.usage_details.total_tokens !== 120) { - throw new Error("generation token usage is missing"); -} -const tool = generation.children.find((item) => item.type === "tool"); -if (!tool || tool.name !== "rg --files" || tool.output !== "README.md\npackage.json") { - throw new Error("tool activity was not reconstructed"); -} -const subagent = trace.observations.find( - (item) => item.type === "agent" && item.name === "Codex Subagent Turn", -); -if (!subagent || subagent.session_id !== "child-1") { - throw new Error("spawned subagent turn was not nested under the parent trace"); -} -if (!subagent.children.some((item) => item.type === "generation" && item.model === "gpt-5.6-luna")) { - throw new Error("subagent model activity is missing"); -} -NODE - -printf 'observability Stop hook reconstruction test passed\n' diff --git a/plugins/capability/darrow-observability-langfuse/hooks/strict.test.sh b/plugins/capability/darrow-observability-langfuse/hooks/strict.test.sh deleted file mode 100644 index c1e0edd1..00000000 --- a/plugins/capability/darrow-observability-langfuse/hooks/strict.test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/darrow-langfuse-strict.XXXXXX") -work_dir=$(cd "$work_dir" && pwd -P) -trap 'rm -rf "$work_dir"' EXIT HUP INT TERM -mkdir -p "$work_dir/home/.codex" "$work_dir/project/.codex" -payload=$(printf '{"session_id":"test","turn_id":"0","hook_event_name":"Stop","cwd":"%s","transcript_path":"%s"}' "$work_dir/project" "$work_dir/missing.jsonl") - -for location in "$work_dir/home" "$work_dir/project"; do - printf '%s\n' '{"enabled":true,"strict":true}' > "$location/.codex/darrow-langfuse.json" - if printf '%s\n' "$payload" | env -i PATH="$PATH" HOME="$work_dir/home" UV_OFFLINE=1 \ - "$BASH" "$script_dir/stop.sh" 2>"$work_dir/error"; then - printf 'FAIL: file strict failure became success\n' >&2 - exit 1 - fi - grep -F 'Langfuse credentials are missing' "$work_dir/error" >/dev/null - rm "$location/.codex/darrow-langfuse.json" -done - -for strict in 1 true yes on ' TRUE ' ' Yes '; do - if printf '%s\n' "$payload" | env -i PATH="$PATH" HOME="$work_dir/home" UV_OFFLINE=1 \ - DARROW_LANGFUSE_ENABLED=true DARROW_LANGFUSE_STRICT="$strict" \ - "$BASH" "$script_dir/stop.sh" 2>"$work_dir/error"; then - printf 'FAIL: environment strict failure became success: %s\n' "$strict" >&2 - exit 1 - fi - grep -F 'Langfuse credentials are missing' "$work_dir/error" >/dev/null - if env -i PATH=/usr/bin:/bin HOME="$work_dir/home" DARROW_LANGFUSE_STRICT="$strict" \ - "$BASH" "$script_dir/stop.sh" "$work_dir/error"; then - printf 'FAIL: strict missing-UV failure became success: %s\n' "$strict" >&2 - exit 1 - fi -done - -# Environment false overrides file strict, and a pre-Python failure stays open. -printf '%s\n' '{"enabled":true,"strict":true}' > "$work_dir/project/.codex/darrow-langfuse.json" -printf '%s\n' "$payload" | env -i PATH="$PATH" HOME="$work_dir/home" UV_OFFLINE=1 \ - DARROW_LANGFUSE_STRICT=off "$BASH" "$script_dir/stop.sh" -printf '%s\n' "$payload" | env -i PATH="$PATH" HOME="$work_dir/home" UV_PROJECT_ENVIRONMENT=/dev/null \ - "$BASH" "$script_dir/stop.sh" 2>"$work_dir/error" -printf 'observability strict tests passed\n' diff --git a/plugins/capability/darrow-observability-langfuse/tests/fresh-install.test.sh b/plugins/capability/darrow-observability-langfuse/tests/fresh-install.test.sh deleted file mode 100755 index f0563539..00000000 --- a/plugins/capability/darrow-observability-langfuse/tests/fresh-install.test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -plugin_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -fixture=$(mktemp -d "${TMPDIR:-/tmp}/darrow-langfuse-install.XXXXXX") -trap 'rm -rf "$fixture"' EXIT - -mkdir -p "$fixture/plugin" -cp -R "$plugin_root/." "$fixture/plugin/" -rm -rf "$fixture/plugin/backend/.venv" -rm -f "$fixture/plugin/backend/.coverage" "$fixture/plugin/backend/coverage.json" - -uv sync --locked --no-dev --project "$fixture/plugin/backend" -runtime_tree=$(uv tree --locked --no-dev --project "$fixture/plugin/backend") -for development_tool in coverage hypothesis mypy pytest ruff; do - if printf '%s\n' "$runtime_tree" | grep -E "(^|[[:space:]])${development_tool}([[:space:]]|$)" >/dev/null; then - echo "error: development dependency installed at runtime: $development_tool" >&2 - exit 1 - fi -done - -uv run --quiet --frozen --no-dev --project "$fixture/plugin/backend" \ - python -c 'import darrow_observability_langfuse' - -printf '%s\n' '{}' | env \ - DARROW_LANGFUSE_ENABLED=false \ - PLUGIN_ROOT="$fixture/plugin" \ - bash "$fixture/plugin/hooks/stop.sh" diff --git a/plugins/capability/darrow-observability-langfuse/tests/issue-77-verification.md b/plugins/capability/darrow-observability-langfuse/tests/issue-77-verification.md index 64f35725..cb2c4686 100644 --- a/plugins/capability/darrow-observability-langfuse/tests/issue-77-verification.md +++ b/plugins/capability/darrow-observability-langfuse/tests/issue-77-verification.md @@ -126,9 +126,10 @@ uv run --quiet --frozen --project plugins/capability/darrow-observability-langfu python -m unittest discover -s plugins/capability/darrow-observability-langfuse/backend/tests -q ``` -Run each of these with both `bash` and `/bin/bash`: `tests/package.test.sh`, -`hooks/stop.test.sh`, `hooks/export-failure.test.sh`, and -`hooks/refusal.test.sh`, relative to the plugin root. All eight invocations pass. +Run the package, Stop reconstruction, and failure checks with +`uv run --quiet --frozen --all-groups --project backend pytest backend/tests/test_packaged_hooks.py backend/tests/test_hook_failures.py`, +relative to the plugin root. The launcher scenarios run under both `bash` and +`/bin/bash` through pytest. The export-failure launcher check now supplies valid session/turn IDs and checks successful local capture followed by a failing background drain, rather than mistaking malformed input for exporter failure. diff --git a/plugins/capability/darrow-observability-langfuse/tests/package.test.sh b/plugins/capability/darrow-observability-langfuse/tests/package.test.sh deleted file mode 100644 index 04dbfb8e..00000000 --- a/plugins/capability/darrow-observability-langfuse/tests/package.test.sh +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -plugin_dir=$(CDPATH='' cd -- "$script_dir/.." && pwd) -repo_dir=$(CDPATH='' cd -- "$plugin_dir/../../.." && pwd) -claude_manifest="$plugin_dir/.claude-plugin/plugin.json" -codex_manifest="$plugin_dir/.codex-plugin/plugin.json" -hook_manifest="$plugin_dir/hooks/hooks.json" -hook_launcher="$plugin_dir/hooks/stop.sh" -marketplace="$repo_dir/.claude-plugin/marketplace.json" - -fail() { - printf 'FAIL: %s\n' "$1" >&2 - exit 1 -} - -for required in "$claude_manifest" "$codex_manifest" "$hook_manifest" "$hook_launcher"; do - test -r "$required" || fail "missing packaged file: $required" -done - -node --input-type=module - "$claude_manifest" "$codex_manifest" "$hook_manifest" "$marketplace" <<'NODE' -import fs from "node:fs"; - -const [claudePath, codexPath, hookPath, marketplacePath] = process.argv.slice(2); -const claude = JSON.parse(fs.readFileSync(claudePath, "utf8")); -const codex = JSON.parse(fs.readFileSync(codexPath, "utf8")); -const hooks = JSON.parse(fs.readFileSync(hookPath, "utf8")); -const marketplace = JSON.parse(fs.readFileSync(marketplacePath, "utf8")); - -const expectedName = "darrow-observability-langfuse"; -if (claude.name !== expectedName || codex.name !== expectedName) { - throw new Error("manifest names do not match the plugin boundary"); -} -if (!/^\d+\.\d+\.\d+$/.test(claude.version) || claude.version !== codex.version) { - throw new Error("manifest versions are not matching semantic versions"); -} -if (codex.skills !== "./skills/") { - throw new Error("Codex manifest does not point at ./skills/"); -} -if (codex.hooks !== "./hooks/hooks.json") { - throw new Error("Codex manifest does not register the packaged hooks file"); -} - -for (const event of ["UserPromptSubmit", "Stop"]) { - const registration = hooks?.hooks?.[event]; - const command = registration?.[0]?.hooks?.[0]?.command; - if ( - registration?.[0]?.hooks?.[0]?.type !== "command" || - typeof command !== "string" - ) { - throw new Error(`${event} command hook is not registered`); - } - if (!command.includes("${PLUGIN_ROOT}") || !command.includes("hooks/stop.sh")) { - throw new Error( - `${event} hook does not resolve its packaged launcher through PLUGIN_ROOT`, - ); - } -} - -const entry = marketplace.plugins.find((plugin) => plugin.name === expectedName); -for (const event of ["Stop", "UserPromptSubmit"]) { - const handlers = hooks.hooks[event].flatMap((registration) => registration.hooks); - if (!handlers.some((handler) => !handler.async && !handler.command.includes("--drain"))) { - throw new Error(`${event} is missing synchronous local capture`); - } - if (!handlers.some((handler) => handler.async === true && handler.command.includes("--drain"))) { - throw new Error(`${event} is missing native asynchronous delivery`); - } -} -if (!hooks.hooks.SessionStart.some((registration) => registration.hooks.some((handler) => handler.async === true && handler.command.includes("--drain")))) { - throw new Error("SessionStart cannot recover cancelled background delivery"); -} -for (const event of ["Interrupt", "SessionEnd"]) { - const handlers = hooks.hooks[event]?.flatMap((registration) => registration.hooks); - if (!handlers?.length || handlers.some((handler) => handler.async || handler.command.includes("--drain") || handler.timeout > 3)) { - throw new Error(`${event} must record a bounded synchronous local receipt`); - } -} -if (!entry) throw new Error("marketplace entry is missing"); -if (entry.source !== "./plugins/capability/darrow-observability-langfuse") { - throw new Error("marketplace entry has the wrong source"); -} -NODE - -hook_command=$(node --input-type=module - "$hook_manifest" <<'NODE' -import fs from "node:fs"; - -const hooks = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -process.stdout.write(hooks.hooks.Stop[0].hooks[0].command); -NODE -) - -PLUGIN_ROOT="$plugin_dir" \ - DARROW_LANGFUSE_ENABLED=false \ - /bin/sh -c "$hook_command" <<< '{}' - -printf 'observability plugin package tests passed\n' diff --git a/plugins/capability/darrow-review/.claude-plugin/plugin.json b/plugins/capability/darrow-review/.claude-plugin/plugin.json index fdb99773..cf4704b7 100644 --- a/plugins/capability/darrow-review/.claude-plugin/plugin.json +++ b/plugins/capability/darrow-review/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "darrow-review", "description": "Read-only comprehensive code review and fix-scoped repair verification", - "version": "0.5.3", + "version": "0.5.4", "license": "BUSL-1.1", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-review/.codex-plugin/plugin.json b/plugins/capability/darrow-review/.codex-plugin/plugin.json index 83735e39..aa949566 100644 --- a/plugins/capability/darrow-review/.codex-plugin/plugin.json +++ b/plugins/capability/darrow-review/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-review", - "version": "0.5.3", + "version": "0.5.4", "description": "Read-only comprehensive code review and fix-scoped repair verification", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-review/backend/tests/evals/eval_routes.py b/plugins/capability/darrow-review/backend/tests/evals/eval_routes.py new file mode 100644 index 00000000..6af73b92 --- /dev/null +++ b/plugins/capability/darrow-review/backend/tests/evals/eval_routes.py @@ -0,0 +1,183 @@ +"""Independent oracle for retained review-reader identities and routes.""" + +import argparse +import json +import re +from pathlib import Path +from typing import Any + + +def row(path: Path, key: str) -> list[str]: + matches = [ + line.split("\t")[1:] + for line in path.read_text(encoding="utf-8").splitlines() + if line.split("\t")[0] == key + ] + assert len(matches) == 1, f"{path}: expected one {key}" + return matches[0] + + +def route(host: str, profile: str) -> list[str]: + routes = { + ("codex", "default"): ["codex", "openai", "gpt-5.6-sol", "xhigh"], + ("claude", "default"): ["claude", "anthropic", "claude-opus-5", "xhigh"], + ("codex", "override"): ["codex", "openai", "gpt-5.5", "xhigh"], + ("claude", "override"): ["claude", "anthropic", "claude-sonnet-5", "high"], + } + return routes[host, profile] + + +def identities(directory: Path, axes: list[str], expected: list[str]) -> dict[str, str]: + field = "requested_route" if expected[0] == "codex" else "observed_route" + result = {} + for axis in axes: + path = directory / f"{axis}-route.tsv" + assert row(path, "axis") == [axis] + assert row(path, field) == expected + assert row(path, "route_bound") == ["true"] + agent = row(path, "agent_id") + assert len(agent) == 1 and agent[0] + result[axis] = agent[0] + assert len(set(result.values())) == len(axes), "reader identities must be distinct" + if axes == ["standards"]: + assert not (directory / "spec-route.tsv").exists() + return result + + +def events_of(events: list[dict[str, Any]], kind: str) -> list[dict[str, Any]]: + return [event for event in events if event.get("type") == kind] + + +def ordinal(event: dict[str, Any], key: str) -> int: + value = event.get(key) + assert isinstance(value, int) and not isinstance(value, bool) and value > 0 + return value + + +def codex_evidence( + events: list[dict[str, Any]], + agents: dict[str, str], + expected: list[str], +) -> None: + launches = events_of(events, "darrow.codex_native_spawn") + assert len(launches) == len(agents), "unexpected native spawn count" + ordinals: list[int] = [] + for axis, agent in agents.items(): + matches = [event for event in launches if event.get("agent_ref") == agent] + assert len(matches) == 1 + event = matches[0] + assert event.get("status") == "accepted" + assert event.get("review_axis") == axis + assert event.get("model") == expected[2] + assert event.get("reasoning_effort") == expected[3] + assert event.get("fork_turns") == "none" + if len(agents) > 1: + ordinals.append(ordinal(event, "accepted_ordinal")) + if len(agents) > 1: + waits = [ + ordinal(event, "ordinal") + for event in events_of(events, "darrow.codex_native_wait") + ] + assert waits + assert max(ordinals) < min(waits), ( + "both readers must be accepted before waiting" + ) + + +def agent_calls(value: Any) -> list[dict[str, Any]]: + if isinstance(value, list): + return [call for item in value for call in agent_calls(item)] + if isinstance(value, dict): + if value.get("name") == "Agent": + return [value] + return [call for item in value.values() for call in agent_calls(item)] + return [] + + +def claude_calls(events: list[dict[str, Any]], axes: list[str], subagent: str) -> None: + batches = [agent_calls(event) for event in events] + calls = [call for batch in batches for call in batch] + assert len(calls) == len(axes) + assert any(len(batch) == len(axes) for batch in batches), ( + "readers need one launch batch" + ) + observed_axes = [] + for call in calls: + arguments = call["input"] + assert arguments["subagent_type"] == subagent + matches = re.findall( + r"(?m)^- review_axis: (standards|spec)\s*$", arguments["prompt"] + ) + assert len(matches) == 1 + observed_axes.extend(matches) + assert sorted(observed_axes) == sorted(axes) + + +def claude_evidence( + directory: Path, + events: list[dict[str, Any]], + agents: dict[str, str], + expected: list[str], +) -> None: + subagent = f"darrow-review:review-reader-{expected[2]}-{expected[3]}" + claude_calls(events, list(agents), subagent) + launches = events_of(events, "darrow.review_agent_launch") + assert len(launches) == len(agents) + batches = [] + for axis, agent in agents.items(): + observed = directory / f"{axis}-observed-route.tsv" + assert row(observed, "agent_id") == [agent] + assert row(observed, "observed_route") == expected + verify_provider(directory, axis, len(agents)) + matching = [event for event in launches if event.get("agent_id") == agent] + assert len(matching) == 1 + event = matching[0] + assert event.get("review_axis") == axis + assert event.get("subagent_type") == subagent + batches.append(event.get("batch")) + if len(agents) > 1: + assert all(type(batch) is int for batch in batches) + assert len(set(batches)) == 1 + + +def verify_provider(directory: Path, axis: str, count: int) -> None: + if count > 1: + for suffix in ("route", "observed-route"): + assert row(directory / f"{axis}-{suffix}.tsv", "provider_evidence") == [ + "current-host-environment-default" + ] + + +def verify(git_dir: Path, host: str, profile: str, axes: list[str]) -> None: + selections = sorted(git_dir.glob("**/darrow-review.*/reviewer-route.tsv")) + assert selections, "missing reviewer route" + directory = selections[-1].parent + expected = route(host, profile) + agents = identities(directory, axes, expected) + events = [ + json.loads(line) + for line in (git_dir / "retained-harness.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() + ] + assert all(isinstance(event, dict) for event in events) + if host == "codex": + codex_evidence(events, agents, expected) + else: + claude_evidence(directory, events, agents, expected) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--host", choices=("codex", "claude"), required=True) + parser.add_argument("--profile", choices=("default", "override"), required=True) + parser.add_argument( + "--axes", choices=("standards", "spec"), nargs="+", required=True + ) + arguments = parser.parse_args() + verify(Path(".git"), arguments.host, arguments.profile, arguments.axes) + + +if __name__ == "__main__": + main() diff --git a/plugins/capability/darrow-review/backend/tests/evals/test_eval_routes.py b/plugins/capability/darrow-review/backend/tests/evals/test_eval_routes.py new file mode 100644 index 00000000..bb281e86 --- /dev/null +++ b/plugins/capability/darrow-review/backend/tests/evals/test_eval_routes.py @@ -0,0 +1,116 @@ +"""Valid evidence and false-positive counterexamples for the independent eval oracle.""" + +import json +from pathlib import Path +from typing import Any + +import pytest +from eval_routes import row, verify + + +def evidence(root: Path, host: str, axes: list[str]) -> list[dict[str, Any]]: + directory = root / "darrow-review.fixture" + directory.mkdir() + (directory / "reviewer-route.tsv").write_text("fixture\n") + model, provider = ( + ("gpt-5.6-sol", "openai") if host == "codex" else ("claude-opus-5", "anthropic") + ) + route = f"{host}\t{provider}\t{model}\txhigh" + subagent = f"darrow-review:review-reader-{model}-xhigh" + events: list[dict[str, Any]] = [] + calls = [] + for index, axis in enumerate(axes, 1): + record = ( + f"axis\t{axis}\nagent_id\t{axis}-child\n" + f"requested_route\t{route}\nobserved_route\t{route}\nroute_bound\ttrue\n" + "provider_evidence\tcurrent-host-environment-default\n" + ) + for suffix in ("route", "observed-route"): + (directory / f"{axis}-{suffix}.tsv").write_text(record) + if host == "codex": + events.append( + { + "type": "darrow.codex_native_spawn", + "status": "accepted", + "agent_ref": f"{axis}-child", + "review_axis": axis, + "model": model, + "reasoning_effort": "xhigh", + "fork_turns": "none", + "accepted_ordinal": index, + } + ) + else: + events.append( + { + "type": "darrow.review_agent_launch", + "agent_id": f"{axis}-child", + "review_axis": axis, + "subagent_type": subagent, + "batch": 1, + } + ) + calls.append( + { + "name": "Agent", + "input": { + "subagent_type": subagent, + "prompt": f"- review_axis: {axis}", + }, + } + ) + if host == "codex": + events.append({"type": "darrow.codex_native_wait", "ordinal": 3}) + else: + events.append({"type": "assistant", "message": {"content": calls}}) + return events + + +def retain(root: Path, events: list[dict[str, Any]]) -> None: + # Whitespace and key order differ from the compact harness serialization. + (root / "retained-harness.jsonl").write_text( + "\n".join( + json.dumps(event, sort_keys=True, separators=(", ", ": ")) + for event in events + ) + + "\n", + ) + + +@pytest.mark.parametrize("host", ["codex", "claude"]) +@pytest.mark.parametrize("axes", [["standards"], ["standards", "spec"]]) +def test_structured_evidence_ignores_json_formatting( + tmp_path: Path, + host: str, + axes: list[str], +) -> None: + retain(tmp_path, evidence(tmp_path, host, axes)) + verify(tmp_path, host, "default", axes) + + +@pytest.mark.parametrize("host", ["codex", "claude"]) +@pytest.mark.parametrize("mutation", ["missing", "duplicate", "axis", "identity"]) +def test_single_axis_rejects_uncorrelated_launches( + tmp_path: Path, + host: str, + mutation: str, +) -> None: + events = evidence(tmp_path, host, ["standards"]) + if mutation == "missing": + events.pop(0) + elif mutation == "duplicate": + events.append(events[0]) + elif mutation == "axis": + events[0]["review_axis"] = "spec" + else: + events[0]["agent_ref" if host == "codex" else "agent_id"] = "unrelated" + retain(tmp_path, events) + with pytest.raises(AssertionError): + verify(tmp_path, host, "default", ["standards"]) + + +def test_duplicate_tsv_identity_is_not_evidence(tmp_path: Path) -> None: + path = tmp_path / "route.tsv" + path.write_text("agent_id\tfirst\nagent_id\tsecond\n") + with pytest.raises(AssertionError, match="expected one agent_id"): + row(path, "agent_id") diff --git a/plugins/capability/darrow-review/skills/code-review/evals/fix-verification-resolved.yaml b/plugins/capability/darrow-review/skills/code-review/evals/fix-verification-resolved.yaml index 0b04e0bf..903c6ffc 100644 --- a/plugins/capability/darrow-review/skills/code-review/evals/fix-verification-resolved.yaml +++ b/plugins/capability/darrow-review/skills/code-review/evals/fix-verification-resolved.yaml @@ -34,6 +34,10 @@ fixture: export const DEBUG = false; export const TIMEOUT_MS = 2500; setup: | + mkdir -p .git/eval-checks/review/tests/evals + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/eval-checks/review/ + cp -R "{{case_dir}}/../../../backend/src" .git/eval-checks/review/ + cp "{{case_dir}}/../../../backend/tests/evals/eval_routes.py" .git/eval-checks/review/tests/evals/ set -eu backend=$(cd "{{case_dir}}/../../../backend" && pwd -P) prior_output=$(uv run --quiet --frozen --no-dev --project "$backend" review-scope prepare --repo "$PWD" --base HEAD^ --target HEAD) @@ -82,84 +86,9 @@ checks: END { exit !(count == 3 && !invalid) }' "$record" - name: both fix verifiers retain exact default route evidence run: >- - selection=$(find .git -path '*/darrow-review.*/*' -name reviewer-route.tsv -type f | sort | tail -n '1'); - test -n "$selection" || exit 1; artifact_dir=$(dirname "$selection"); - case "$DARROW_EVAL_HARNESS" in - codex) host=codex; provider=openai; model=gpt-5.6-sol; effort=xhigh ;; - claude) host=claude; provider=anthropic; model=claude-opus-5; effort=xhigh ;; - *) exit 2 ;; esac; - case "$DARROW_EVAL_HARNESS" in codex) - test "$(grep -F '"type":"darrow.codex_native_spawn"' - .git/retained-harness.jsonl | grep -c .)" -eq 2 || exit 1; - ! grep -F '"type":"darrow.codex_native_spawn"' .git/retained-harness.jsonl | - grep -v -F '"status":"accepted"' >/dev/null || exit 1 ;; - claude) test "$(grep -F '"type":"darrow.review_agent_launch"' - .git/retained-harness.jsonl | grep -c .)" -eq 2 || exit 1 ;; esac; - for axis in standards spec; do record="$artifact_dir/$axis-route.tsv"; - test -f "$record" || exit 1; - awk -F '\t' -v a="$axis" - '$1 == "axis" && $2 == a { found = 1 } END { exit !found }' "$record" || exit 1; - agent=$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$record"); - test -n "$agent" || exit 1; - case "$axis" in standards) standards_agent=$agent ;; spec) spec_agent=$agent ;; esac; - case "$DARROW_EVAL_HARNESS" in codex) route_field=requested_route; proof_field=route_bound ;; - claude) route_field=observed_route; proof_field=route_bound ;; esac; - awk -F '\t' -v f="$route_field" -v h="$host" -v p="$provider" -v m="$model" -v e="$effort" - '$1 == f && $2 == h && $3 == p && $4 == m && $5 == e { found = 1 } - END { exit !found }' "$record" || exit 1; - awk -F '\t' -v f="$proof_field" '$1 == f && $2 == "true" { found = 1 } - END { exit !found }' "$record" || exit 1; - if test "$DARROW_EVAL_HARNESS" = codex; then - native=$(grep -F '"type":"darrow.codex_native_spawn"' .git/retained-harness.jsonl | - grep -F '"status":"accepted"' | grep -F "\"agent_ref\":\"$agent\"" | - grep -F "\"model\":\"$model\"" | grep -F "\"reasoning_effort\":\"$effort\"") || exit 1; - case "$native" in *'"fork_turns":"none"'*) ;; *) exit 1 ;; esac; - case "$native" in *"\"review_axis\":\"$axis\""*) ;; *) exit 1 ;; esac; fi; done; - test "$standards_agent" != "$spec_agent" || exit 1; - case "$DARROW_EVAL_HARNESS" in codex) - awk -v s="$standards_agent" -v p="$spec_agent" - 'index($0, "\"type\":\"darrow.codex_native_spawn\"") { - value = $0; sub(/^.*"accepted_ordinal":/, "", value); sub(/[^0-9].*$/, "", value); - if (index($0, "\"agent_ref\":\"" s "\"")) standards_accept = value + 0; - if (index($0, "\"agent_ref\":\"" p "\"")) spec_accept = value + 0 } - index($0, "\"type\":\"darrow.codex_native_wait\"") { - value = $0; sub(/^.*"ordinal":/, "", value); sub(/[^0-9].*$/, "", value); - if (!first_wait || value + 0 < first_wait) first_wait = value + 0 } - END { exit !(standards_accept && spec_accept && first_wait && - standards_accept < first_wait && spec_accept < first_wait) }' - .git/retained-harness.jsonl || exit 1 ;; claude) - subagent="darrow-review:review-reader-$model-$effort"; - awk 'index($0, "\"name\":\"Agent\"") { - value = $0; agents += gsub("\"name\":\"Agent\"", "", value) } - END { exit agents != 2 }' .git/retained-harness.jsonl || exit 1; - awk -v a="$subagent" - 'index($0, "\"name\":\"Agent\"") && - index($0, "- review_axis: standards") && index($0, "- review_axis: spec") { - value = $0; agents = gsub("\"name\":\"Agent\"", "", value); - value = $0; types = gsub("\"subagent_type\":\"" a "\"", "", value); - if (agents == 2 && types == 2) found = 1 } - END { exit !found }' .git/retained-harness.jsonl || exit 1 ;; esac; - if test "$DARROW_EVAL_HARNESS" = claude; then for axis in standards spec; - do observed="$artifact_dir/$axis-observed-route.tsv"; test -f "$observed" || exit 1; - application="$artifact_dir/$axis-route.tsv"; - agent=$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$application"); - grep -F -x 'provider_evidence current-host-environment-default' - "$observed" >/dev/null || exit 1; - grep -F -x 'provider_evidence current-host-environment-default' - "$application" >/dev/null || exit 1; - test "$agent" = - "$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$observed")" || exit 1; - awk -F '\t' -v m="$model" -v e="$effort" - '$1 == "observed_route" && $2 == "claude" && $3 == "anthropic" && $4 == m && $5 == e { found = 1 } - END { exit !found }' "$observed" || exit 1; - subagent="darrow-review:review-reader-$model-$effort"; - launch=$(grep -F '"type":"darrow.review_agent_launch"' .git/retained-harness.jsonl | - grep -F "\"agent_id\":\"$agent\"" | grep -F "\"review_axis\":\"$axis\"" | - grep -F "\"subagent_type\":\"$subagent\"") || exit 1; - test "$(printf '%s\n' "$launch" | grep -c .)" -eq 1 || exit 1; - batch=$(printf '%s\n' "$launch" | sed -E 's/.*"batch":([0-9]+).*/\1/'); - case "$axis" in standards) standards_batch=$batch ;; spec) spec_batch=$batch ;; esac; - done; test "$standards_batch" = "$spec_batch" || exit 1; fi + uv run --quiet --frozen --no-dev --project .git/eval-checks/review + python .git/eval-checks/review/tests/evals/eval_routes.py + --host "$DARROW_EVAL_HARNESS" --profile default --axes standards spec - name: final response is the complete rendered verification report run: >- record=$(find .git -path '*/darrow-review.*/*' -name verification.tsv -type f | sort | tail -n 1) && diff --git a/plugins/capability/darrow-review/skills/code-review/evals/reviewer-route-override.yaml b/plugins/capability/darrow-review/skills/code-review/evals/reviewer-route-override.yaml index 8033dbd6..a256128e 100644 --- a/plugins/capability/darrow-review/skills/code-review/evals/reviewer-route-override.yaml +++ b/plugins/capability/darrow-review/skills/code-review/evals/reviewer-route-override.yaml @@ -24,6 +24,11 @@ fixture: files: src/config.js: | export const mode = "unsafe"; + setup: | + mkdir -p .git/eval-checks/review/tests/evals + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/eval-checks/review/ + cp -R "{{case_dir}}/../../../backend/src" .git/eval-checks/review/ + cp "{{case_dir}}/../../../backend/tests/evals/eval_routes.py" .git/eval-checks/review/tests/evals/ checks: - name: repository reviewer policy is retained with the pinned scope run: >- @@ -38,84 +43,9 @@ checks: *) exit 2 ;; esac - name: both isolated axes retain exact route application evidence run: >- - selection=$(find .git -path '*/darrow-review.*/*' -name reviewer-route.tsv -type f | sort | tail -n '1'); - test -n "$selection" || exit 1; artifact_dir=$(dirname "$selection"); - case "$DARROW_EVAL_HARNESS" in - codex) host=codex; provider=openai; model=gpt-5.5; effort=xhigh ;; - claude) host=claude; provider=anthropic; model=claude-sonnet-5; effort=high ;; - *) exit 2 ;; esac; - case "$DARROW_EVAL_HARNESS" in codex) - test "$(grep -F '"type":"darrow.codex_native_spawn"' - .git/retained-harness.jsonl | grep -c .)" -eq 2 || exit 1; - ! grep -F '"type":"darrow.codex_native_spawn"' .git/retained-harness.jsonl | - grep -v -F '"status":"accepted"' >/dev/null || exit 1 ;; - claude) test "$(grep -F '"type":"darrow.review_agent_launch"' - .git/retained-harness.jsonl | grep -c .)" -eq 2 || exit 1 ;; esac; - for axis in standards spec; do record="$artifact_dir/$axis-route.tsv"; - test -f "$record" || exit 1; - awk -F '\t' -v a="$axis" - '$1 == "axis" && $2 == a { found = 1 } END { exit !found }' "$record" || exit 1; - agent=$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$record"); - test -n "$agent" || exit 1; - case "$axis" in standards) standards_agent=$agent ;; spec) spec_agent=$agent ;; esac; - case "$DARROW_EVAL_HARNESS" in codex) route_field=requested_route; proof_field=route_bound ;; - claude) route_field=observed_route; proof_field=route_bound ;; esac; - awk -F '\t' -v f="$route_field" -v h="$host" -v p="$provider" -v m="$model" -v e="$effort" - '$1 == f && $2 == h && $3 == p && $4 == m && $5 == e { found = 1 } - END { exit !found }' "$record" || exit 1; - awk -F '\t' -v f="$proof_field" '$1 == f && $2 == "true" { found = 1 } - END { exit !found }' "$record" || exit 1; - if test "$DARROW_EVAL_HARNESS" = codex; then - native=$(grep -F '"type":"darrow.codex_native_spawn"' .git/retained-harness.jsonl | - grep -F '"status":"accepted"' | grep -F "\"agent_ref\":\"$agent\"" | - grep -F "\"model\":\"$model\"" | grep -F "\"reasoning_effort\":\"$effort\"") || exit 1; - case "$native" in *'"fork_turns":"none"'*) ;; *) exit 1 ;; esac; - case "$native" in *"\"review_axis\":\"$axis\""*) ;; *) exit 1 ;; esac; fi; done; - test "$standards_agent" != "$spec_agent" || exit 1; - case "$DARROW_EVAL_HARNESS" in codex) - awk -v s="$standards_agent" -v p="$spec_agent" - 'index($0, "\"type\":\"darrow.codex_native_spawn\"") { - value = $0; sub(/^.*"accepted_ordinal":/, "", value); sub(/[^0-9].*$/, "", value); - if (index($0, "\"agent_ref\":\"" s "\"")) standards_accept = value + 0; - if (index($0, "\"agent_ref\":\"" p "\"")) spec_accept = value + 0 } - index($0, "\"type\":\"darrow.codex_native_wait\"") { - value = $0; sub(/^.*"ordinal":/, "", value); sub(/[^0-9].*$/, "", value); - if (!first_wait || value + 0 < first_wait) first_wait = value + 0 } - END { exit !(standards_accept && spec_accept && first_wait && - standards_accept < first_wait && spec_accept < first_wait) }' - .git/retained-harness.jsonl || exit 1 ;; claude) - subagent="darrow-review:review-reader-$model-$effort"; - awk 'index($0, "\"name\":\"Agent\"") { - value = $0; agents += gsub("\"name\":\"Agent\"", "", value) } - END { exit agents != 2 }' .git/retained-harness.jsonl || exit 1; - awk -v a="$subagent" - 'index($0, "\"name\":\"Agent\"") && - index($0, "- review_axis: standards") && index($0, "- review_axis: spec") { - value = $0; agents = gsub("\"name\":\"Agent\"", "", value); - value = $0; types = gsub("\"subagent_type\":\"" a "\"", "", value); - if (agents == 2 && types == 2) found = 1 } - END { exit !found }' .git/retained-harness.jsonl || exit 1 ;; esac; - if test "$DARROW_EVAL_HARNESS" = claude; then for axis in standards spec; - do observed="$artifact_dir/$axis-observed-route.tsv"; test -f "$observed" || exit 1; - application="$artifact_dir/$axis-route.tsv"; - agent=$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$application"); - grep -F -x 'provider_evidence current-host-environment-default' - "$observed" >/dev/null || exit 1; - grep -F -x 'provider_evidence current-host-environment-default' - "$application" >/dev/null || exit 1; - test "$agent" = - "$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$observed")" || exit 1; - awk -F '\t' -v m="$model" -v e="$effort" - '$1 == "observed_route" && $2 == "claude" && $3 == "anthropic" && $4 == m && $5 == e { found = 1 } - END { exit !found }' "$observed" || exit 1; - subagent="darrow-review:review-reader-$model-$effort"; - launch=$(grep -F '"type":"darrow.review_agent_launch"' .git/retained-harness.jsonl | - grep -F "\"agent_id\":\"$agent\"" | grep -F "\"review_axis\":\"$axis\"" | - grep -F "\"subagent_type\":\"$subagent\"") || exit 1; - test "$(printf '%s\n' "$launch" | grep -c .)" -eq 1 || exit 1; - batch=$(printf '%s\n' "$launch" | sed -E 's/.*"batch":([0-9]+).*/\1/'); - case "$axis" in standards) standards_batch=$batch ;; spec) spec_batch=$batch ;; esac; - done; test "$standards_batch" = "$spec_batch" || exit 1; fi + uv run --quiet --frozen --no-dev --project .git/eval-checks/review + python .git/eval-checks/review/tests/evals/eval_routes.py + --host "$DARROW_EVAL_HARNESS" --profile override --axes standards spec - name: review remains read-only run: test "$(git status --porcelain --untracked-files=all)" = ' M src/config.js' - name: canonical TSV is retained beneath the review scope artifact diff --git a/plugins/capability/darrow-review/skills/code-review/evals/standards-only.yaml b/plugins/capability/darrow-review/skills/code-review/evals/standards-only.yaml index adef8e76..18c410b4 100644 --- a/plugins/capability/darrow-review/skills/code-review/evals/standards-only.yaml +++ b/plugins/capability/darrow-review/skills/code-review/evals/standards-only.yaml @@ -22,7 +22,12 @@ fixture: export function normalizeName(value) { return value.trim(); } - setup: git status --porcelain --untracked-files=all >.git/status-before + setup: | + mkdir -p .git/eval-checks/review/tests/evals + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/eval-checks/review/ + cp -R "{{case_dir}}/../../../backend/src" .git/eval-checks/review/ + cp "{{case_dir}}/../../../backend/tests/evals/eval_routes.py" .git/eval-checks/review/tests/evals/ + git status --porcelain --untracked-files=all >.git/status-before checks: - name: review preserves the complete working tree run: git status --porcelain --untracked-files=all >.git/status-after; cmp .git/status-before .git/status-after @@ -38,54 +43,9 @@ checks: expect_regex: "^valid: darrow-review-result-v1$" - name: only the standards reader retains exact default route evidence run: >- - selection=$(find .git -path '*/darrow-review.*/*' -name reviewer-route.tsv -type f | sort | tail -n '1'); - test -n "$selection" || exit 1; artifact_dir=$(dirname "$selection"); - case "$DARROW_EVAL_HARNESS" in - codex) host=codex; provider=openai; model=gpt-5.6-sol; effort=xhigh ;; - claude) host=claude; provider=anthropic; model=claude-opus-5; effort=xhigh ;; - *) exit 2 ;; esac; - record="$artifact_dir/standards-route.tsv"; test -f "$record" || exit 1; - test ! -e "$artifact_dir/spec-route.tsv" || exit 1; - awk -F '\t' '$1 == "axis" && $2 == "standards" { found = 1 } END { exit !found }' - "$record" || exit 1; - agent=$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$record"); - test -n "$agent" || exit 1; - case "$DARROW_EVAL_HARNESS" in codex) route_field=requested_route; proof_field=route_bound ;; - claude) route_field=observed_route; proof_field=route_bound ;; esac; - awk -F '\t' -v f="$route_field" -v h="$host" -v p="$provider" -v m="$model" -v e="$effort" - '$1 == f && $2 == h && $3 == p && $4 == m && $5 == e { found = 1 } - END { exit !found }' "$record" || exit 1; - awk -F '\t' -v f="$proof_field" '$1 == f && $2 == "true" { found = 1 } - END { exit !found }' "$record" || exit 1; - if test "$DARROW_EVAL_HARNESS" = codex; then - native=$(grep -F '"type":"darrow.codex_native_spawn"' .git/retained-harness.jsonl | - grep -F '"status":"accepted"' | grep -F "\"agent_ref\":\"$agent\"" | - grep -F "\"model\":\"$model\"" | grep -F "\"reasoning_effort\":\"$effort\"") || exit 1; - case "$native" in *'"fork_turns":"none"'*) ;; *) exit 1 ;; esac; - case "$native" in *'"review_axis":"standards"'*) ;; *) exit 1 ;; esac; - test "$(printf '%s\n' "$native" | grep -c .)" -eq 1 || exit 1; - test "$(grep -F '"type":"darrow.codex_native_spawn"' .git/retained-harness.jsonl | - grep -c .)" -eq 1 || exit 1; fi; - if test "$DARROW_EVAL_HARNESS" = claude; then observed="$artifact_dir/standards-observed-route.tsv"; - test -f "$observed" || exit 1; - test "$agent" = "$(awk -F '\t' '$1 == "agent_id" { print $2 }' "$observed")" || exit 1; - awk -F '\t' -v m="$model" -v e="$effort" - '$1 == "observed_route" && $2 == "claude" && $3 == "anthropic" && $4 == m && $5 == e { found = 1 } - END { exit !found }' "$observed" || exit 1; - subagent="darrow-review:review-reader-$model-$effort"; - awk -v a="$subagent" - 'index($0, "\"name\":\"Agent\"") { - value = $0; agents += gsub("\"name\":\"Agent\"", "", value); - value = $0; types += gsub("\"subagent_type\":\"" a "\"", "", value); - if (index($0, "- review_axis: standards")) found = 1 } - END { exit !(agents == 1 && types == 1 && found) }' - .git/retained-harness.jsonl || exit 1; - launch=$(grep -F '"type":"darrow.review_agent_launch"' .git/retained-harness.jsonl | - grep -F "\"agent_id\":\"$agent\"" | grep -F '"review_axis":"standards"' | - grep -F "\"subagent_type\":\"$subagent\"") || exit 1; - test "$(printf '%s\n' "$launch" | grep -c .)" -eq 1 || exit 1; - ! grep -F '"type":"darrow.review_agent_launch"' .git/retained-harness.jsonl | - grep -F '"review_axis":"spec"' >/dev/null || exit 1; fi + uv run --quiet --frozen --no-dev --project .git/eval-checks/review + python .git/eval-checks/review/tests/evals/eval_routes.py + --host "$DARROW_EVAL_HARNESS" --profile default --axes standards - name: default response does not duplicate canonical TSV run: "! grep -F 'format darrow-review-result-v1' .git/last-message.md" - name: final response preserves every canonical rendered line diff --git a/plugins/capability/darrow-tickets-github/.claude-plugin/plugin.json b/plugins/capability/darrow-tickets-github/.claude-plugin/plugin.json index b3b9135a..6285b28d 100644 --- a/plugins/capability/darrow-tickets-github/.claude-plugin/plugin.json +++ b/plugins/capability/darrow-tickets-github/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "darrow-tickets-github", "description": "GitHub Issues ticket skills: create-ticket, read-ticket, update-ticket, list-tickets", - "version": "0.4.1", + "version": "0.4.2", "hooks": "./.claude-plugin/hooks.json", "license": "BUSL-1.1", "author": { diff --git a/plugins/capability/darrow-tickets-github/.codex-plugin/plugin.json b/plugins/capability/darrow-tickets-github/.codex-plugin/plugin.json index 085a0d7a..ff8b7069 100644 --- a/plugins/capability/darrow-tickets-github/.codex-plugin/plugin.json +++ b/plugins/capability/darrow-tickets-github/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-tickets-github", - "version": "0.4.1", + "version": "0.4.2", "description": "GitHub Issues ticket skills: create-ticket, read-ticket, update-ticket, list-tickets", "author": { "name": "Björn Rochel" diff --git a/plugins/capability/darrow-tickets-github/backend/tests/eval_gh.py b/plugins/capability/darrow-tickets-github/backend/tests/eval_gh.py new file mode 100644 index 00000000..ff4491b9 --- /dev/null +++ b/plugins/capability/darrow-tickets-github/backend/tests/eval_gh.py @@ -0,0 +1,217 @@ +"""Stateful GitHub fixture for ticket evals, independent of the ticket facade.""" + +import json +import re +import shutil +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any + + +def flag(arguments: list[str], name: str, default: str = "") -> str: + try: + return arguments[arguments.index(name) + 1] + except ValueError: + return default + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.exists() else "" + + +def append(path: Path, text: str) -> None: + with path.open("a", encoding="utf-8") as stream: + stream.write(text + "\n") + + +def issue_list(directory: Path) -> list[dict[str, Any]]: + records = [] + for line in read(directory / "list").splitlines(): + match = re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) + assert match, line + labels = match[4].split(", ") if match[4] else [] + records.append( + { + "number": int(match[1]), + "state": match[2].upper(), + "title": match[3], + "labels": [{"name": n} for n in labels], + } + ) + return records + + +def issue_view(directory: Path, arguments: list[str]) -> dict[str, Any]: + number = arguments[2] + if not (directory / f"issue-{number}-state").exists(): + raise ValueError( + f"GraphQL: Could not resolve to an Issue with the number of {number}." + ) + fields = { + name: read(directory / f"issue-{number}-{name}") + for name in ("state", "title", "body", "labels") + } + record: dict[str, Any] = { + "number": int(number), + "state": fields["state"].upper(), + "title": fields["title"], + "body": fields["body"], + "url": f"https://github.test/o/r/issues/{number}", + "labels": [{"name": n} for n in fields["labels"].splitlines()], + } + return {field: record[field] for field in flag(arguments, "--json").split(",")} + + +def copy_body(directory: Path, arguments: list[str], *names: str) -> None: + source = flag(arguments, "--body-file") + if source: + for name in names: + shutil.copyfile(source, directory / name) + + +def create_issue(directory: Path, arguments: list[str]) -> str: + copy_body(directory, arguments, "created-body") + title = flag(arguments, "--title") + if "--title" in arguments: + (directory / "created-title").write_text(title, encoding="utf-8") + labels = [arguments[i + 1] for i, arg in enumerate(arguments) if arg == "--label"] + (directory / "created-labels").write_text( + "".join(label + "\n" for label in labels), encoding="utf-8" + ) + (directory / "issue-99-state").write_text("open", encoding="utf-8") + for name in ("title", "body"): + source = directory / f"created-{name}" + if source.exists(): + shutil.copyfile(source, directory / f"issue-99-{name}") + return "https://github.test/o/r/issues/99" + + +def mutate_issue(directory: Path, arguments: list[str]) -> str | None: + action, number = arguments[1:3] + if action == "comment": + copy_body(directory, arguments, f"comment-body-{number}") + return f"https://github.test/o/r/issues/{number}#issuecomment-1" + if action == "edit": + copy_body(directory, arguments, f"edited-body-{number}", f"issue-{number}-body") + else: + append(directory / "transitions", f"{action} {number}") + return None + + +def api_arguments(arguments: list[str]) -> tuple[str, str, dict[str, str]]: + method, endpoint, fields = "GET", "", {} + iterator = iter(arguments[1:]) + for argument in iterator: + if argument == "-X": + method = next(iterator) + elif argument == "-F": + key, value = next(iterator).split("=", 1) + fields[key] = value + elif argument not in ("--paginate", "--slurp"): + endpoint = argument + return method, endpoint, fields + + +def api_get(directory: Path, number: str, suffix: str) -> Any: + prefix = directory / f"issue-{number}" + if not suffix: + if not prefix.with_name(prefix.name + "-state").exists(): + raise ValueError("gh: Not Found (HTTP 404)") + return {"id": 10000 + int(number)} + if suffix == "/parent": + parent = read(prefix.with_name(prefix.name + "-parent")).strip() + if not parent: + raise ValueError("gh: No parent issue found (HTTP 404)") + return { + "number": int(parent), + "html_url": f"https://github.test/o/r/issues/{parent}", + } + return [ + [ + {"number": int(n)} + for n in read(directory / f"issue-{number}-blockedby").splitlines() + ] + ] + + +def api_mutate( + directory: Path, number: str, method: str, suffix: str, fields: dict[str, str] +) -> None: + blocked = directory / f"issue-{number}-blockedby" + if method == "POST" and suffix == "/dependencies/blocked_by": + append(blocked, str(int(fields["issue_id"]) - 10000)) + elif method == "DELETE" and suffix.startswith("/dependencies/blocked_by/"): + removed = str(int(suffix.rsplit("/", 1)[1]) - 10000) + blocked.write_text( + "".join(n + "\n" for n in read(blocked).splitlines() if n != removed), + encoding="utf-8", + ) + else: + parent = directory / f"issue-{int(fields['sub_issue_id']) - 10000}-parent" + if method == "POST": + parent.write_text(number + "\n", encoding="utf-8") + else: + parent.unlink(missing_ok=True) + + +def api(directory: Path, arguments: list[str]) -> Any: + method, endpoint, fields = api_arguments(arguments) + match = re.fullmatch(r".*/issues/([0-9]+)(.*)", endpoint) + if not match: + raise ValueError(f"mock gh: unsupported api call: {method} {endpoint}") + number, suffix = match.groups() + if method == "GET" and suffix in ("", "/parent", "/dependencies/blocked_by"): + return api_get(directory, number, suffix) + supported = ( + method == "POST" and suffix in ("/dependencies/blocked_by", "/sub_issues") + ) or ( + method == "DELETE" + and (suffix == "/sub_issue" or suffix.startswith("/dependencies/blocked_by/")) + ) + if not supported: + raise ValueError(f"mock gh: unsupported api call: {method} {endpoint}") + api_mutate(directory, number, method, suffix, fields) + return None + + +def dispatch(directory: Path, arguments: list[str]) -> Any: + pair = tuple(arguments[:2]) + queries: dict[tuple[str, ...], Callable[[], Any]] = { + ("repo", "view"): lambda: {"hasIssuesEnabled": True}, + ("label", "list"): lambda: [ + {"name": n} for n in read(directory / "labels").splitlines() + ], + ("issue", "list"): lambda: issue_list(directory), + ("issue", "view"): lambda: issue_view(directory, arguments), + ("issue", "create"): lambda: create_issue(directory, arguments), + } + if arguments[0] == "api": + return api(directory, arguments) + if pair in queries: + return queries[pair]() + if pair in {("issue", action) for action in ("comment", "edit", "close", "reopen")}: + return mutate_issue(directory, arguments) + raise ValueError("mock gh: unsupported: " + " ".join(arguments)) + + +def main() -> None: + git_dir = subprocess.check_output( + ["git", "rev-parse", "--absolute-git-dir"], text=True + ).strip() + directory = Path(git_dir) / "fixture-gh" + directory.mkdir(exist_ok=True) + arguments = sys.argv[1:] + append(directory / "calls", " ".join(arguments)) + try: + result = dispatch(directory, arguments) + if result is not None: + print(result if isinstance(result, str) else json.dumps(result)) + except (ValueError, IndexError, StopIteration) as error: + print(error, file=sys.stderr) + raise SystemExit(1) from error + + +if __name__ == "__main__": + main() diff --git a/plugins/capability/darrow-tickets-github/backend/tests/test_eval_gh.py b/plugins/capability/darrow-tickets-github/backend/tests/test_eval_gh.py new file mode 100644 index 00000000..1b859869 --- /dev/null +++ b/plugins/capability/darrow-tickets-github/backend/tests/test_eval_gh.py @@ -0,0 +1,170 @@ +"""Public fixture responses and effects used by ticket evals.""" + +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.eval_gh import dispatch + + +def test_queries_and_field_selection(tmp_path: Path) -> None: + assert dispatch(tmp_path, ["issue", "list"]) == [] + assert dispatch(tmp_path, ["label", "list"]) == [] + assert dispatch(tmp_path, ["repo", "view"]) == {"hasIssuesEnabled": True} + (tmp_path / "labels").write_text("bug\nmaintenance\n") + (tmp_path / "list").write_text( + "#12 open A title (bug, maintenance)\n#13 closed Plain\n" + ) + assert dispatch(tmp_path, ["label", "list"]) == [ + {"name": "bug"}, + {"name": "maintenance"}, + ] + assert dispatch(tmp_path, ["issue", "list"]) == [ + { + "number": 12, + "state": "OPEN", + "title": "A title", + "labels": [{"name": "bug"}, {"name": "maintenance"}], + }, + {"number": 13, "state": "CLOSED", "title": "Plain", "labels": []}, + ] + (tmp_path / "issue-12-state").write_text("open") + (tmp_path / "issue-12-title").write_text('Quoted "title"') + assert dispatch( + tmp_path, ["issue", "view", "12", "--json", "number,title,state"] + ) == { + "number": 12, + "title": 'Quoted "title"', + "state": "OPEN", + } + + +def test_creation_comments_edits_and_transitions(tmp_path: Path) -> None: + body = tmp_path / "body with spaces" + body.write_bytes(b"body\nwith newline\n") + assert ( + dispatch( + tmp_path, + [ + "issue", + "create", + "--title", + "Unicode ä", + "--body-file", + str(body), + "--label", + "bug", + "--label", + "maintenance", + ], + ) + == "https://github.test/o/r/issues/99" + ) + assert (tmp_path / "issue-99-title").read_text(encoding="utf-8") == "Unicode ä" + assert (tmp_path / "issue-99-body").read_bytes() == body.read_bytes() + assert (tmp_path / "created-labels").read_text() == "bug\nmaintenance\n" + assert ( + dispatch( + tmp_path, + [ + "issue", + "comment", + "99", + "--body-file", + str(body), + ], + ) + == "https://github.test/o/r/issues/99#issuecomment-1" + ) + assert (tmp_path / "comment-body-99").read_bytes() == body.read_bytes() + body.write_text("replacement") + dispatch(tmp_path, ["issue", "edit", "99", "--body-file", str(body)]) + assert (tmp_path / "edited-body-99").read_text() == "replacement" + assert (tmp_path / "issue-99-body").read_text() == "replacement" + for action in ("close", "reopen"): + dispatch(tmp_path, ["issue", action, "99"]) + assert (tmp_path / "transitions").read_text() == "close 99\nreopen 99\n" + + +def test_relationships(tmp_path: Path) -> None: + (tmp_path / "issue-7-state").write_text("open") + endpoint = "repos/o/r/issues/7" + assert dispatch(tmp_path, ["api", endpoint]) == {"id": 10007} + assert dispatch(tmp_path, ["api", endpoint + "/dependencies/blocked_by"]) == [[]] + dispatch( + tmp_path, + [ + "api", + "-X", + "POST", + endpoint + "/dependencies/blocked_by", + "-F", + "issue_id=10009", + ], + ) + assert dispatch( + tmp_path, + ["api", "--paginate", "--slurp", endpoint + "/dependencies/blocked_by"], + ) == [[{"number": 9}]] + dispatch( + tmp_path, ["api", "-X", "DELETE", endpoint + "/dependencies/blocked_by/10009"] + ) + assert (tmp_path / "issue-7-blockedby").read_text() == "" + dispatch( + tmp_path, + ["api", "-X", "POST", endpoint + "/sub_issues", "-F", "sub_issue_id=10099"], + ) + assert dispatch(tmp_path, ["api", "repos/o/r/issues/99/parent"]) == { + "number": 7, + "html_url": "https://github.test/o/r/issues/7", + } + dispatch( + tmp_path, + ["api", "-X", "DELETE", endpoint + "/sub_issue", "-F", "sub_issue_id=10099"], + ) + assert not (tmp_path / "issue-99-parent").exists() + + +@pytest.mark.parametrize( + "arguments,diagnostic", + [ + (["api", "repos/o/r/issues/8"], "Not Found"), + (["api", "repos/o/r/issues/8/parent"], "No parent issue"), + (["issue", "view", "8", "--json", "title"], "Could not resolve"), + (["api", "bad"], "unsupported api call"), + (["api", "-X", "PATCH", "repos/o/r/issues/8"], "unsupported api call"), + (["release", "delete"], "unsupported"), + ], +) +def test_refusals(tmp_path: Path, arguments: list[str], diagnostic: str) -> None: + with pytest.raises(ValueError, match=diagnostic): + dispatch(tmp_path, arguments) + + +def test_cli_records_calls_and_reports_failure(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + script = Path(__file__).with_name("eval_gh.py") + command = [sys.executable, str(script)] + result = subprocess.run( + [*command, "issue", "list"], + cwd=tmp_path, + capture_output=True, + text=True, + check=True, + ) + assert json.loads(result.stdout) == [] + result = subprocess.run( + [*command, "issue", "view", "9"], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 1 + assert "Could not resolve" in result.stderr + assert ( + tmp_path / ".git/fixture-gh/calls" + ).read_text() == "issue list\nissue view 9\n" diff --git a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/dedup-stop.yaml b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/dedup-stop.yaml index 66d4d04b..6c7ea0c8 100644 --- a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/dedup-stop.yaml +++ b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/dedup-stop.yaml @@ -13,144 +13,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/no-attribution.yaml b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/no-attribution.yaml index 229ba95a..7e0daaab 100644 --- a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/no-attribution.yaml +++ b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/no-attribution.yaml @@ -15,144 +15,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/relations-by-request.yaml b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/relations-by-request.yaml index 7b635949..81bd663b 100644 --- a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/relations-by-request.yaml +++ b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/relations-by-request.yaml @@ -14,144 +14,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ntask\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/structure-evidence.yaml b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/structure-evidence.yaml index b14954b7..ad24eb72 100644 --- a/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/structure-evidence.yaml +++ b/plugins/capability/darrow-tickets-github/skills/create-ticket/evals/structure-evidence.yaml @@ -14,144 +14,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/filter-readonly.yaml b/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/filter-readonly.yaml index 6225992e..ad1508a5 100644 --- a/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/filter-readonly.yaml +++ b/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/filter-readonly.yaml @@ -11,144 +11,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/state-asked.yaml b/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/state-asked.yaml index fbe0cc27..6b895936 100644 --- a/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/state-asked.yaml +++ b/plugins/capability/darrow-tickets-github/skills/list-tickets/evals/state-asked.yaml @@ -11,144 +11,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/ambiguous-target.yaml b/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/ambiguous-target.yaml index c709e915..80a31b34 100644 --- a/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/ambiguous-target.yaml +++ b/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/ambiguous-target.yaml @@ -11,144 +11,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/close-transition.yaml b/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/close-transition.yaml index f98173d6..85b0c452 100644 --- a/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/close-transition.yaml +++ b/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/close-transition.yaml @@ -11,144 +11,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/comment-only.yaml b/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/comment-only.yaml index 6b2e5237..c1d076a5 100644 --- a/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/comment-only.yaml +++ b/plugins/capability/darrow-tickets-github/skills/update-ticket/evals/comment-only.yaml @@ -14,144 +14,13 @@ fixture: bin: gh: | #!/bin/sh - d="$(git rev-parse --git-dir)/fixture-gh" - mkdir -p "$d" - echo "$*" >> "$d/calls" - find_flag() { - want=$1; shift - prev="" - for a in "$@"; do - if [ "$prev" = "$want" ]; then printf '%s' "$a"; return 0; fi - prev=$a - done - return 1 - } - if [ "$1" = "api" ]; then - shift - method=GET; endpoint=""; f_issue=""; f_sub="" - while [ $# -gt 0 ]; do - case "$1" in - -X) method=$2; shift 2 ;; - --paginate|--slurp) shift ;; - -F) - case "$2" in - issue_id=*) f_issue=${2#issue_id=} ;; - sub_issue_id=*) f_sub=${2#sub_issue_id=} ;; - esac - shift 2 ;; - *) endpoint=$1; shift ;; - esac - done - n=${endpoint##*/issues/}; n=${n%%/*} - suffix=${endpoint#*/issues/"$n"} - case "$method $suffix" in - "GET ") - if [ ! -f "$d/issue-$n-state" ]; then - echo "gh: Not Found (HTTP 404)" >&2 - exit 1 - fi - printf '{"id":%s}\n' $((10000 + n)) - ;; - "GET /parent") - if [ -f "$d/issue-$n-parent" ]; then - printf '{"number":%s,"html_url":"https://github.test/o/r/issues/%s"}\n' "$(cat "$d/issue-$n-parent")" "$(cat "$d/issue-$n-parent")" - else - echo "gh: No parent issue found (HTTP 404)" >&2 - exit 1 - fi - ;; - "GET /dependencies/blocked_by") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([[{"number":int(n)} for n in p.read_text().splitlines()]] if p.exists() else [[]]))' "$d/issue-$n-blockedby" - ;; - "POST /dependencies/blocked_by") - echo $((f_issue - 10000)) >> "$d/issue-$n-blockedby" - ;; - "DELETE /dependencies/blocked_by/"*) - dep=$(( ${suffix##*/} - 10000 )) - grep -vx -- "$dep" "$d/issue-$n-blockedby" > "$d/issue-$n-blockedby.new" 2>/dev/null || : - mv "$d/issue-$n-blockedby.new" "$d/issue-$n-blockedby" - ;; - "POST /sub_issues") - echo "$n" > "$d/issue-$((f_sub - 10000))-parent" - ;; - "DELETE /sub_issue") - rm -f "$d/issue-$((f_sub - 10000))-parent" - ;; - *) - echo "mock gh: unsupported api call: $method $endpoint" >&2 - exit 1 - ;; - esac - exit 0 - fi - case "$1 $2" in - "repo view") - printf '{"hasIssuesEnabled":true}\n' - ;; - "label list") - python3 -c 'import json,pathlib,sys; p=pathlib.Path(sys.argv[1]); print(json.dumps([{"name":n} for n in p.read_text().splitlines()] if p.exists() else []))' "$d/labels" - ;; - "issue list") - python3 -c 'import json,pathlib,re,sys; p=pathlib.Path(sys.argv[1]); rows=[] - for line in p.read_text().splitlines() if p.exists() else []: - m=re.fullmatch(r"#([0-9]+) (open|closed) (.*?)(?: \((.*)\))?", line) - assert m, line - rows.append({"number":int(m[1]),"state":m[2].upper(),"title":m[3],"labels":[{"name":n} for n in m[4].split(", ")] if m[4] else []}) - print(json.dumps(rows))' "$d/list" - ;; - "issue view") - id=$3 - if [ ! -f "$d/issue-$id-state" ]; then - echo "GraphQL: Could not resolve to an Issue with the number of $id." >&2 - exit 1 - fi - json=$(find_flag --json "$@") - python3 - "$d" "$id" "$json" <<'PY' - import json, pathlib, sys - directory, number, fields = sys.argv[1:] - def read(field): - path = pathlib.Path(directory) / f"issue-{number}-{field}" - return path.read_text() if path.exists() else "" - record = {"number": int(number), "state": read("state").upper(), - "title": read("title"), "body": read("body"), - "url": f"https://github.test/o/r/issues/{number}", - "labels": [{"name": name} for name in read("labels").splitlines()]} - print(json.dumps({field: record[field] for field in fields.split(",")})) - PY - ;; - "issue create") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/created-body"; fi - if tl=$(find_flag --title "$@"); then printf '%s' "$tl" > "$d/created-title"; fi - : > "$d/created-labels" - prev="" - for a in "$@"; do - if [ "$prev" = "--label" ]; then echo "$a" >> "$d/created-labels"; fi - prev=$a - done - printf 'open' > "$d/issue-99-state" - if [ -f "$d/created-title" ]; then cp "$d/created-title" "$d/issue-99-title"; fi - if [ -f "$d/created-body" ]; then cp "$d/created-body" "$d/issue-99-body"; fi - echo "https://github.test/o/r/issues/99" - ;; - "issue comment") - if bf=$(find_flag --body-file "$@"); then cp "$bf" "$d/comment-body-$3"; fi - echo "https://github.test/o/r/issues/$3#issuecomment-1" - ;; - "issue edit") - if bf=$(find_flag --body-file "$@"); then - cp "$bf" "$d/edited-body-$3" - cp "$bf" "$d/issue-$3-body" - fi - ;; - "issue close"|"issue reopen") - echo "$2 $3" >> "$d/transitions" - ;; - *) - echo "mock gh: unsupported: $*" >&2 - exit 1 - ;; - esac + backend="$(git rev-parse --absolute-git-dir)/tickets-fixture" + exec uv run --quiet --frozen --no-dev --project "$backend" python "$backend/tests/eval_gh.py" "$@" setup: | + mkdir -p .git/tickets-fixture/tests + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/tickets-fixture/ + cp -R "{{case_dir}}/../../../backend/src" .git/tickets-fixture/ + cp "{{case_dir}}/../../../backend/tests/eval_gh.py" .git/tickets-fixture/tests/ git remote add origin https://github.test/o/r.git mkdir -p .git/fixture-gh printf 'bug\nenhancement\ndocumentation\n' > .git/fixture-gh/labels diff --git a/plugins/capability/darrow-verification/.claude-plugin/plugin.json b/plugins/capability/darrow-verification/.claude-plugin/plugin.json index 0035ba09..387b39d2 100644 --- a/plugins/capability/darrow-verification/.claude-plugin/plugin.json +++ b/plugins/capability/darrow-verification/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-verification", - "version": "0.2.3", + "version": "0.2.4", "description": "Bounded acceptance verification through replaceable independent code review", "license": "BUSL-1.1", "author": { diff --git a/plugins/capability/darrow-verification/.codex-plugin/plugin.json b/plugins/capability/darrow-verification/.codex-plugin/plugin.json index fd7b3cc7..c8ae9808 100644 --- a/plugins/capability/darrow-verification/.codex-plugin/plugin.json +++ b/plugins/capability/darrow-verification/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-verification", - "version": "0.2.3", + "version": "0.2.4", "description": "Bounded acceptance verification through replaceable independent code review", "author": { "name": "Björn Rochel", diff --git a/plugins/capability/darrow-verification/skills/verify-change/backend/tests/fresh_install.py b/plugins/capability/darrow-verification/skills/verify-change/backend/tests/fresh_install.py new file mode 100644 index 00000000..04725327 --- /dev/null +++ b/plugins/capability/darrow-verification/skills/verify-change/backend/tests/fresh_install.py @@ -0,0 +1,137 @@ +"""Validate a fresh copied plugin using only locked runtime dependencies.""" + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + + +def command(cwd: Path, *args: str) -> bytes: + return subprocess.run(args, cwd=cwd, check=True, capture_output=True).stdout + + +def installed_runtime(backend: Path, fixture: Path) -> None: + command(fixture, "uv", "sync", "--frozen", "--no-dev", "--project", str(backend)) + tree = command( + fixture, "uv", "tree", "--frozen", "--no-dev", "--project", str(backend) + ).decode() + assert not re.search(r"\b(coverage|hypothesis|mypy|pytest|ruff)\b", tree), tree + + +def probe( + backend: Path, + cwd: Path, + args: list[str], + *, + output: str = "", + error: str = "", + status: int = 0, +) -> None: + result = subprocess.run( + [ + "uv", + "run", + "--quiet", + "--isolated", + "--frozen", + "--no-dev", + "--project", + str(backend), + "darrow-render-assessment", + *args, + ], + cwd=cwd, + capture_output=True, + check=False, + ) + assert (result.returncode, result.stdout, result.stderr) == ( + status, + output.encode("utf-8"), + error.encode("utf-8"), + ), result + + +def validate(backend: Path, fixture: Path) -> None: + shutil.rmtree(backend / ".venv") + outside = fixture / "outside" + outside.mkdir() + assessment = fixture / "assessment.md" + report = fixture / ( + "provider report #%.md" if os.name == "nt" else "provider report #%<>.md" + ) + assessment_text = "Conclusion: progress.\n\nF1 remains blocking; A1 is advisory.\n" + assessment.write_bytes(assessment_text.encode()) + report_bytes = b"Complete provider report, unchanged.\n" + report.write_bytes(report_bytes) + # Independent encoding oracle; preserve ordinary path punctuation. + from urllib.parse import quote + + destination = quote(str(report.resolve()), safe="/:().-_~") + probe( + backend, + outside, + ["--assessment", str(assessment), "--provider-report", str(report)], + output=assessment_text + + "\n\nComplete provider result: " + + f"[report](<{destination}>)\n", + ) + assert report.read_bytes() == report_bytes + probe( + backend, + outside, + [], + status=2, + error=( + "render-assessment: usage: render-assessment --assessment ABSOLUTE_FILE " + "--provider-report ABSOLUTE_FILE\n" + ), + ) + probe( + backend, + outside, + ["--assessment", str(assessment), "--provider-report", "relative.md"], + status=2, + error="render-assessment: provider-report requires an absolute path\n", + ) + empty = fixture / "empty.md" + empty.touch() + probe( + backend, + outside, + ["--assessment", str(empty), "--provider-report", str(report)], + status=2, + error="render-assessment: assessment is not a readable " + + f"nonempty regular file: {empty}\n", + ) + assert not (backend / ".venv").exists() + + +def main() -> None: + plugin = Path(__file__).resolve().parents[4] + with tempfile.TemporaryDirectory(prefix="darrow verify-change ") as temporary: + fixture = Path(temporary).resolve() + copy = fixture / "plugin copy" + shutil.copytree( + plugin, + copy, + ignore=shutil.ignore_patterns( + ".venv", + "__pycache__", + ".hypothesis", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".coverage*", + "coverage.json", + ), + ) + backend = copy / "skills" / "verify-change" / "backend" + installed_runtime(backend, fixture) + validate(backend, fixture) + print("verify-change fresh install passed") + + +if __name__ == "__main__": + main() diff --git a/plugins/capability/darrow-verification/tests/fresh-install.test.ps1 b/plugins/capability/darrow-verification/tests/fresh-install.test.ps1 deleted file mode 100644 index 57965e80..00000000 --- a/plugins/capability/darrow-verification/tests/fresh-install.test.ps1 +++ /dev/null @@ -1,135 +0,0 @@ -$ErrorActionPreference = "Stop" - -$pluginRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -$fixture = Join-Path ([System.IO.Path]::GetTempPath()) ("darrow verification " + [guid]::NewGuid()) -$copy = Join-Path $fixture "plugin copy" -$utf8 = [System.Text.UTF8Encoding]::new($false) - -function Assert-Bytes([string]$Path, [byte[]]$Expected, [string]$Message) { - $actual = [System.IO.File]::ReadAllBytes($Path) - if ([Convert]::ToBase64String($actual) -ne [Convert]::ToBase64String($Expected)) { - throw $Message - } -} - -function Invoke-Renderer( - [string]$Backend, - [string]$WorkingDirectory, - [string[]]$Arguments, - [string]$StdoutPath, - [string]$StderrPath -) { - $startInfo = [System.Diagnostics.ProcessStartInfo]::new() - $startInfo.FileName = (Get-Command uv).Source - $startInfo.UseShellExecute = $false - $startInfo.RedirectStandardOutput = $true - $startInfo.RedirectStandardError = $true - $startInfo.WorkingDirectory = $WorkingDirectory - foreach ($argument in @( - "run", "--quiet", "--isolated", "--frozen", "--no-dev", "--project", $Backend, - "darrow-render-assessment" - ) + $Arguments) { - [void]$startInfo.ArgumentList.Add($argument) - } - - $process = [System.Diagnostics.Process]::new() - $process.StartInfo = $startInfo - $stdoutStream = $null - $stderrStream = $null - try { - if (-not $process.Start()) { throw "renderer process did not start" } - $stdoutStream = [System.IO.File]::Open( - $StdoutPath, - [System.IO.FileMode]::Create, - [System.IO.FileAccess]::Write, - [System.IO.FileShare]::None - ) - $stderrStream = [System.IO.File]::Open( - $StderrPath, - [System.IO.FileMode]::Create, - [System.IO.FileAccess]::Write, - [System.IO.FileShare]::None - ) - $stdoutCopy = $process.StandardOutput.BaseStream.CopyToAsync($stdoutStream) - $stderrCopy = $process.StandardError.BaseStream.CopyToAsync($stderrStream) - $process.WaitForExit() - [System.Threading.Tasks.Task]::WaitAll( - [System.Threading.Tasks.Task[]]@($stdoutCopy, $stderrCopy) - ) - return $process.ExitCode - } - finally { - if ($null -ne $stdoutStream) { $stdoutStream.Dispose() } - if ($null -ne $stderrStream) { $stderrStream.Dispose() } - $process.Dispose() - } -} - -try { - New-Item -ItemType Directory -Path $fixture | Out-Null - Copy-Item -LiteralPath $pluginRoot -Destination $copy -Recurse - $backend = Join-Path $copy "skills/verify-change/backend" - foreach ($generatedDirectory in @(".venv", ".hypothesis", ".mypy_cache", ".pytest_cache", ".ruff_cache")) { - Remove-Item -LiteralPath (Join-Path $backend $generatedDirectory) -Recurse -Force -ErrorAction SilentlyContinue - } - Get-ChildItem -LiteralPath $backend -Directory -Recurse -Filter "__pycache__" | - Remove-Item -Recurse -Force - Get-ChildItem -LiteralPath $backend -File -Filter ".coverage*" | - Remove-Item -Force - Remove-Item -LiteralPath (Join-Path $backend "coverage.json") -Force -ErrorAction SilentlyContinue - - & uv sync --locked --no-dev --project $backend - if ($LASTEXITCODE -ne 0) { throw "runtime sync failed" } - $runtimeTree = (& uv tree --locked --no-dev --project $backend) -join "`n" - if ($LASTEXITCODE -ne 0) { throw "runtime tree failed" } - foreach ($tool in @("coverage", "hypothesis", "mypy", "pytest", "ruff")) { - if ($runtimeTree -match "(?m)(^|\s)$tool(\s|$)") { - throw "development dependency installed at runtime: $tool" - } - } - Remove-Item -LiteralPath (Join-Path $backend ".venv") -Recurse -Force - - $outside = Join-Path $fixture "outside" - New-Item -ItemType Directory -Path $outside | Out-Null - $assessment = Join-Path $fixture "assessment.md" - $report = Join-Path $fixture "provider report #%.md" - $stdout = Join-Path $fixture "stdout" - $stderr = Join-Path $fixture "stderr" - $assessmentText = "Conclusion: progress.`n`nF1 remains blocking; A1 is advisory.`n" - [System.IO.File]::WriteAllText($assessment, $assessmentText, $utf8) - [System.IO.File]::WriteAllText($report, "Complete provider report, unchanged.`n", $utf8) - $reportBefore = [System.IO.File]::ReadAllBytes($report) - - $exitCode = Invoke-Renderer $backend $outside @( - "--assessment", $assessment, "--provider-report", $report - ) $stdout $stderr - if ($exitCode -ne 0) { throw "renderer failed" } - $destination = ([System.IO.Path]::GetFullPath($report)).Replace("%", "%25") - $destination = $destination.Replace(" ", "%20").Replace("#", "%23") - $destination = $destination.Replace("?", "%3F").Replace("<", "%3C").Replace(">", "%3E") - $destination = $destination.Replace("\", "%5C") - $expected = $assessmentText + "`n`nComplete provider result: [report](<$destination>)`n" - Assert-Bytes $stdout ($utf8.GetBytes($expected)) "renderer output bytes differed from the public contract" - Assert-Bytes $stderr ([byte[]]@()) "valid renderer invocation wrote stderr" - Assert-Bytes $report $reportBefore "renderer changed the provider report" - - $exitCode = Invoke-Renderer $backend $outside @() $stdout $stderr - if ($exitCode -ne 2) { throw "invalid renderer invocation did not exit 2" } - Assert-Bytes $stdout ([byte[]]@()) "invalid renderer invocation wrote stdout" - $usage = "render-assessment: usage: render-assessment --assessment ABSOLUTE_FILE --provider-report ABSOLUTE_FILE`n" - Assert-Bytes $stderr ($utf8.GetBytes($usage)) "renderer usage bytes differed" - - $exitCode = Invoke-Renderer $backend $outside @( - "--assessment", $assessment, "--provider-report", "relative.md" - ) $stdout $stderr - if ($exitCode -ne 2) { throw "relative report path did not exit 2" } - Assert-Bytes $stdout ([byte[]]@()) "relative report path wrote stdout" - $diagnostic = "render-assessment: provider-report requires an absolute path`n" - Assert-Bytes $stderr ($utf8.GetBytes($diagnostic)) "relative report diagnostic differed" - if (Test-Path -LiteralPath (Join-Path $backend ".venv")) { - throw "renderer wrote a virtual environment into the installed skill" - } -} -finally { - Remove-Item -LiteralPath $fixture -Recurse -Force -ErrorAction SilentlyContinue -} diff --git a/plugins/capability/darrow-verification/tests/fresh-install.test.sh b/plugins/capability/darrow-verification/tests/fresh-install.test.sh deleted file mode 100755 index 1bbb0a78..00000000 --- a/plugins/capability/darrow-verification/tests/fresh-install.test.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/sh -set -eu - -plugin_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -temp_parent=${TMPDIR:-/tmp} -case "$temp_parent" in - /) temp_template=/darrow-verification-install.XXXXXX ;; - */) temp_template=${temp_parent}darrow-verification-install.XXXXXX ;; - *) temp_template=$temp_parent/darrow-verification-install.XXXXXX ;; -esac -fixture=$(mktemp -d "$temp_template") -fixture=$(CDPATH='' cd -- "$fixture" && pwd -P) -trap 'rm -rf "$fixture"' EXIT HUP INT TERM - -copy="$fixture/plugin copy" -mkdir -p "$copy" -cp -R "$plugin_root/." "$copy/" -backend="$copy/skills/verify-change/backend" -rm -rf \ - "$backend/.venv" \ - "$backend/.hypothesis" \ - "$backend/.mypy_cache" \ - "$backend/.pytest_cache" \ - "$backend/.ruff_cache" -rm -f "$backend"/.coverage* "$backend/coverage.json" -find "$backend" -type d -name __pycache__ -prune -exec rm -rf {} \; - -uv sync --locked --no-dev --project "$backend" -runtime_tree=$(uv tree --locked --no-dev --project "$backend") -for development_tool in coverage hypothesis mypy pytest ruff; do - if printf '%s\n' "$runtime_tree" | grep -E "(^|[[:space:]])${development_tool}([[:space:]]|$)" >/dev/null; then - echo "error: development dependency installed at runtime: $development_tool" >&2 - exit 1 - fi -done -rm -rf "$backend/.venv" - -mkdir "$fixture/outside" -assessment="$fixture/assessment.md" -report="$fixture/provider report #%<>.md" -actual="$fixture/actual" -errors="$fixture/errors" -expected="$fixture/expected" -printf 'Conclusion: progress.\n\nF1 remains blocking; A1 is advisory.\n' >"$assessment" -printf 'Complete provider report, unchanged.\n' >"$report" -cp "$report" "$fixture/report-before.md" - -cd "$fixture/outside" -uv run --quiet --isolated --frozen --no-dev --project "$backend" \ - darrow-render-assessment \ - --assessment "$assessment" --provider-report "$report" \ - >"$actual" 2>"$errors" - -canonical_report=$(CDPATH='' cd -- "${report%/*}" && pwd -P)/${report##*/} -destination=$(printf '%s\n' "$canonical_report" | LC_ALL=C sed \ - -e 's/%/%25/g' -e 's/ /%20/g' -e 's/#/%23/g' -e 's/?/%3F/g' \ - -e 's//%3E/g' -e 's/\\/%5C/g') -{ - cat "$assessment" - printf '\n\nComplete provider result: [report](<%s>)\n' "$destination" -} >"$expected" -cmp "$expected" "$actual" -cmp "$fixture/report-before.md" "$report" -test ! -s "$errors" - -status=0 -uv run --quiet --isolated --frozen --no-dev --project "$backend" \ - darrow-render-assessment >"$actual" 2>"$errors" || status=$? -test "$status" -eq 2 -test ! -s "$actual" -test "$(cat "$errors")" = \ - 'render-assessment: usage: render-assessment --assessment ABSOLUTE_FILE --provider-report ABSOLUTE_FILE' - -status=0 -uv run --quiet --isolated --frozen --no-dev --project "$backend" \ - darrow-render-assessment \ - --assessment "$assessment" --provider-report relative.md \ - >"$actual" 2>"$errors" || status=$? -test "$status" -eq 2 -test ! -s "$actual" -test "$(cat "$errors")" = \ - 'render-assessment: provider-report requires an absolute path' - -empty="$fixture/empty.md" -: >"$empty" -status=0 -uv run --quiet --isolated --frozen --no-dev --project "$backend" \ - darrow-render-assessment \ - --assessment "$empty" --provider-report "$report" \ - >"$actual" 2>"$errors" || status=$? -test "$status" -eq 2 -test ! -s "$actual" -test "$(cat "$errors")" = \ - "render-assessment: assessment is not a readable nonempty regular file: $empty" -test ! -e "$backend/.venv" - -printf '%s\n' 'verification fresh install passed' diff --git a/plugins/foundation/darrow-information-architecture/.claude-plugin/plugin.json b/plugins/foundation/darrow-information-architecture/.claude-plugin/plugin.json index f2bef465..7c39f53c 100644 --- a/plugins/foundation/darrow-information-architecture/.claude-plugin/plugin.json +++ b/plugins/foundation/darrow-information-architecture/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-information-architecture", "description": "Set up and doctor lean, routed repository information architecture", - "version": "0.2.2", + "version": "0.2.3", "license": "BUSL-1.1" } diff --git a/plugins/foundation/darrow-information-architecture/.codex-plugin/plugin.json b/plugins/foundation/darrow-information-architecture/.codex-plugin/plugin.json index 25c3cc20..3895362c 100644 --- a/plugins/foundation/darrow-information-architecture/.codex-plugin/plugin.json +++ b/plugins/foundation/darrow-information-architecture/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-information-architecture", - "version": "0.2.2", + "version": "0.2.3", "description": "Set up and doctor lean, routed repository information architecture", "author": { "name": "Björn Rochel", diff --git a/plugins/foundation/darrow-information-architecture/backend/tests/evals/eval_release_skill.py b/plugins/foundation/darrow-information-architecture/backend/tests/evals/eval_release_skill.py new file mode 100644 index 00000000..7225c32c --- /dev/null +++ b/plugins/foundation/darrow-information-architecture/backend/tests/evals/eval_release_skill.py @@ -0,0 +1,149 @@ +"""Independent release-skill metadata and reachability checks for the doctor eval.""" + +import argparse +import os +import re +import subprocess +from itertools import takewhile +from pathlib import Path + + +def skills(root: Path) -> list[Path]: + visited: set[Path] = set() + found = [] + for directory, children, files in os.walk(root, followlinks=True): + path = Path(directory) + if path.resolve() in visited: + children.clear() + continue + visited.add(path.resolve()) + if "SKILL.md" in files and "doctor-information-architecture" not in str(path): + found.append(path / "SKILL.md") + return sorted(found) + + +def release_skill(root: Path) -> Path: + matches = [ + path + for path in skills(root) + if "build-release" in path.read_text(encoding="utf-8").lower() + ] + assert matches, "release skill is missing" + return matches[0] + + +def scalar(value: str) -> bool: + value = re.sub(r"\s+#.*$", "", value).strip() + return bool(value) and not ( + value.startswith(("[", "{")) + or value in ('""', "''") + or re.fullmatch(r"null|~|true|false|yes|no|on|off|[0-9]+", value, re.I) + or block_header(value) + ) + + +def block_header(value: str) -> bool: + return re.fullmatch(r"[|>](?:[1-9][+-]?|[+-][1-9]?)?", value) is not None + + +def field_valid(lines: list[str], index: int, value: str) -> bool: + value = re.sub(r"\s+#.*$", "", value).strip() + if block_header(value): + content = takewhile( + lambda line: not line or line.startswith(" "), lines[index + 1 :] + ) + return bool("\n".join(content).strip()) + return scalar(value) + + +def metadata(path: Path) -> None: + lines = path.read_text(encoding="utf-8").splitlines() + assert lines and lines[0] == "---", "missing frontmatter" + end = lines.index("---", 1) + fields = {} + for index, line in enumerate(lines[1:end], 1): + match = re.match(r"^(name|description)\s*:(.*)$", line) + if match: + assert match[1] not in fields, "duplicate discovery metadata" + fields[match[1]] = field_valid(lines[:end], index, match[2]) + assert all(fields.get(key, False) for key in ("name", "description")) + + +def visible(root: Path, path: Path) -> bool: + result = subprocess.run( + ["git", "check-ignore", "-q", "--", str(path)], + cwd=root, + check=False, + ) + assert result.returncode in (0, 1), "git check-ignore failed" + return result.returncode == 1 + + +def native_release(root: Path, directory: str) -> bool: + pattern = ( + r"preflight.*package\.json.*build-release.*dist/checksums\.txt" + r".*tools/publish.*explicit confirmation" + ) + return any( + visible(root, path) + and re.search(pattern, path.read_text(encoding="utf-8"), re.I | re.S) + for path in skills(root / directory) + ) + + +def resident_intent(text: str, target: str) -> bool: + return any( + target in line + and re.search(r"release|publish", line, re.I) + and re.search(r"read|load|use", line, re.I) + for line in text.splitlines() + ) + + +def reachable(root: Path) -> None: + skill = release_skill(root) + target = str(skill.relative_to(root)) + result = subprocess.run( + [ + "uv", + "run", + "--quiet", + "--frozen", + "--no-dev", + "--project", + str(root / ".git/ia-backend"), + "ia-doctor", + "inspect", + "--runtime", + "codex", + ".", + ], + cwd=root, + text=True, + capture_output=True, + check=True, + ) + graph_route = f"{root / 'AGENTS.md'} -> {skill.resolve()}" in result.stdout + text = (root / "AGENTS.md").read_text(encoding="utf-8") + if graph_route and resident_intent(text, target): + return + assert native_release(root, ".agents/skills") and native_release( + root, ".claude/skills" + ), "release procedure needs an exact intent route or both native loaders" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("check", choices=("metadata", "reachable")) + arguments = parser.parse_args() + root = Path.cwd().resolve() + if arguments.check == "metadata": + metadata(release_skill(root)) + print("valid") + else: + reachable(root) + print("discoverable") + + +if __name__ == "__main__": + main() diff --git a/plugins/foundation/darrow-information-architecture/backend/tests/evals/test_eval_release_skill.py b/plugins/foundation/darrow-information-architecture/backend/tests/evals/test_eval_release_skill.py new file mode 100644 index 00000000..633d8115 --- /dev/null +++ b/plugins/foundation/darrow-information-architecture/backend/tests/evals/test_eval_release_skill.py @@ -0,0 +1,70 @@ +"""Release eval checks reject metadata and reachability counterexamples.""" + +import subprocess +from pathlib import Path + +import pytest +from eval_release_skill import metadata, native_release, resident_intent, skills + + +@pytest.mark.parametrize("value", ["Release a build", '"true"', "'quoted string'"]) +def test_string_metadata(tmp_path: Path, value: str) -> None: + path = tmp_path / "SKILL.md" + path.write_text(f"---\nname: release-build\ndescription: {value}\n---\n") + metadata(path) + + +@pytest.mark.parametrize( + "value", ["", "null", "~", "true", "123", "[]", "{}", '""', "''"] +) +def test_non_string_metadata_is_rejected(tmp_path: Path, value: str) -> None: + path = tmp_path / "SKILL.md" + path.write_text(f"---\nname: release-build\ndescription: {value}\n---\n") + with pytest.raises(AssertionError): + metadata(path) + + +def test_resident_route_must_bind_intent_and_exact_path() -> None: + target = "skills/release-build/SKILL.md" + assert resident_intent(f"Read {target} when publishing a release.", target) + assert not resident_intent(f"See {target}\nRead release instructions.", target) + assert not resident_intent("Read skills/other/SKILL.md for releases.", target) + + +@pytest.mark.parametrize("header", ["|", ">", "|-", ">+", "|2", ">-2"]) +def test_block_description_requires_nonempty_content( + tmp_path: Path, header: str +) -> None: + path = tmp_path / "SKILL.md" + template = f"---\nname: release-build\ndescription: {header}\n" + path.write_text(template + " \n---\n") + with pytest.raises(AssertionError): + metadata(path) + path.write_text(template + " Release the build.\n---\n") + metadata(path) + + +def test_native_loader_requires_visible_complete_procedure(tmp_path: Path) -> None: + subprocess.run(["git", "init", "-q", str(tmp_path)], check=True) + path = tmp_path / ".agents/skills/release/SKILL.md" + path.parent.mkdir(parents=True) + path.write_text( + "preflight package.json build-release dist/checksums.txt tools/publish" + ) + assert not native_release(tmp_path, ".agents/skills") + path.write_text(path.read_text() + " explicit confirmation") + assert native_release(tmp_path, ".agents/skills") + (tmp_path / ".gitignore").write_text(".agents/\n") + assert not native_release(tmp_path, ".agents/skills") + + +def test_skill_walk_handles_spaces_and_symlink_cycles(tmp_path: Path) -> None: + directory = tmp_path / "release skill" + directory.mkdir() + path = directory / "SKILL.md" + path.write_text("release") + try: + (directory / "cycle").symlink_to(tmp_path, target_is_directory=True) + except OSError: + pytest.skip("symlinks are unavailable on this host") + assert skills(tmp_path) == [path] diff --git a/plugins/foundation/darrow-information-architecture/skills/doctor-information-architecture/evals/move-procedure-to-skill.yaml b/plugins/foundation/darrow-information-architecture/skills/doctor-information-architecture/evals/move-procedure-to-skill.yaml index 85d221c7..979815f3 100644 --- a/plugins/foundation/darrow-information-architecture/skills/doctor-information-architecture/evals/move-procedure-to-skill.yaml +++ b/plugins/foundation/darrow-information-architecture/skills/doctor-information-architecture/evals/move-procedure-to-skill.yaml @@ -27,7 +27,11 @@ fixture: tools/build-release: "#!/bin/sh\nexit 0\n" tools/publish: "#!/bin/sh\nexit 0\n" dist/checksums.txt: "fixture\n" - setup: mkdir -p .git/ia-backend; cp -R "{{case_dir}}/../../../backend/src" .git/ia-backend/; cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/ia-backend/ + setup: | + mkdir -p .git/ia-backend .git/eval-checks/ia + cp -R "{{case_dir}}/../../../backend/src" .git/ia-backend/ + cp "{{case_dir}}/../../../backend/pyproject.toml" "{{case_dir}}/../../../backend/uv.lock" .git/ia-backend/ + cp "{{case_dir}}/../../../backend/tests/evals/eval_release_skill.py" .git/eval-checks/ia/ checks: - name: release procedure becomes an intent-triggered project skill run: >- @@ -52,10 +56,8 @@ checks: flags: i - name: release skill has valid discovery metadata run: >- - skill=$(find . -type f -name SKILL.md - ! -path '*doctor-information-architecture*' -exec grep -il 'build-release' {} + - | sed -n '1p'); - awk 'function valid(value, lower) {sub(/^[^:]*:[[:space:]]*/, "", value); sub(/[[:space:]]+#.*$/, "", value); sub(/^[[:space:]]+|[[:space:]]+$/, "", value); lower=tolower(value); return value != "" && value !~ /^(\[|\{)/ && lower !~ /^(null|~|true|false|yes|no|on|off|[0-9]+)$/ && value != "\"\"" && value != "\047\047"} NR==1 && $0=="---" {front=1; next} front && /^name[[:space:]]*:/ && valid($0) {name=1} front && /^description[[:space:]]*:/ && valid($0) {description=1} front && $0=="---" {exit} END {if (name && description) print "valid"}' "$skill" + uv run --quiet --frozen --no-dev --project .git/ia-backend + python .git/eval-checks/ia/eval_release_skill.py metadata expect_regex: "^valid$" - name: universal clean-worktree invariant stays resident run: cat AGENTS.md @@ -74,31 +76,8 @@ checks: flags: i - name: release procedure is reachable by an exact route or both native skill loaders run: >- - skill=$(find . -type f -name SKILL.md - ! -path '*doctor-information-architecture*' -exec grep -il 'build-release' {} + - | sed -n '1p'); - target=${skill#./}; test -n "$target"; - skill_abs=$(cd "$(dirname "$target")" && pwd -P)/$(basename "$target"); - root=$(pwd -P); routes=$(uv run --quiet --frozen --no-dev --project .git/ia-backend ia-doctor inspect --runtime codex .); - graph_route=false; - case "$routes" in *"$root/AGENTS.md -> $skill_abs"*) graph_route=true;; esac; - intent_route=false; - while IFS= read -r line; do - case "$line" in *"$target"*) - lower=$(printf '%s\n' "$line" | tr '[:upper:]' '[:lower:]'); - case "$lower" in *release*|*publish*) - case "$lower" in *read*|*load*|*use*) intent_route=true;; esac;; esac;; esac; - done < AGENTS.md; - if [ "$graph_route" = true ] && [ "$intent_route" = true ]; then echo discoverable; exit 0; fi; - codex=$(find -L .agents/skills -type f -name SKILL.md -exec grep -il 'build-release' {} + 2>/dev/null || :); - claude=$(find -L .claude/skills -type f -name SKILL.md -exec grep -il 'build-release' {} + 2>/dev/null || :); - valid_native() { for native in $1; do - if git check-ignore -q -- "$native"; then continue; fi; - flat=$(tr '\n' ' ' < "$native"); - if printf '%s\n' "$flat" | grep -Ei 'preflight.*package\.json.*build-release.*dist/checksums\.txt.*tools/publish.*explicit confirmation' >/dev/null; then return 0; fi; - done; return 1; }; - if valid_native "$codex" && valid_native "$claude"; then echo discoverable; - else printf '%s\n' "$routes" >&2; cat AGENTS.md >&2; exit 1; fi + uv run --quiet --frozen --no-dev --project .git/ia-backend + python .git/eval-checks/ia/eval_release_skill.py reachable expect_regex: "^discoverable$" - name: procedure move passes deterministic dual-runtime verification run: >- diff --git a/plugins/foundation/darrow-skill-authoring/.claude-plugin/plugin.json b/plugins/foundation/darrow-skill-authoring/.claude-plugin/plugin.json index fee18b32..88aceb21 100644 --- a/plugins/foundation/darrow-skill-authoring/.claude-plugin/plugin.json +++ b/plugins/foundation/darrow-skill-authoring/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "darrow-skill-authoring", "description": "Create and improve reliable cross-runtime agent skills", - "version": "0.2.4", + "version": "0.2.6", "license": "BUSL-1.1", "author": { "name": "Björn Rochel", diff --git a/plugins/foundation/darrow-skill-authoring/.codex-plugin/plugin.json b/plugins/foundation/darrow-skill-authoring/.codex-plugin/plugin.json index 73d076a7..c7607671 100644 --- a/plugins/foundation/darrow-skill-authoring/.codex-plugin/plugin.json +++ b/plugins/foundation/darrow-skill-authoring/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "darrow-skill-authoring", - "version": "0.2.4", + "version": "0.2.6", "description": "Create and improve reliable cross-runtime agent skills", "author": { "name": "Björn Rochel", diff --git a/plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/fresh_install.py b/plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/fresh_install.py new file mode 100644 index 00000000..05879a01 --- /dev/null +++ b/plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/fresh_install.py @@ -0,0 +1,107 @@ +"""Validate a fresh copied plugin using only locked runtime dependencies.""" + +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + + +def command(cwd: Path, *args: str) -> bytes: + return subprocess.run(args, cwd=cwd, check=True, capture_output=True).stdout + + +def installed_runtime(backend: Path, fixture: Path) -> None: + command(fixture, "uv", "sync", "--frozen", "--no-dev", "--project", str(backend)) + tree = command( + fixture, "uv", "tree", "--frozen", "--no-dev", "--project", str(backend) + ).decode() + assert not re.search(r"\b(coverage|hypothesis|mypy|pytest|ruff)\b", tree), tree + + +def validate(backend: Path, fixture: Path) -> None: + skill = backend.parent + inspection = command( + fixture, + "uv", + "run", + "--quiet", + "--frozen", + "--no-dev", + "--project", + str(backend), + "inspect-skill", + "inspect", + str(skill), + str(fixture / "plugin copy"), + ).decode() + assert "status\tvalid" in inspection.splitlines(), inspection + scripts = [ + "inspect-skill.test.sh", + "verify-shell-tests.test.sh", + "interpreter-routing.test.sh", + ] + paths = [str(backend / "tests" / "shell" / name) for name in scripts] + if os.name == "nt": + passing = fixture / "passing test.sh" + passing.write_bytes(b"#!/usr/bin/env bash\nexit 0\n") + paths = [str(passing)] + result = subprocess.run( + [ + "uv", + "run", + "--quiet", + "--frozen", + "--no-dev", + "--project", + str(backend), + "verify-shell-tests", + "--", + *paths, + ], + cwd=fixture, + capture_output=True, + text=True, + check=False, + ) + print(result.stdout, end="") + assert result.returncode in (0, 3), result + assert re.search(r"(?m)^format\s+darrow-shell-test-matrix-v1$", result.stdout) + assert not re.search(r"(?m)^test_result.*failed", result.stdout) + if os.name != "nt": + for name in scripts: + assert re.search( + r"test_result.*" + re.escape(name) + r"\s+passed$", + result.stdout, + re.MULTILINE, + ), result.stdout + + +def main() -> None: + plugin = Path(__file__).resolve().parents[4] + with tempfile.TemporaryDirectory(prefix="darrow author-agent-skill ") as temporary: + fixture = Path(temporary).resolve() + copy = fixture / "plugin copy" + shutil.copytree( + plugin, + copy, + ignore=shutil.ignore_patterns( + ".venv", + "__pycache__", + ".hypothesis", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".coverage*", + "coverage.json", + ), + ) + backend = copy / "skills" / "author-agent-skill" / "backend" + installed_runtime(backend, fixture) + validate(backend, fixture) + print("author-agent-skill fresh install passed") + + +if __name__ == "__main__": + main() diff --git a/plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/test_fresh_install.py b/plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/test_fresh_install.py new file mode 100644 index 00000000..85571872 --- /dev/null +++ b/plugins/foundation/darrow-skill-authoring/skills/author-agent-skill/backend/tests/test_fresh_install.py @@ -0,0 +1,48 @@ +"""Cross-platform output contracts for copied-plugin validation.""" + +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +import fresh_install + + +@pytest.mark.parametrize("ending", [b"\n", b"\r\n"]) +def test_valid_inspection_accepts_native_line_endings( + tmp_path: Path, ending: bytes +) -> None: + inspection = ending.join( + [b"format\tdarrow-skill-inspection-v1", b"status\tvalid", b""] + ) + matrix = "format\tdarrow-shell-test-matrix-v1\n" + "".join( + f"test_result\tbash-5\t{name}\tpassed\n" + for name in ( + "inspect-skill.test.sh", + "verify-shell-tests.test.sh", + "interpreter-routing.test.sh", + ) + ) + result = subprocess.CompletedProcess( + ["verify-shell-tests"], 3, stdout=matrix, stderr="" + ) + with ( + patch.object(fresh_install, "command", return_value=inspection), + patch.object(subprocess, "run", return_value=result), + ): + fresh_install.validate(tmp_path / "backend", tmp_path) + + +@pytest.mark.parametrize( + "inspection", + [b"status\tinvalid\r\n", b"status\tvalid-extra\r\n", b"name\tvalid\r\n"], +) +def test_invalid_inspection_stops_validation(tmp_path: Path, inspection: bytes) -> None: + with ( + patch.object(fresh_install, "command", return_value=inspection), + patch.object(subprocess, "run") as run, + pytest.raises(AssertionError), + ): + fresh_install.validate(tmp_path / "backend", tmp_path) + run.assert_not_called() diff --git a/plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.ps1 b/plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.ps1 deleted file mode 100644 index 7bbc277f..00000000 --- a/plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.ps1 +++ /dev/null @@ -1,43 +0,0 @@ -$ErrorActionPreference = "Stop" - -$pluginRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path -$fixture = Join-Path ([System.IO.Path]::GetTempPath()) ("darrow skill authoring " + [guid]::NewGuid()) -$copy = Join-Path $fixture "plugin" - -try { - New-Item -ItemType Directory -Path $fixture | Out-Null - Copy-Item -LiteralPath $pluginRoot -Destination $copy -Recurse - $backend = Join-Path $copy "skills/author-agent-skill/backend" - $skill = Join-Path $copy "skills/author-agent-skill" - Remove-Item -LiteralPath (Join-Path $backend ".venv") -Recurse -Force -ErrorAction SilentlyContinue - Remove-Item -LiteralPath (Join-Path $backend ".coverage") -Force -ErrorAction SilentlyContinue - Remove-Item -LiteralPath (Join-Path $backend "coverage.json") -Force -ErrorAction SilentlyContinue - - & uv sync --locked --no-dev --project $backend - if ($LASTEXITCODE -ne 0) { throw "runtime sync failed" } - $runtimeTree = (& uv tree --locked --no-dev --project $backend) -join "`n" - if ($LASTEXITCODE -ne 0) { throw "runtime tree failed" } - foreach ($tool in @("coverage", "hypothesis", "mypy", "pytest", "ruff")) { - if ($runtimeTree -match "(?m)(^|\s)$tool(\s|$)") { - throw "development dependency installed at runtime: $tool" - } - } - - $inspection = (& uv run --quiet --frozen --no-dev --project $backend inspect-skill inspect $skill $copy) -join "`n" - if ($LASTEXITCODE -ne 0) { throw "inspection failed" } - Write-Output $inspection - if ($inspection -notmatch "(?m)^status\s+valid$") { throw "inspection omitted valid status" } - - $testScript = Join-Path $fixture "passing test.sh" - "#!/usr/bin/env bash`nexit 0`n" | Set-Content -Path $testScript -NoNewline - $matrix = (& uv run --quiet --frozen --no-dev --project $backend verify-shell-tests -- $testScript) -join "`n" - $matrixStatus = $LASTEXITCODE - Write-Output $matrix - if ($matrixStatus -notin @(0, 3)) { throw "shell matrix failed with $matrixStatus" } - if ($matrix -notmatch "(?m)^format\s+darrow-shell-test-matrix-v1$") { throw "matrix omitted format" } -} -finally { - Remove-Item -LiteralPath $fixture -Recurse -Force -ErrorAction SilentlyContinue -} - -exit 0 diff --git a/plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh b/plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh deleted file mode 100755 index 470d73f0..00000000 --- a/plugins/foundation/darrow-skill-authoring/tests/fresh-install.test.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -set -eu - -plugin_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) -fixture=$(mktemp -d "${TMPDIR:-/tmp}/darrow-skill-authoring-install.XXXXXX") -trap 'rm -rf "$fixture"' EXIT HUP INT TERM - -mkdir -p "$fixture/plugin" -cp -R "$plugin_root/." "$fixture/plugin/" -backend="$fixture/plugin/skills/author-agent-skill/backend" -skill="$fixture/plugin/skills/author-agent-skill" -rm -rf "$backend/.venv" -rm -f "$backend/.coverage" "$backend/coverage.json" - -uv sync --locked --no-dev --project "$backend" -runtime_tree=$(uv tree --locked --no-dev --project "$backend") -for development_tool in coverage hypothesis mypy pytest ruff; do - if printf '%s\n' "$runtime_tree" | grep -E "(^|[[:space:]])${development_tool}([[:space:]]|$)" >/dev/null; then - echo "error: development dependency installed at runtime: $development_tool" >&2 - exit 1 - fi -done - -inspection=$(uv run --quiet --frozen --no-dev --project "$backend" \ - inspect-skill inspect "$skill" "$fixture/plugin") -printf '%s\n' "$inspection" | grep '^status[[:space:]]valid$' >/dev/null - -set +e -matrix=$(uv run --quiet --frozen --no-dev --project "$backend" \ - verify-shell-tests -- \ - "$backend/tests/shell/inspect-skill.test.sh" \ - "$backend/tests/shell/verify-shell-tests.test.sh" \ - "$backend/tests/shell/interpreter-routing.test.sh") -matrix_status=$? -set -e -case "$matrix_status" in - 0|3) ;; - *) printf '%s\n' "$matrix" >&2; exit "$matrix_status" ;; -esac -printf '%s\n' "$matrix" | grep '^format[[:space:]]darrow-shell-test-matrix-v1$' >/dev/null -if printf '%s\n' "$matrix" | grep '^test_result.*failed' >/dev/null; then - printf '%s\n' "$matrix" >&2 - exit 1 -fi -for test_name in inspect-skill.test.sh verify-shell-tests.test.sh interpreter-routing.test.sh; do - printf '%s\n' "$matrix" | grep "test_result.*${test_name}[[:space:]]passed$" >/dev/null -done