Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 62 additions & 0 deletions .github/workflows/python-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,59 @@ jobs:
enable-cache: true
- run: scripts/check-python --package plugins/capability/darrow-verification/skills/verify-change/backend

git-windows:
name: Git Python ${{ matrix.python-version }} on windows-latest
needs: changes
if: contains(needs.changes.outputs.packages, 'plugins/capability/darrow-git/backend')
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
runs-on: windows-latest
defaults:
run:
shell: bash
env:
UV_PYTHON: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- run: scripts/check-python --package plugins/capability/darrow-git/backend

git-fresh-install:
name: Fresh install git on ${{ matrix.os }}
needs: changes
if: contains(needs.changes.outputs.packages, 'plugins/capability/darrow-git/backend')
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- uses: astral-sh/setup-uv@v6
- if: runner.os != 'Windows'
run: bash plugins/capability/darrow-git/tests/fresh-install.test.sh
- name: Verify Git CLI regression scenarios
if: runner.os != 'Windows'
shell: bash
run: |
/bin/bash --version
for test in plugins/capability/darrow-git/backend/tests/shell/*.test.sh; do
/bin/bash "$test"
done
- if: runner.os == 'Windows'
shell: pwsh
run: "& plugins/capability/darrow-git/tests/fresh-install.test.ps1"

inventory:
name: Python inventory guard
runs-on: ubuntu-latest
Expand Down Expand Up @@ -278,6 +331,8 @@ jobs:
- skill-authoring-windows
- discovery-windows
- verification-windows
- git-windows
- git-fresh-install
- inventory
- skill-authoring-fresh-install
- observability-fresh-install
Expand Down Expand Up @@ -305,6 +360,9 @@ jobs:
DISCOVERY_INSTALL_RESULT: ${{ needs.discovery-fresh-install.result }}
VERIFICATION_INSTALL_RESULT: ${{ needs.verification-fresh-install.result }}
PERFORMANCE_RESULT: ${{ needs.performance.result }}
GIT_SELECTED: ${{ contains(needs.changes.outputs.packages, 'plugins/capability/darrow-git/backend') }}
GIT_WINDOWS_RESULT: ${{ needs.git-windows.result }}
GIT_INSTALL_RESULT: ${{ needs.git-fresh-install.result }}
run: |
scripts/verify-python-quality-results \
"$CHANGES_RESULT" "$INVENTORY_RESULT" \
Expand All @@ -316,3 +374,7 @@ jobs:
"$DISCOVERY_INSTALL_RESULT" \
"$VERIFICATION_CHANGED" "$VERIFICATION_WINDOWS_RESULT" \
"$VERIFICATION_INSTALL_RESULT"
expected=skipped
if [ "$GIT_SELECTED" = true ]; then expected=success; fi
test "$GIT_WINDOWS_RESULT" = "$expected"
test "$GIT_INSTALL_RESULT" = "$expected"
58 changes: 58 additions & 0 deletions docs/specs/git-workflow.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,43 @@ Plugin: `darrow-git`. Skills: `create-commit` (M0), `create-branch`,
GW-B6 applies only when the caller explicitly asks the `create-branch`
capability to allocate an additional linked worktree.

## Native discovery

- **GW-A1 — Route before preflight.** A matching Git workflow request selects
its skill before repository inspection or a terminal response. Missing input,
a clean tree, conflicts, and hook failures remain within the owning skill's
scope. General inspection and unrelated comments do not select a mutation
skill. Claude receives a plugin-local SessionStart routing reminder through
its native context hook. The hook supplies static context only: no provider
calls, prompt classification, workflow execution, state, or authority grants.
Native skill discovery and the selected skill retain contextual decisions;
installation never starts an operation or chains capabilities.

## Executable mechanics

- **GW-M1 — Contained portable implementation.** The five capabilities share
one Python package contained in `darrow-git/backend`, with frozen UV runtime
entrypoints and no runtime dependency on another plugin. Git and GitHub CLI
remain the provider boundaries. Frozen UV console commands are the runtime
interface on every platform; legacy shell-script paths are not retained.
Command arguments, stable records, refusal statuses, and mutation safeguards
remain intact. Internal refactoring preserves argument consumption order,
repeated-option behavior, diagnostic text, and stdout/stderr routing as well
as exit codes and repository effects. No workflow requires a Bash launcher.
- **GW-M2 — Literal process and filesystem boundaries.** Provider commands use
argument vectors, never interpolated shell programs. Paths and temporary
artifacts use native filesystem APIs. Hook remediation retains its diagnostic
and index-snapshot authorization boundary; the literal authorized diagnostic
command uses the native host command interpreter (POSIX sh or Windows cmd)
with its native command-string quoting. This compatibility exception never applies to Git/GitHub
provider arguments.
- **GW-M3 — Migration evidence.** The package meets the repository Python
quality standard, including separate 95% statement and branch coverage,
deterministic real-Git integration fixtures, mocked GitHub publication,
meaningful property tests, and fresh copied-plugin tests on Linux, macOS,
and native Windows. The propagation and presentation fixes tracked in #171
and #174 remain separately identifiable from this migration.

## Why

Agents left to improvise git usage produce inconsistent messages, stage
Expand Down Expand Up @@ -69,6 +106,11 @@ splitting one described change into multiple commits unless asked.

## create-branch

A request to create a branch from described ticket work belongs to
`create-branch` even when it supplies a canonical ticket token. That token does
not bind a complete branch name or turn creation into task-branch discovery;
derive the new name without asking the caller to supply one.

### Intent triggers

"create a branch", "branch for this", "start a branch", "new branch for X",
Expand Down Expand Up @@ -217,6 +259,13 @@ draft state, intended commit, remote branch commit, forge PR-head commit and
whether a push occurred. Success requires all three full commit identities to
agree after the operation.

The complete publication record is a machine-facing verification artifact.
The user-facing success report is concise: link the PR, state its base and
draft state, name the verified commit once, and mention excluded local work or
material notes. Expand repository, head and mutation details only when requested
or needed to explain uncertainty. Never summarize an incomplete observation as
verified publication.

### Invariants

- **GW-P1 — Conventional title.** The PR title follows the Conventional
Expand Down Expand Up @@ -266,6 +315,15 @@ agree after the operation.
contract as reuse. A URL or successful command exit alone is not completion.
If creation or push has occurred but observation fails, report the known
effects and uncertainty without creating another PR.
- **GW-P11 — Bounded propagation observation.** When the remote branch already
equals the intended commit but the forge reports another valid full commit,
allow at most five observations, one second apart. Pin the first observed PR
number and URL and recheck the local branch and commit before each observation.
A changed identity, shape, unavailable observation, malformed commit, or remote
movement refuses immediately. The observation loop never pushes or creates a
PR; timeout preserves the known mutation effects and reports incomplete
verification. Initial reuse preflight still requires remote/forge agreement
before deciding whether a push is necessary.

### Non-goals

Expand Down
54 changes: 53 additions & 1 deletion evals/runner/composition-fixture.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,63 @@
import { expect, test } from "bun:test";
import { readFile } from "node:fs/promises";
import { resolve } from "node:path";
import { dirname, resolve } from "node:path";
import { parse } from "yaml";
import { runChecks } from "./checks";
import { buildFixture, destroyFixture } from "./fixture";
import type { EvalCase } from "./types";

test("composition records only successful UV publication", async () => {
const path = resolve(
import.meta.dir,
"../../plugins/task-recipe/darrow-ticket-to-pr/skills/ticket-to-pr/evals/composition-existing-pr.yaml",
);
const entry = parse(await readFile(path, "utf8")) as EvalCase;
const repo = await buildFixture({
fixture: entry.fixture,
caseDir: dirname(path),
skillDir: resolve(
import.meta.dir,
"../../plugins/capability/darrow-git/skills/create-pr",
),
skillMounts: [],
sourceClaudePlugin: true,
});
const command =
"uv run --quiet --frozen --no-dev --project .git/eval-plugin/backend darrow-create-pr";
try {
const results = await runChecks(repo, [
{
name: "inspection is not publication",
run: `${command} inspect && test ! -e .git/builtin-publications`,
},
{
name: "stale remote refuses verification",
run: `${command} verify --expected-head "$(git rev-parse HEAD)"`,
exit_code: 4,
},
{
name: "failure is not evidence",
run: "test ! -e .git/builtin-publications",
},
{
name: "publish current commit",
run: `${command} publish-existing --expected-head "$(git rev-parse HEAD)"`,
},
{
name: "verify current commit",
run: `${command} verify --expected-head "$(git rev-parse HEAD)"`,
},
{
name: "exact successful observations",
run: 'test "$(wc -l < .git/builtin-publications | tr -d " ")" = 2 && test "$(sort -u .git/builtin-publications)" = "$(git rev-parse HEAD)" && test ! -e .git/forbidden-forge-effects',
},
]);
expect(results.filter((result) => !result.passed)).toEqual([]);
} finally {
await destroyFixture(repo);
}
}, 30000);

test("ticket-to-pr options oracle preserves a stricter caller repair limit", async () => {
const path = resolve(
import.meta.dir,
Expand Down
2 changes: 2 additions & 0 deletions evals/runner/environment.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ describe("isolated harness environment", () => {
expect(env.CLAUDE_CONFIG_DIR).toStartWith(home);
expect(env.CLAUDE_CODE_OAUTH_TOKEN).toBe("claude-test-token");
expect(env.DARROW_ADAPTIVE_DELIVERY_EXTERNAL_SANDBOX).toBe("1");
expect(env.UV_PROJECT_ENVIRONMENT).toBe(join(env.TMPDIR!, "uv-project"));
expect(env.UV_CACHE_DIR).toBe(join(env.TMPDIR!, "uv-cache"));
expect(env.UNRELATED_EVAL_SECRET).toBeUndefined();
expect(await readFile(join(codexHome, "auth.json"), "utf8")).toBe(
'{"token":"test"}',
Expand Down
4 changes: 4 additions & 0 deletions evals/runner/environment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,10 @@ export async function isolatedHarnessEnvironment(
);
env.HOME = stateRoot;
env.TMPDIR = tempRoot;
// Installed plugin sources live in the protected host config tree. Runtime
// environments belong in writable, per-trial scratch, outside that tree.
env.UV_PROJECT_ENVIRONMENT = join(tempRoot, "uv-project");
env.UV_CACHE_DIR = join(tempRoot, "uv-cache");
env.ZDOTDIR = shellRoot;
// The runner wraps the evaluated agent in sandboxedAgentCommand. Nested
// adaptive-delivery-preflight mechanics must reuse that boundary instead of attempting an
Expand Down
53 changes: 53 additions & 0 deletions evals/runner/fixture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,59 @@ describe("eval fixture skill mounts", () => {
cleanup.splice(cleanup.indexOf(fixture), 1);
});

test.each(["project", "claude", "codex"])(
"mounts a plugin-level Python backend for %s without generated state",
async (host) => {
const root = await mkdtemp(join(tmpdir(), "darrow-fixture-backend-"));
cleanup.push(root);
const plugin = join(root, "plugin");
const skill = join(plugin, "skills", "primary");
const backend = join(plugin, "backend");
await mkdir(skill, { recursive: true });
await mkdir(join(backend, ".venv"), { recursive: true });
await mkdir(join(backend, "evals"), { recursive: true });
await mkdir(join(plugin, ".claude-plugin"), { recursive: true });
await mkdir(join(plugin, ".codex-plugin"), { recursive: true });
await writeFile(
join(skill, "SKILL.md"),
"---\nname: primary\ndescription: Primary\n---\n",
);
await writeFile(join(backend, "pyproject.toml"), "[project]\n");
await writeFile(join(backend, ".venv", "generated"), "private\n");
await writeFile(join(backend, "evals", "secret.yaml"), "hidden\n");
await writeFile(
join(plugin, ".claude-plugin", "hooks.json"),
'{"hooks":{"SessionStart":[]}}\n',
);
for (const manifest of [".claude-plugin", ".codex-plugin"]) {
await writeFile(
join(plugin, manifest, "plugin.json"),
JSON.stringify({ name: "fixture-backend", version: "1.0.0" }),
);
}
const fixture = await buildFixture({
fixture: {},
skillDir: skill,
skillMounts: host === "project" ? [".agents/skills"] : [],
sourceClaudePlugin: host === "claude",
sourceCodexPlugin: host === "codex",
});
cleanup.push(fixture);
const destinations: Record<string, string> = {
project: join(fixture, ".agents", "backend"),
claude: join(fixture, ".git", "eval-plugin", "backend"),
codex: join(fixture, ".git", "eval-marketplace", "plugin", "backend"),
};
const destination = destinations[host]!;
expect(existsSync(join(destination, "pyproject.toml"))).toBe(true);
expect(existsSync(join(destination, ".venv"))).toBe(false);
expect(existsSync(join(destination, "evals"))).toBe(false);
expect(
existsSync(join(dirname(destination), ".claude-plugin", "hooks.json")),
).toBe(host !== "project");
},
);

test("optionally mounts every plugin skill without exposing colocated evals", async () => {
const root = await mkdtemp(join(tmpdir(), "darrow-fixture-plugin-"));
cleanup.push(root);
Expand Down
21 changes: 19 additions & 2 deletions evals/runner/fixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ async function copySkillWithoutEvals(
}

interface PluginMountPaths {
backend: string;
manifest: string;
codexManifest: string;
agents: string;
Expand All @@ -253,6 +254,11 @@ async function mountPluginMechanics(
mount: string,
paths: PluginMountPaths,
): Promise<void> {
if (existsSync(paths.backend))
await copySkillWithoutEvals(
paths.backend,
join(repoDir, mount, "..", "backend"),
);
if (existsSync(paths.bin))
await cp(paths.bin, join(repoDir, mount, "..", "bin"), { recursive: true });
if (existsSync(paths.config))
Expand All @@ -272,7 +278,10 @@ async function mountSourceClaudePlugin(
evalPlugin = join(repoDir, ".git", "eval-plugin"),
): Promise<void> {
await mkdir(join(evalPlugin, ".claude-plugin"), { recursive: true });
await cp(paths.manifest, join(evalPlugin, ".claude-plugin", "plugin.json"));
await copySkillWithoutEvals(
dirname(paths.manifest),
join(evalPlugin, ".claude-plugin"),
);
for (const mountedSkillDir of skillDirs) {
const name = mountedSkillDir.split("/").filter(Boolean).pop()!;
await copySkillWithoutEvals(
Expand All @@ -288,6 +297,8 @@ async function mountSourceClaudePlugin(
await cp(paths.config, join(evalPlugin, "config"), { recursive: true });
if (existsSync(paths.hooks))
await cp(paths.hooks, join(evalPlugin, "hooks"), { recursive: true });
if (existsSync(paths.backend))
await copySkillWithoutEvals(paths.backend, join(evalPlugin, "backend"));
}

async function mountSourceCodexPlugin(
Expand All @@ -297,7 +308,10 @@ async function mountSourceCodexPlugin(
): Promise<string> {
await mkdir(join(plugin, ".claude-plugin"), { recursive: true });
await mkdir(join(plugin, ".codex-plugin"), { recursive: true });
await cp(paths.manifest, join(plugin, ".claude-plugin", "plugin.json"));
await copySkillWithoutEvals(
dirname(paths.manifest),
join(plugin, ".claude-plugin"),
);
await cp(paths.codexManifest, join(plugin, ".codex-plugin", "plugin.json"));
for (const mountedSkillDir of skillDirs) {
const name = mountedSkillDir.split("/").filter(Boolean).pop()!;
Expand All @@ -311,6 +325,8 @@ async function mountSourceCodexPlugin(
await cp(paths.config, join(plugin, "config"), { recursive: true });
if (existsSync(paths.hooks))
await cp(paths.hooks, join(plugin, "hooks"), { recursive: true });
if (existsSync(paths.backend))
await copySkillWithoutEvals(paths.backend, join(plugin, "backend"));
const manifest = JSON.parse(await readFile(paths.codexManifest, "utf8")) as {
name?: unknown;
};
Expand Down Expand Up @@ -382,6 +398,7 @@ function pluginMountPaths(
): PluginMountPaths {
const pluginRoot = sourcePluginRoot ?? dirname(dirname(skillDir));
return {
backend: join(pluginRoot, "backend"),
bin: join(pluginRoot, "bin"),
config: join(pluginRoot, "config"),
agents: join(pluginRoot, "agents"),
Expand Down
Loading
Loading