Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 50 additions & 0 deletions .github/workflows/python-quality.yml
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,47 @@ jobs:
- uses: astral-sh/setup-uv@v6
- run: uv run --quiet --frozen --no-dev --project plugins/foundation/darrow-information-architecture/backend python plugins/foundation/darrow-information-architecture/backend/tests/fresh_install.py

adaptive-delivery-windows:
name: Adaptive delivery Python ${{ matrix.python-version }} on windows-latest
needs: changes
if: contains(needs.changes.outputs.packages, 'plugins/orchestration/darrow-adaptive-delivery/backend')
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
runs-on: windows-latest
defaults:
run:
shell: bash
env:
UV_PYTHON: ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- run: scripts/check-python --package plugins/orchestration/darrow-adaptive-delivery/backend

adaptive-delivery-fresh-install:
name: Fresh install adaptive delivery on ${{ matrix.os }}
needs: changes
if: contains(needs.changes.outputs.packages, 'plugins/orchestration/darrow-adaptive-delivery/backend')
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.13"
- uses: astral-sh/setup-uv@v6
- run: uv run --quiet --frozen --no-dev --project plugins/orchestration/darrow-adaptive-delivery/backend python plugins/orchestration/darrow-adaptive-delivery/backend/tests/fresh_install.py

inventory:
name: Python inventory guard
runs-on: ubuntu-latest
Expand Down Expand Up @@ -510,6 +551,8 @@ jobs:
- decisions-fresh-install
- information-architecture-windows
- information-architecture-fresh-install
- adaptive-delivery-windows
- adaptive-delivery-fresh-install
- inventory
- skill-authoring-fresh-install
- observability-fresh-install
Expand Down Expand Up @@ -552,6 +595,9 @@ jobs:
IA_SELECTED: ${{ contains(needs.changes.outputs.packages, 'plugins/foundation/darrow-information-architecture/backend') }}
IA_WINDOWS_RESULT: ${{ needs.information-architecture-windows.result }}
IA_INSTALL_RESULT: ${{ needs.information-architecture-fresh-install.result }}
ADAPTIVE_SELECTED: ${{ contains(needs.changes.outputs.packages, 'plugins/orchestration/darrow-adaptive-delivery/backend') }}
ADAPTIVE_WINDOWS_RESULT: ${{ needs.adaptive-delivery-windows.result }}
ADAPTIVE_INSTALL_RESULT: ${{ needs.adaptive-delivery-fresh-install.result }}
run: |
scripts/verify-python-quality-results \
"$CHANGES_RESULT" "$INVENTORY_RESULT" \
Expand Down Expand Up @@ -583,3 +629,7 @@ jobs:
if [ "$REVIEW_SELECTED" = true ]; then expected=success; fi
test "$REVIEW_WINDOWS_RESULT" = "$expected"
test "$REVIEW_INSTALL_RESULT" = "$expected"
expected=skipped
if [ "$ADAPTIVE_SELECTED" = true ]; then expected=success; fi
test "$ADAPTIVE_WINDOWS_RESULT" = "$expected"
test "$ADAPTIVE_INSTALL_RESULT" = "$expected"
45 changes: 38 additions & 7 deletions docs/specs/adaptive-delivery.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,12 +87,43 @@ reason: explicit-orchestration-entrypoint-required

## Read-only preflight

The bundled helper is `<plugin-root>/bin/adaptive-delivery-preflight`. On Codex, the activated
file `<plugin-root>/skills/adaptive-delivery/SKILL.md` binds that same plugin root.
Relative traversal to `../../bin` starts at the directory containing
`SKILL.md`, not at the file path. Check the helper at that exact location;
The contained Python package is `<plugin-root>/backend`, invoked through
`uv run --quiet --frozen --no-dev --project <absolute-backend>` followed by
`adaptive-delivery-preflight` or `claude-agent-route`. On Codex, the activated
file `<plugin-root>/skills/adaptive-delivery/SKILL.md` binds that same plugin root;
`../../backend` starts at the directory containing `SKILL.md`. Claude binds
`${CLAUDE_PLUGIN_ROOT}/backend`. Check the package and lock at that exact location;
unavailability does not authorize searching for a different installation.

The package supports Python 3.10–3.13, UV, and Git on Linux, macOS, and native
Windows, with no runtime dependencies or Bash entrypoint adapters. Native paths
and argument-vector subprocesses preserve spaces, Unicode, and linked-worktree
identity. It retains the prepared-v2, route-v2, and Claude-agent-route-v1 records,
record ordering, policy precedence, refusal exit 2, and help exit 0. Commands emit
UTF-8 stdout/stderr with LF records regardless of the host's redirected console
encoding, including native Windows code pages. JSON uses
the standard-library parser; syntax diagnostics may name its line/column while
semantic validation, duplicate rejection, and fail-closed behavior remain required.
No helper starts a host process or implements orchestration authority checks;
authority, provider discovery by advertised intent, and native launch remain
owned by the skill and its host guides.

Editable installation must preserve Unicode source paths when Python 3.10/3.11
reads startup path files using a Windows legacy code page. Package bootstrap
records use ASCII-safe Python string literals; the runtime retains the original
paths. Subprocess test diagnostics retain undecodable startup error bytes.

Eval-only readiness, review, verification, and completion-proof mechanics live
in the same contained package, with inert fixture templates. They preserve
candidate fingerprints, closed finding sets, delayed combined assessment,
canonical artifact validation, and passive evidence beneath Git metadata.
Fresh copied-artifact tests exercise runtime-only installation and both host
routes on all three native platforms; Python quality gates independently
require 95% statement and branch coverage. Python regression tests preserve
route refusals, installed fixture contracts, closed review histories, and
mutation during independent and combined assessment on all three native
platforms, without a Bash test runner.

The helper binds discovery and every Git operation to the requested `--repo`
working tree, including a linked worktree or its subdirectory. Ambient Git
repository, worktree, common-directory, index, object-store, discovery, and
Expand Down Expand Up @@ -679,9 +710,9 @@ Nested host processes are not an adaptive-delivery fallback.

The bundled helper may prepare deterministic repository evidence and resolve a
configured route. It does not launch models, persist objectives, record owner
lifecycle, render completion, or supervise work. Plugin-shipped scripts remain
portable across Bash 5 and macOS Bash 3.2 and use only baseline Unix utilities
plus the host CLIs they explicitly wrap.
lifecycle, render completion, or supervise work. The contained Python package
and its regression tests support Python 3.10–3.13 on macOS, Linux, and native
Windows, using UV and Git without a Bash dependency.

Every required runtime file remains inside the plugin. Bundled helpers are
resolved without filesystem search: Claude uses its host-substituted plugin
Expand Down
10 changes: 9 additions & 1 deletion evals/runner/adapters/claude.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4345,7 +4345,15 @@ async function claudeHarnessResult(options: {
}

function hasAdaptiveDeliveryPreflight(pluginRoot: string): boolean {
return existsSync(join(pluginRoot, "bin", "adaptive-delivery-preflight"));
return existsSync(
join(
pluginRoot,
"backend",
"src",
"darrow_adaptive_delivery",
"preflight.py",
),
);
}

async function executeClaude(
Expand Down
88 changes: 87 additions & 1 deletion evals/runner/adapters/codex.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
Expand All @@ -15,6 +15,7 @@ import {
codexSpawnGuardRequested,
codexThreadId,
codexTokenUsage,
legacyAdaptiveDeliveryPreflight,
retainedCodexEvidence,
retainedCodexEvidenceForThread,
} from "./codex";
Expand All @@ -28,6 +29,91 @@ import {
} from "../codex-spawn-guard";

const REPO = "/tmp/eval";

test("Python plugin commands do not claim historical shell enforcement", async () => {
const root = await mkdtemp(join(tmpdir(), "darrow-python-plugin-"));
const plugin = join(root, "plugin");
const repo = join(root, "repo");
const run = (cwd: string, argv: string[]) => {
const result = Bun.spawnSync(argv, { cwd, stdout: "pipe", stderr: "pipe" });
expect(result.exitCode).toBe(0);
return result.stdout.toString();
};
try {
await cp(
join(
import.meta.dir,
"../../../plugins/orchestration/darrow-adaptive-delivery",
),
plugin,
{
recursive: true,
filter: (path) =>
!path
.split("/")
.some((part) =>
[
".venv",
"__pycache__",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".hypothesis",
].includes(part),
),
},
);
await mkdir(repo);
run(repo, ["git", "init", "-qb", "main"]);
run(repo, [
"git",
"-c",
"user.name=Fixture",
"-c",
"user.email=fixture@example.invalid",
"commit",
"--allow-empty",
"-qm",
"fixture",
]);
const argv = [
"uv",
"run",
"--quiet",
"--frozen",
"--no-dev",
"--project",
join(plugin, "backend"),
"adaptive-delivery-preflight",
];
expect(
run(repo, [...argv, "prepare", "--repo", repo, "--host", "codex"]),
).toContain("working_tree\tclean");
expect(
run(repo, [
...argv,
"route",
"--repo",
repo,
"--host",
"codex",
"--profile",
"routine",
]),
).toContain("selected_route\tcodex");
expect(await legacyAdaptiveDeliveryPreflight([plugin])).toBeUndefined();
await mkdir(join(plugin, "bin"));
await writeFile(
join(plugin, "bin/adaptive-delivery-preflight"),
"legacy fixture",
);
expect(await legacyAdaptiveDeliveryPreflight([plugin])).toEndWith(
"/bin/adaptive-delivery-preflight",
);
} finally {
await rm(root, { recursive: true, force: true });
}
}, 30000);
const COMPLETE_CONTRACT = [
"Role: You are the already-launched sole engineering owner. Perform this contract directly; do not invoke adaptive-delivery or seek another owner.",
"Outcome: Implement the requested fixture behavior.",
Expand Down
18 changes: 11 additions & 7 deletions evals/runner/adapters/codex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4046,6 +4046,16 @@ async function installedCodexPluginContext(
};
}

/** Historical step attestations require their actual shell helper. */
export async function legacyAdaptiveDeliveryPreflight(
pluginRoots: string[],
): Promise<string | undefined> {
const candidate = pluginRoots
.map((root) => join(root, "bin", "adaptive-delivery-preflight"))
.find((path) => existsSync(path));
return candidate ? await realpath(candidate) : undefined;
}

async function codexProcessContext(
repoDir: string,
prompt: string,
Expand All @@ -4061,14 +4071,8 @@ async function codexProcessContext(
env.TMPDIR = objectiveRoot;
const { installedSkillsRoots, installedPluginRoots } =
await installedCodexPluginContext(repoDir, env);
const adaptiveDeliveryPreflightCandidate = installedPluginRoots
.map((pluginRoot) => join(pluginRoot, "bin", "adaptive-delivery-preflight"))
.find((path) => existsSync(path));
const adaptiveDeliveryPreflightPath =
adaptiveDeliveryPreflightCandidate &&
existsSync(adaptiveDeliveryPreflightCandidate)
? await realpath(adaptiveDeliveryPreflightCandidate)
: undefined;
await legacyAdaptiveDeliveryPreflight(installedPluginRoots);
const spawnGuard =
adaptiveDeliveryPreflightPath && enableSpawnGuard
? await installCodexSpawnGuard(canonicalRepoDir, env, {
Expand Down
6 changes: 3 additions & 3 deletions evals/runner/fixture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ describe("eval fixture skill mounts", () => {
}
await mkdir(recipe, { recursive: true });
await mkdir(adaptive, { recursive: true });
await mkdir(join(orchestrationRoot, "bin"), { recursive: true });
await mkdir(join(orchestrationRoot, "backend"), { recursive: true });
await writeFile(
join(recipe, "SKILL.md"),
"---\nname: recipe\ndescription: Recipe\n---\n",
Expand Down Expand Up @@ -414,7 +414,7 @@ describe("eval fixture skill mounts", () => {
);
}
await writeFile(
join(orchestrationRoot, "bin", "adaptive-delivery-preflight"),
join(orchestrationRoot, "backend", "pyproject.toml"),
"fixture runner\n",
);

Expand Down Expand Up @@ -444,7 +444,7 @@ describe("eval fixture skill mounts", () => {
).toBe(true);
expect(existsSync(join(claudeAdditional, "skills", "recipe"))).toBe(false);
expect(
existsSync(join(claudeAdditional, "bin", "adaptive-delivery-preflight")),
existsSync(join(claudeAdditional, "backend", "pyproject.toml")),
).toBe(true);

const marketplace = join(fixture, ".git", "eval-marketplace");
Expand Down
9 changes: 5 additions & 4 deletions evals/runner/goal-readiness-fixture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ for (const [name, trace, passes] of [
} finally {
await destroyFixture(repo);
}
});
}, 30_000);
}

async function postLaunchFixture() {
Expand Down Expand Up @@ -88,7 +88,7 @@ test("caller inspection explicitly assigns the new behavior to the normalization
const [initial] = await runChecks(repo, [
{
name: "initial readiness",
run: 'bash .agents/bin/implementation-readiness-fixture "$PWD"',
run: 'uv run --quiet --frozen --no-dev --project .agents/backend adaptive-delivery-fixture readiness "$PWD"',
},
]);
expect(initial?.passed).toBe(true);
Expand All @@ -108,7 +108,8 @@ test("caller inspection explicitly assigns the new behavior to the normalization
for (const mode of ["before-edit", "omitted", "after-edit"] as const) {
test(`post-launch readiness gate: reassessment ${mode}`, async () => {
const { repo, evalCase } = await postLaunchFixture();
const assess = 'bash .agents/bin/implementation-readiness-fixture "$PWD"';
const assess =
'uv run --quiet --frozen --no-dev --project .agents/backend adaptive-delivery-fixture readiness "$PWD"';
try {
const [initial] = await runChecks(repo, [
{
Expand Down Expand Up @@ -155,5 +156,5 @@ for (const mode of ["before-edit", "omitted", "after-edit"] as const) {
} finally {
await destroyFixture(repo);
}
});
}, 30_000);
}
Loading
Loading