diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7f1f869 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +# The gate every change passes before it merges: formatting, lints, and the +# library and binary tests, on the three platforms incode runs on. +# +# `/pr merge` refuses a pull request whose checks failed or are still +# running, so this workflow is what stands between a branch and `main`. +name: ci + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + fmt: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - run: cargo fmt --check + + test: + name: test (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + # The Linux desktop backends (xcap, enigo's x11rb, AT-SPI over zbus) + # link against these at build time — the same set the Linux live tests + # were built with. + - name: Linux build dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + pkg-config clang libclang-dev libwayland-dev libpipewire-0.3-dev \ + libxcb1-dev libxcb-randr0-dev libxcb-shm0-dev libdbus-1-dev \ + libegl-dev libgbm-dev + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + # The tests create git repositories and commit in them. + - run: | + git config --global user.email "ci@example.invalid" + git config --global user.name "ci" + - run: cargo clippy --all-targets + - run: cargo test --lib + - run: cargo test --bins diff --git "a/docs/Incord SI \342\200\224 Self-Improving Agent Harness (PRD).md" "b/docs/Incord SI \342\200\224 Self-Improving Agent Harness (PRD).md" index c3e25d1..f4a2ec8 100644 --- "a/docs/Incord SI \342\200\224 Self-Improving Agent Harness (PRD).md" +++ "b/docs/Incord SI \342\200\224 Self-Improving Agent Harness (PRD).md" @@ -105,7 +105,17 @@ Insi is a standalone Rust service with its own binary, `insi`, not code inside a | **MCP server** (read side) | Any agent, inside incode or not | Briefings, lesson lookup by error signature, topic load. Advisory only: MCP is called at the agent's discretion, so nothing that must hold is enforced here | | **GitHub webhooks** (a GitHub App) | GitHub | Report labelled issues and CI results; never a write path into Insi's state | -**How a worker runs.** Insi does not start Claude Code, Codex or agy itself. It starts **incode** (`incode -t … --output-format stream-json`, or `--acp`) in a sandbox worktree. incode drives the chosen agent, so every worker gets incode's guards, permission gate, checkpoints and proof by exit code without Insi re-implementing them. +**How a worker runs.** Insi drives a worker through a **worker adapter**: a command line, configured per repo, that takes the task's prompt and works in the sandbox worktree it is started in. The first adapter is **incode** (`incode -t … --output-format stream-json`): it drives Claude Code, Codex or agy and gives every worker incode's guards, permission gate, checkpoints and proof by exit code without Insi re-implementing them. Any other agent that runs headless in a folder is a second adapter, not a code change — so incode is the recommended worker, never a requirement. + +**Usable from any project.** Insi is one binary that works in a repository of any language, added in one of three ways: + +| Layer | What it is | Who uses it | +| --- | --- | --- | +| **The `insi` binary** | CLI and MCP server in one program, installed once per machine | Every project, any language | +| **`insi init`** | Run inside a repo: writes `.insi/config.toml`, an `evals/` folder, a CI workflow that runs the checks on agent PRs, and the MCP/hook registration for the agents in use | How a project adopts Insi | +| **Packages per ecosystem** | `insi` as a Rust library crate, and thin npm and pip packages that fetch the binary — the pattern Ruff, Biome and esbuild use | Teams that want it pinned in their own lockfile | + +Nothing is published until the Phase 1 gate is met: private first. **Outputs.** The engine produces two things only: audit records in Incord Memory, and pull requests a human reviews. @@ -172,19 +182,20 @@ Insi is a standalone Rust service with its own binary, `insi`, not code inside a ### F9. Per-repo config -A repo already carries incode's `.incode/` folder. Insi adds one file there, and **reuses** what incode already has rather than repeating it: - -- protected paths: incode's `.incode/guards.json` (plus the built-in protected set); -- the check command: incode's `verify_command`; -- `.incord/` is not used: it belongs to Incord Memory. +Insi keeps its own folder, `.insi/`, so it works in a repo that has never seen incode; `insi init` writes it. Where incode is present, its settings are read rather than repeated: protected paths from `.incode/guards.json`, the check command from incode's `verify_command`. `.incord/` belongs to Incord Memory and is not used. -`.incode/si.toml`: +`.insi/config.toml`: ```toml topic = "incord/memory-service" -eval_tasks = "evals/tasks" # public task definitions -private_checks = "git@github.com:org/insi-private-checks.git" +test_cmd = "cargo test --workspace" # the public check +protected = [".github/workflows/**", "evals/**"] +eval_tasks = "evals/tasks" # public task definitions +private_checks = "" trusted_issue_label = "agent-ok" + +[worker] # the worker adapter +command = ["incode", "-t", "{prompt}", "--output-format", "stream-json"] ``` ## Security and governance @@ -215,17 +226,16 @@ One Cargo workspace, `insi`, with its own binary, next to `incord-memory-service | Crate | Responsibility | Starting point | | --- | --- | --- | -| `si-core` | Shared types: Topic, Lesson, Task, EvalResult, Candidate, Trust | `serde`, `thiserror` | -| `si-config` | Parse `.incode/si.toml` and `harness-config/` | `toml`, `serde` | -| `si-memory` | Client for Incord Memory: topic load, lesson write, audit events | HTTP (`reqwest`) | -| `si-connectors` | `Connector` trait + GitHub, code, CI logs, email | `gh` CLI first | -| `si-sandbox` | Worktree create/cleanup, sandboxed run, resource limits | `git` CLI; containers later | -| `si-eval` | Task runner, budgets, public/private checks, multi-seed stats | `tokio` | -| `si-loop` | Proposer, bandit, compare, accept/reject | `tokio`, seeded `rand` | -| `si-guard` | Path policy, secret/PII scan, dependency-change detector | `globset`, `regex` | -| `si-pr` | Branch, commit, PR body with evidence | `gh` CLI first; GitHub App before unattended runs | -| `si-server` | GitHub App webhooks, MCP server, scheduler | `axum`, an MCP server crate | -| `si-cli` | `insi eval`, `insi run`, `insi audit`, `insi propose` | `clap` | +Built first (Phase 1), at `D:\Projects\insi`: + +| Crate | Responsibility | Starting point | +| --- | --- | --- | +| `insi-core` | Shared types (Task, Budget, EvalResult, Lesson, Trust), `.insi/config.toml`, the error-signature normalizer | `serde`, `toml` | +| `insi-eval` | Mining tasks from history, the task runner in a throwaway worktree, budgets, public/private checks, multi-seed results | `git` CLI, std processes | +| `insi-memory` | Client for Incord Memory: lesson write, briefing load, audit events | HTTP | +| `insi` (binary) | `insi mine`, `insi eval`, `insi report`, later `insi init`, `insi run`, `insi propose` | std argument parsing first | + +Later phases add `insi-loop` (proposer, bandit, accept/reject), `insi-guard` (path policy, secret/PII scan, dependency-change detector — a dependency of the PR step, so no PR is opened without it), `insi-pr` (`gh` first; GitHub App before unattended runs), `insi-connectors` and `insi-server` (webhooks, MCP server, scheduler). Dependency names are starting candidates; pick per crate after a spike, vetted like any other dependency. @@ -305,7 +315,7 @@ Each phase is useful on its own: phase 1 alone shows whether topic memory reduce ## Status: what exists today, and what is left -As of Oct 1, 2026. "incode" paths are in `D:\Projects\incord-code`. +As of Oct 2, 2026. "incode" paths are in `D:\Projects\incord-code`. ### Already built (Insi reuses these) @@ -324,34 +334,59 @@ As of Oct 1, 2026. "incode" paths are in `D:\Projects\incord-code`. | Memory client | Recorder lookup and handoff, `remember`, recall | incode `src/recorder.rs`, `src/knowledge.rs` | | Running under the Watchman app | incode as the lead of a Watchman orchestration reaches its orchestrator (the control credentials pass incode's environment filter) | incode `src/tools/env_filter.rs` | +### Built in the Insi repo (Phase 1, Oct 2, 2026) + +`D:\Projects\insi`, a Cargo workspace, local commits only. Crates `insi-core`, `insi-eval`, `insi-memory` and the `insi` binary; 36 tests, clippy and fmt clean. + +| Piece | What exists | +| --- | --- | +| `insi mine` | Every fix commit that changed source and a test file of its own becomes a task: parent = start, message = prompt, test files = hidden check, commit = reference, commit date kept. Reads the source repository only | +| Private checks | Hidden test files stored apart (`--private`), outside every checkout; copied into the worktree only after the worker stops, over anything it wrote at those paths | +| `insi validate` | Controls per task: the check must fail at the base and pass at the reference; only such tasks are "fair" (`--fair-only`) | +| `insi eval` | Each attempt in its own worktree of Insi's own clone; setup, worker (wall-clock budget), hidden check; records pass, time, tokens (from incode's `stream-json`), failure signature, tags, artifacts (worker diff, outputs). `--seeds`, `--jobs N`, `--half older\|newer` | +| Signature normalizer | Terminal colour stripped; paths, hashes, numbers normalised; only lines that say what failed | +| Lesson writer (`insi learn`) | A failed attempt on a fair task + its verified fix → one lesson (issue, root cause, use, avoid), refused when a field is empty, oversized or holds code; `verified_by: check` | +| Briefing | Lessons of the task's repository learned strictly before the task (compared as instants), never from the task itself, newest first, within 4,000 characters; attempts tagged `lessons=on\|off` and record which lessons they got | +| `insi report` | Pass rate, repeat failures (same signature again), timeouts, errors, tokens and time per condition | +| Watchman benchmark | 38 tasks mined, all validated: 35 fair | +| Interface brief (`insi brief`) | The prompt names the module paths and exported names the hidden tests import that the base lacks — never a test or expected value — and says nobody will answer questions. Insi `dd99838` | + +**Pilot findings (Oct 2).** The first pilot (incode worker, routed to `gemini-3.1-pro-high`, 1800 s) passed 0 of 4. Every failure was a name the prompt never gave: a module the fix created (`editorDoc`, `ompInterjections`) or a function it exported (`isHiddenEditorHost`, `decodePtyChunk`); one worker also asked a question and waited out its budget. 20 of the 38 tasks have such a gap. A benchmark whose baseline is zero cannot show lessons helping, so the interface brief above was added before measuring. Three attempts at once also ran the machine out of memory: attempts run one at a time. + +Decision: benchmark lessons are kept in a JSONL file under the work folder, not in Incord Memory. `lesson_write` files an item under the project of the folder it names, which would put benchmark lessons in Watchman's memory, and its retrieval is semantic; an experiment needs a set it can reset and reproduce. Writing verified lessons to Incord Memory is the production path, behind the same store trait. + ### Left to build -| # | Piece | Phase | -| --- | --- | --- | -| 1 | Benchmark format and `insi eval`: tasks, public and private checks, seeds, results | 1 | -| 2 | Private-check repository, outside every checkout | 1 | -| 3 | First 20–30 tasks, from bug-fix commits in incord-code's history that came with a test | 1 | -| 4 | Dollar and wall-time budgets (incode has tokens only) | 1 | -| 5 | Error-signature normalizer | 1 | -| 6 | Lesson writer: failed-then-verified task → lesson with evidence, in Incord Memory | 1 | -| 7 | Topic resolution rules (repo, path, command) and topic fields on lessons — confirm what Incord Memory already stores | 1 | -| 8 | Briefing builder and just-in-time injection through the pre-tool hook | 1 | -| 9 | `insi` workspace and binary; CLI | 1 | -| 10 | MCP server (read side) | 2 | -| 11 | Check inside a worktree before any merge or PR (also needed by `/run` lanes-as-PRs) | 2 | -| 12 | PR body with evidence; secret/PII scan before opening | 2 | -| 13 | Trusted-label issue intake | 2 | -| 14 | GitHub App with short-lived, PR-only tokens | 2 | -| 15 | `harness-config/` folder, read by incode | 3 | -| 16 | Proposer, accept-if-better, UCB1 bandit, audit events | 3 | -| 17 | Connectors: CI logs, error tracking, docs, email | 4 | -| 18 | Containers for issue work | 4 | +| # | Piece | Phase | Status | +| --- | --- | --- | --- | +| 1 | Benchmark format and `insi eval`: tasks, public and private checks, seeds, results | 1 | Built. Public check (Insi `b2597b7`): a task's optional `public_check` (`insi mine` fills it from `test_cmd`, or `--public-check`) runs on the worker's tree before the hidden tests arrive; worker attempts record `public_passed`, controls never run it; `insi report` gives the reward-hacking rate per condition (public passed, hidden failed). The Watchman tasks carry none yet, so run 2 does not measure it | +| 2 | Private-check repository, outside every checkout | 1 | Built as a folder; a repository of its own is not needed yet | +| 3 | First 20–30 tasks, from bug-fix commits in the Watchman app's history that changed a separate test file | 1 | Built: 38 mined, all validated — 35 fair (fail at base, pass at reference); 3 pass at base and are left out | +| 4 | Dollar and wall-time budgets (incode has tokens only) | 1 | Built: wall time; dollars — `[prices]` per model in config, `cost_usd` per attempt, `insi eval --max-dollars` (Insi `7587c56`) | +| 5 | Error-signature normalizer | 1 | Built | +| 6 | Lesson writer: failed-then-verified task → lesson with evidence, in Incord Memory | 1 | Built: file store, and written to Incord Memory with `insi learn --to-memory`. Reading briefings back FROM Incord Memory is blocked on the engine: its tools offer semantic search only, no listing by tag or kind, and a briefing needs the complete, reproducible set. Briefings consolidate lessons (Insi `0bf85d8`): one item per action, beliefs (learned ≥2 times) first, a later lesson whose `avoid` is an earlier `use` retires it, all as of the task's date | +| 7 | Topic resolution rules (repo, path, command) and topic fields on lessons — confirm what Incord Memory already stores | 1 | Built (`42ecbb7`): hierarchical topics (`shop` covers `shop/api`); `[[topic_rules]]` by path globs and command words, most specific wins, no model. Lessons filed on the topic of the files their fix changed; briefing = topic + parents; `insi hook` brings narrower lessons on the first edit in their part, and matches failures on the command's topic. Lessons reach Incord Memory tagged `topic:` (insi-memory `incord.rs`). Promotion built (Insi `0bf85d8`): an action learned in ≥2 sibling topics is briefed on their shared parent, marked `[across ]`. Not feasible as things stand: the hook inside `insi eval` workers — the hook is a Claude Code hook, and the measured worker runs incode on the antigravity backend, which calls no such hook | +| 8 | Briefing builder and just-in-time injection through the pre-tool hook | 1 | Built: pre-task briefing, and `insi hook` (Insi `bfb64c3`) — lessons naming a file before it is edited, lessons matching a failure after a command fails; verified live in Claude Code 2.1.288 | +| 9 | `insi` workspace and binary; CLI | 1 | Built | +| — | **The Phase 1 measurement**: lessons off vs on, newer half, 3 seeds | 1 | Run 2 done (Oct 9, ONE seed, incode on gemini-3.1-pro-high, 1800 s budget): newer half lessons off **5/16 passed (31%)**, on **6/17 (35%)**. Paired over the 16 tasks judged both ways: 5 passed both, 1 fail→pass (`879ae4b8`), 0 pass→fail, 10 failed both — one discordant task, so no claim that lessons help is supported yet. Timeouts 6 → 3; median attempt 1310 s → 549 s, but the off round ran under heavy load from other builds, so time is confounded. Tokens are not comparable: an attempt cut off at its budget reports none (incode prints usage only at the end). Gaps found: every lessons-on attempt got the SAME 3 newest lessons (11 lessons, all on one topic, ~1.2 KB each in a 4 KB briefing), so this measured three generic lessons, not relevant ones; 4 of 15 lessons were lost to incode bugs since fixed (long-prompt cut `e83a227`, fragment joining `b21c55d`) and Insi's argv limit (`06c2382`). Next: relevance-ranked briefing, the 4 lessons recovered, 3 seeds | +| 10 | MCP server (read side) | 2 | Built: `insi mcp` (`insi_briefing`, `insi_lessons_for_error`), Insi `5b68733` | +| 11 | Check inside a worktree before any merge or PR (also needed by `/run` lanes-as-PRs) | 2 | Done in incode: lane check before merge (`INCODE_LANE_CHECK`, 6f1ecdf) and lanes opened as draft PRs (`INCODE_LANES_AS=pr`), settled on the next run (`docs/LANES_AS_PRS.md`). Not yet run against GitHub live | +| 12 | PR body with evidence; secret/PII scan before opening | 2 | incode `/pr` credential scan built (`289090a`); `insi issue` PR body with evidence and the same scan built (Insi `5b68733`). PII scan built in both: card numbers (Luhn), US SSNs, real-domain emails (Insi `61ac225`, incode `/pr`) | +| 13 | Trusted-label issue intake | 2 | Built: `insi issue --number N` — open + labelled only, draft PR after `test_cmd` passes (Insi `5b68733`) | +| 14 | GitHub App with short-lived, PR-only tokens | 2 | Not started; `gh`'s login is used meanwhile | +| 15 | `harness-config/` folder, read by incode | 3 | Built: Insi runs a worker with `INSI_HARNESS=` (Insi `ec27d8d`; `insi improve` passes each candidate's own snapshot). incode `src/harness.rs` reads `instructions.md` (system context, after CLAUDE.md/AGENTS.md), `roles.json` (roles/tiers over admin.json per key, at read time only so `/roles` never saves it; `INCODE_ROLE_*` still wins; `--admin` shows `<- harness`), `agents/*.md` (between project and operator). A harness file that does not read refuses the run. Verified live on the debug binary. Left: retry caps (`MAX_ATTEMPTS`, `MAX_REPLANS`) are consts used by `/run`'s ledger, not by `-t`; Insi's own `prompt.md` is the task-prompt surface | +| 16 | Proposer, accept-if-better, UCB1 bandit, audit events | 3 | Built (Insi `c645085`): crate `insi-loop` (harness snapshots by content id, one-file proposals with path/size checks, paired accept rule with margin + held-out not-worse, UCB1, JSONL audit) and `insi improve --harness ` (select on the older fair half, hold out the newer; scores cached per harness; proposer never shown check output). Tested on the synthetic repo. Surface today: the harness's `prompt.md`, put before every task. Since: incode reads the same folder (row 15); `insi improve --to-memory` files each change kept and the run's end in Incord Memory as `decision` items (Insi `c07d30d`; proposals and scores stay in `audit.jsonl`); `insi improve --pr` opens the best harness as a draft PR — only the files the loop changed, on a fresh worktree of the default branch, after the credential/PII scan — through `gh`'s login until the GitHub App exists (row 14) (Insi `2b841f9`). Neither tested against the live engine or GitHub yet: a stand-in engine and a local bare origin. "Fork from the best every N steps" is inherent: under strict accept-if-better the incumbent is always the best | +| 17 | Connectors: CI logs, error tracking, docs, email | 4 | Built (Insi `32e2a82`): crate `insi-connectors`, one `Source` trait, every item with source + `Trust` (trusted/internal/untrusted, in insi-core) + topic; only the asked topic's line of descent returned, bounded; untrusted text fenced as data and unable to close its fence; a failing source named, not fatal. Sources: `docs` (README + docs/, trusted), `ci` (failed Actions runs via `gh`, internal, topic from files the log names), `[[connectors]]` commands printing JSON lines (error tracking, mail exporters; untrusted by default; Insi holds no credentials). Served by `insi context` and MCP `insi_context`; kept out of eval briefings. Verified live on docs; CI live returned none (incord-code has no Actions runs). Left: a first real error-tracking/mail exporter configured by the owner | +| 18 | Containers for issue work | 4 | Built (Insi `75b1f87`): `[sandbox] command = [...]` with `{tree}`; `insi issue` runs setup, the worker AND the check inside it (the check runs worker-written code); commit, scan and push stay on the host; the PR says where it ran. Tested with a stand-in wrapper. Left: a live Docker run — the Docker CLI is installed but Docker Desktop's daemon is not running on this machine | +| 19 | `insi init` for any repo: `.insi/config.toml`, `evals/`, CI workflow, MCP/hook registration | 2 | `insi init` built (`ded3d68`): detects Node/Rust/Python/Go, writes the config, ignores the work folder. CI workflows for Insi and incord-code committed. `insi mcp` registered in `.mcp.json` (`5ae02c3`) and `insi hook` in `.claude/settings.json` (`bfb64c3`); an existing file is never rewritten. CI workflow for agent PRs built (`a73f599`): `.github/workflows/insi.yml` runs setup and the repo's own `test_cmd` on PRs from `insi/` branches, read-only token not persisted. No `evals/` folder, by decision: a task's definition holds its reference commit (the answer), so tasks stay in `private_checks` outside the checkout. Done | +| 20 | Packages: `insi` on crates.io, npm and pip wrappers — after the Phase 1 gate | 2 | Ready, not published (Insi `c0308a3`): crates publish-ready (`cargo publish --workspace --dry-run` passes all six); `release.yml` drafts a GitHub release with 5 platform archives + `checksums.json` on a `v*` tag matching every manifest; npm and pip wrappers with no dependencies download the archive and check it against the digest pinned in the package (tested offline, including a wrong digest and a path-escaping archive). `RELEASING.md` has the steps. The name `insi` was free on crates.io, npm and PyPI on Oct 2. Left: the owner's accounts and the decision to publish, after the Phase 1 gate; a LICENSE file naming the copyright holder | ## Decisions (were open questions) Proposed answers; each stands until revised. -- **First benchmark repo:** incord-code itself. Its bug-fix commits that added a test are ready-made tasks: the parent commit is the starting state, and the new test is the private check. +- **First benchmark repo:** the Watchman app (`D:\Projects\watchman`), read only — Insi works in its own clone and never touches that checkout. Measured on Oct 2: 722 commits, 427 changing source together with a separate test file, 36 of them fixes. A fix commit's parent is the starting state and its test file the private check. incord-code was the first idea and was measured too small for it: 79 commits, 4 fix-like, and tests inline in the source, so a fix's test cannot be held back without the fix. +- **Published or private:** private until the Phase 1 gate is met. - **Acceptance margin and seeds:** start at 3 seeds; set the margin from the variance the first baseline measures. - **Sandbox runtime:** worktrees plus incode's OS sandbox first; Docker for issue work in phase 4. - **Inside incode or separate:** a separate `insi` binary that drives workers through incode, so it serves any agent and any repo. diff --git a/docs/LANES_AS_PRS.md b/docs/LANES_AS_PRS.md new file mode 100644 index 0000000..2f76e9e --- /dev/null +++ b/docs/LANES_AS_PRS.md @@ -0,0 +1,74 @@ +# Lanes as pull requests + +Status: built. Builds on the lane check (each lane's check runs in its own +checkout before anything leaves it). + +## What changes for the operator + +Today a `/run` with `INCODE_LANES=N` merges each lane's branch into the +checked-out branch on this machine. With `INCODE_LANES_AS=pr`, a lane whose +own check passed is pushed and opened as a **draft** pull request onto the +branch the run is on, and nothing is merged locally. A person reviews and +merges it on GitHub. The run goes on with the other tasks. + +## Why it is more than swapping `merge_lane` for `/pr` + +The ledger's law is that a task reaches `Done` only through a passing check on +the tree the work is in. A lane opened as a pull request is neither done (its +work is not in the tree) nor running (no agent is on it) nor failed. + +## What was built + +1. **It waits `Blocked`, with the URL recorded** (`Task.pull_request`, + `Ledger::await_review`). The proposal was an eighth state, `InReview`; the + ledger holds itself to seven (`src/ledger.rs`), and `Blocked` already + means "a decision is outstanding with a person", which a review is. So + nothing picks it up, its dependants stay `Pending`, a run that drains + around it stops with `NothingRunnable` naming it, and the existing + progress and dashboard views show it as waiting on a person — with the + URL in its note. +2. **Opening** (`parallel::open_for_review`, `Merged::Opened { url }`): after + the lane check passes, the lane is sealed and opened through `/pr`'s own + path (`pull_request::open_onto`) with its refusals — nothing new, a + credential in the diff. `landed()` is false, and no report is cached, so + the runner never checks a tree without the change. A task with no check is + merged here instead: nothing could finish it after a review. +3. **The base is the branch the run is on**, not the forge's default: a lane + cut from a feature branch would otherwise carry that branch's commits. + Checked once at the start (`parallel::review_base`: a branch, not detached; + an `origin`; `gh` reachable; the branch on `origin`) and before each batch + (`parallel::nothing_unpushed`: HEAD has nothing `origin` lacks). If either + fails, the run says why and merges locally, as before. +4. **Coming back** (`reviews::settle`, at the start of every run, whatever + `INCODE_LANES_AS` says): for each task waiting on a pull request, the + forge is asked its state (`Forge::review`, `gh pr view --json + state,baseRefName`). + - Merged: if this checkout is on that base, it is fast-forwarded from + `origin` (only a fast-forward), the task's check runs, and only a pass + completes it. A failing check is a failed attempt. A checkout on another + branch, or one that cannot fast-forward, is left alone and the task + waits with a note saying to pull. + - Closed unmerged: a failed attempt; back to the queue with its branch + remembered (or set aside, if out of attempts). + - Open, or unreadable: left waiting. +5. **Off by default.** Without `INCODE_LANES_AS=pr` nothing is opened. + +## Tests + +- `parallel::a_lane_opened_for_review_is_pushed_and_not_merged` +- `parallel::a_review_base_must_be_on_the_remote_with_nothing_unpushed` +- `parallel::lanes_are_merged_here_unless_pull_requests_are_asked_for` +- `reviews::a_merged_pull_request_is_pulled_checked_and_done` +- `reviews::a_merged_pull_request_whose_check_fails_is_a_failed_attempt` +- `reviews::a_closed_pull_request_sends_the_task_back_with_its_branch` +- `reviews::an_open_pull_request_leaves_the_task_waiting` +- `reviews::a_checkout_on_another_branch_is_not_merged_into` +- `reviews::the_dependants_of_a_task_in_review_are_not_started` +- `pull_request::a_pull_requests_state_and_base_are_read` +- `pull_request::a_branch_is_opened_onto_the_base_it_is_given` + +## Not yet exercised + +A live run against GitHub. Every test stands in for the forge; the `gh` +calls (`pr view --json state,baseRefName`, `pr create --draft`) are the ones +`/pr` already makes, plus the state query. diff --git a/src/admin.rs b/src/admin.rs index 107d421..7243ac6 100644 --- a/src/admin.rs +++ b/src/admin.rs @@ -56,6 +56,9 @@ pub enum Source { Flag, /// An environment variable set for this shell. Environment, + /// The harness folder this run was given (`INSI_HARNESS`), over the + /// admin file for this run only. + Harness, /// The admin file — the standing decision. AdminFile, /// The built-in fallback, which is a last resort and not a recommendation. @@ -68,6 +71,7 @@ impl Source { match self { Self::Flag => "flag", Self::Environment => "env", + Self::Harness => "harness", Self::AdminFile => "admin file", Self::BuiltIn => "built-in", } @@ -758,23 +762,30 @@ impl AdminConfig { #[must_use] pub fn resolve_roles(&self) -> Resolved { let from_env = crate::roles::RoleTable::from_env(); + let harness = crate::harness::routing(); + let (roles, tiers) = crate::harness::overlay(&self.roles, &self.tiers, harness.as_ref()); let mut table = crate::roles::RoleTable::default(); let mut used_file = false; + let mut used_harness = false; let mut used_env = false; for role in crate::roles::KNOWN_ROLES { let from_env_role = from_env.candidates_for(role); if from_env_role.is_empty() { - if let Some(models) = self.roles.get(role).filter(|m| !m.is_empty()) { + if let Some(models) = roles.get(role).filter(|m| !m.is_empty()) { // `@top` becomes the models that tier holds. A role naming a // tier nobody defined is reported by `unknown_tiers`, not // silently emptied. - let (expanded, _) = crate::roles::expand_tiers(models, &self.tiers); + let (expanded, _) = crate::roles::expand_tiers(models, &tiers); table = table.with_candidates(role, expanded); - used_file = true; + if harness.as_ref().is_some_and(|h| h.roles.contains_key(role)) { + used_harness = true; + } else { + used_file = true; + } } } else { - let (expanded, _) = crate::roles::expand_tiers(from_env_role, &self.tiers); + let (expanded, _) = crate::roles::expand_tiers(from_env_role, &tiers); table = table.with_candidates(role, expanded); used_env = true; } @@ -783,14 +794,25 @@ impl AdminConfig { // The table's source is the strongest layer that contributed to it. Where // the layers are mixed, `role_source` is the per-role truth and this is // only a headline. - let source = match (used_env, used_file) { - (true, _) => Source::Environment, - (false, true) => Source::AdminFile, - (false, false) => Source::BuiltIn, + let source = match (used_env, used_harness, used_file) { + (true, ..) => Source::Environment, + (false, true, _) => Source::Harness, + (false, false, true) => Source::AdminFile, + (false, false, false) => Source::BuiltIn, }; Resolved::new(table, source) } + /// `roles` and `tiers` as this run uses them: the admin file's, with the + /// harness's over them key by key when the run was given one. Never + /// saved: the harness is this run's, the file is the operator's. + #[must_use] + pub fn effective_routing( + &self, + ) -> (BTreeMap>, BTreeMap>) { + crate::harness::overlay(&self.roles, &self.tiers, crate::harness::routing().as_ref()) + } + /// Which layer decided one role's candidates. /// /// The headline source on `resolve_roles` cannot answer this when an operator @@ -803,6 +825,10 @@ impl AdminConfig { .is_empty() { Source::Environment + } else if crate::harness::routing() + .is_some_and(|h| h.roles.get(role).is_some_and(|m| !m.is_empty())) + { + Source::Harness } else if self.roles.get(role).is_some_and(|m| !m.is_empty()) { Source::AdminFile } else { diff --git a/src/agent/agy.rs b/src/agent/agy.rs index a8f86f5..607e57c 100644 --- a/src/agent/agy.rs +++ b/src/agent/agy.rs @@ -77,11 +77,17 @@ struct InitBody { #[derive(Debug, Deserialize)] struct StepUpdate { + /// Which step of the turn this update belongs to: every fragment of one + /// reply carries the same index. + #[serde(default)] + step_index: u64, #[serde(default)] step_type: String, - /// The assistant's text for this step. Named `text_delta` on the wire, but - /// measured to arrive as the WHOLE text of a finished step rather than a - /// character fragment — a `DONE` step carried the complete reply. + /// A FRAGMENT of the assistant's text for this step, to be appended to + /// the ones before it. Measured on a long reply: dozens of `ACTIVE` + /// updates of one step, each a few words, the last one `DONE` and itself + /// only a fragment. A short reply fits in one update, which is how it was + /// once mistaken for the whole text of a finished step ([`StepText`]). #[serde(default)] text_delta: String, #[serde(default)] @@ -209,14 +215,14 @@ impl AgyBackend { } /// The prompt actually sent: the harness's context, then the task, budgeted for OS limits. - fn full_prompt(&self, prompt: &str) -> String { + fn full_prompt(&self, prompt: &str) -> Result { let (budgeted_prompt, budgeted_context) = - crate::agent::budget_for_spawn(prompt, &self.system_context); - if budgeted_context.is_empty() { + crate::agent::budget_for_spawn(prompt, &self.system_context, &self.workspace)?; + Ok(if budgeted_context.is_empty() { budgeted_prompt } else { format!("{budgeted_context}\n\n{budgeted_prompt}") - } + }) } /// Override the binary, e.g. an absolute path from agent discovery. @@ -399,7 +405,7 @@ impl AgentBackend for AgyBackend { prompt: &str, on_event: &mut dyn FnMut(&AgentEvent), ) -> Result { - let sent = self.full_prompt(prompt); + let sent = self.full_prompt(prompt)?; let mut command = self.command(&sent); crate::agent::process_tree::prepare(&mut command); let mut child = command @@ -418,6 +424,7 @@ impl AgentBackend for AgyBackend { let mut outcome: Option = None; let mut usage = crate::runtime::usage::TokenUsage::default(); + let mut reply = StepText::default(); for line in BufReader::new(stdout).lines() { if let Some(token) = &self.cancel { @@ -458,12 +465,7 @@ impl AgentBackend for AgyBackend { } } Wire::StepUpdate { step_update } => { - if step_update.step_type == "agent_response" - && !step_update.text_delta.trim().is_empty() - { - on_event(&AgentEvent::TextDelta(step_update.text_delta.clone())); - on_event(&AgentEvent::Text(step_update.text_delta.clone())); - } + reply.feed(&step_update, on_event); if let Some(u) = step_update.usage { usage = to_usage(&u); } @@ -489,6 +491,7 @@ impl AgentBackend for AgyBackend { Wire::Other => {} } } + reply.finish(on_event); on_event(&AgentEvent::Usage { input_tokens: usage.input_tokens, @@ -516,6 +519,41 @@ impl AgentBackend for AgyBackend { } } +/// The assistant's words for a turn, rebuilt from step updates: each +/// fragment is shown as it arrives (`TextDelta`), and each step's whole text +/// is said once (`Text`) when the next step begins or the stream ends. +/// +/// Why once: a caller keeps every `Text` as a separate thing said and joins +/// them with newlines (src/task_run.rs `show`, which builds `-t`'s result). +/// Measured: sending each fragment as a `Text` put a newline every few words — +/// `"issue\n": "… anim\nation …"` — and a JSON answer no longer parsed. +#[derive(Default)] +struct StepText { + step: Option, + text: String, +} + +impl StepText { + fn feed(&mut self, update: &StepUpdate, on_event: &mut dyn FnMut(&AgentEvent)) { + if update.step_type != "agent_response" || update.text_delta.is_empty() { + return; + } + if self.step != Some(update.step_index) { + self.finish(on_event); + self.step = Some(update.step_index); + } + on_event(&AgentEvent::TextDelta(update.text_delta.clone())); + self.text.push_str(&update.text_delta); + } + + fn finish(&mut self, on_event: &mut dyn FnMut(&AgentEvent)) { + let text = std::mem::take(&mut self.text); + if !text.trim().is_empty() { + on_event(&AgentEvent::Text(text)); + } + } +} + /// This CLI's usage figures, mapped to the harness's. /// /// `cache_creation_input_tokens` stays ZERO because this CLI does not report @@ -555,6 +593,58 @@ mod tests { serde_json::from_str(line).expect("a wire line") } + /// **A REPLY STREAMED IN FRAGMENTS IS SAID ONCE, WHOLE.** Lines shaped as + /// a real long reply (measured on agy, Insi's lesson writer): fragments of + /// one step, `ACTIVE` then `DONE`, each a few words and split inside words + /// and inside a JSON key. The step is one `Text`, the fragments joined byte + /// for byte, so its JSON parses; every fragment is still shown live as a + /// `TextDelta`; a later step is a `Text` of its own. + #[test] + fn a_reply_streamed_in_fragments_is_said_once_whole() { + let line = |index: u64, state: &str, kind: &str, text: &str| { + format!( + r#"{{"event":"step_update","step_update":{{"step_index":{index},"state":"{state}","step_type":"{kind}","text_delta":{}}}}}"#, + serde_json::to_string(text).expect("json") + ) + }; + let stream = [ + line(0, "DONE", "user_input", ""), + line(2, "ACTIVE", "agent_response", "{\n \"issue"), + line(2, "ACTIVE", "agent_response", "\": \"The agent added anim"), + line( + 2, + "ACTIVE", + "agent_response", + "ation math\",\n \"use\": \"Wire it", + ), + line(2, "DONE", "agent_response", " end to end.\"\n}"), + line(4, "DONE", "agent_response", "Done."), + ]; + let mut said = Vec::new(); + let mut deltas = 0; + { + let mut on_event = |event: &AgentEvent| match event { + AgentEvent::Text(text) => said.push(text.clone()), + AgentEvent::TextDelta(_) => deltas += 1, + _ => {} + }; + let mut reply = StepText::default(); + for wire in stream.iter().map(|l| parse(l)) { + if let Wire::StepUpdate { step_update } = wire { + reply.feed(&step_update, &mut on_event); + } + } + reply.finish(&mut on_event); + } + assert_eq!(said.len(), 2, "{said:?}"); + let answer: serde_json::Value = + serde_json::from_str(&said[0]).expect("the whole reply parses"); + assert_eq!(answer["issue"], "The agent added animation math"); + assert_eq!(answer["use"], "Wire it end to end."); + assert_eq!(said[1], "Done."); + assert_eq!(deltas, 5, "every fragment shown as it came"); + } + /// The envelope is keyed on `event`, not `type`. /// /// Copied from a real v1.1.23 run. If this ever parses as `Other` the CLI @@ -779,11 +869,14 @@ mod tests { #[test] fn the_harness_context_is_carried_in_the_prompt_when_no_flag_exists() { let bare = AgyBackend::new(PathBuf::from(".")); - assert_eq!(bare.full_prompt("do the work"), "do the work"); + assert_eq!( + bare.full_prompt("do the work").expect("sent"), + "do the work" + ); let briefed = AgyBackend::new(PathBuf::from(".")) .with_system_context(vec!["the guards in force: tests/ is protected".to_string()]); - let sent = briefed.full_prompt("do the work"); + let sent = briefed.full_prompt("do the work").expect("sent"); assert!( sent.contains("tests/ is protected"), "the harness context never reached the agent: {sent}" diff --git a/src/agent/claude_code.rs b/src/agent/claude_code.rs index 3a968a5..ea02ad3 100644 --- a/src/agent/claude_code.rs +++ b/src/agent/claude_code.rs @@ -1346,7 +1346,7 @@ impl ClaudeCodeBackend { on_event: &mut dyn FnMut(&AgentEvent), ) -> Result { let (budgeted_prompt, budgeted_context) = - crate::agent::budget_for_spawn(prompt, &self.system_context); + crate::agent::budget_for_spawn(prompt, &self.system_context, &self.workspace)?; // THE TOOLS ARE incode's. The CLI's own are switched off below and the // model is handed incode's toolbox through the bridge, which this diff --git a/src/agent/codex.rs b/src/agent/codex.rs index d229906..75e01bb 100644 --- a/src/agent/codex.rs +++ b/src/agent/codex.rs @@ -242,14 +242,14 @@ impl CodexBackend { } /// The prompt actually sent: the harness's context, then the task, budgeted for OS limits. - fn full_prompt(&self, prompt: &str) -> String { + fn full_prompt(&self, prompt: &str) -> Result { let (budgeted_prompt, budgeted_context) = - crate::agent::budget_for_spawn(prompt, &self.system_context); - if budgeted_context.is_empty() { + crate::agent::budget_for_spawn(prompt, &self.system_context, &self.workspace)?; + Ok(if budgeted_context.is_empty() { budgeted_prompt } else { format!("{budgeted_context}\n\n{budgeted_prompt}") - } + }) } fn command(&self, prompt: &str) -> Command { @@ -436,7 +436,7 @@ impl AgentBackend for CodexBackend { prompt: &str, on_event: &mut dyn FnMut(&AgentEvent), ) -> Result { - let sent = self.full_prompt(prompt); + let sent = self.full_prompt(prompt)?; // Held to the end of the turn, then removed. let pictures = ImageFiles::write(&std::mem::take(&mut self.images)).map_err(AgentError::new)?; @@ -816,11 +816,14 @@ mod tests { #[test] fn harness_context_is_carried_in_prompt() { let bare = CodexBackend::new(PathBuf::from(".")); - assert_eq!(bare.full_prompt("fix the bug"), "fix the bug"); + assert_eq!( + bare.full_prompt("fix the bug").expect("sent"), + "fix the bug" + ); let briefed = CodexBackend::new(PathBuf::from(".")) .with_system_context(vec!["Guards: no writes outside src/".to_string()]); - let sent = briefed.full_prompt("fix the bug"); + let sent = briefed.full_prompt("fix the bug").expect("sent"); assert!(sent.contains("Guards: no writes outside src/")); assert!(sent.ends_with("fix the bug")); } diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 87ff824..57b79b8 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -346,45 +346,39 @@ pub const MAX_PROMPT_BUDGET: usize = 12_000; /// Minimum context budget preserved when budgeting payload. pub const MIN_CONTEXT_BUDGET: usize = 4_000; -/// How much of the START of an over-budget prompt is kept; the rest of the -/// budget goes to its END, where the instruction is. -const PROMPT_HEAD: usize = 2_000; - /// Compacts / budgets prompt and system context so external agent CLI processes /// spawned via `Command::new` stay safely within OS command-line limits. /// -/// When the total payload exceeds [`MAX_CMDLINE_PAYLOAD_LEN`], high-priority -/// blocks (guards, standing rules, intake, asking, roles) are preserved, while -/// large informational blocks (discovered skills catalog, auditor findings, -/// recorder handoff) are truncated to fit the remaining budget. -#[must_use] -pub fn budget_for_spawn(prompt: &str, system_context: &[String]) -> (String, String) { +/// When the total payload exceeds [`MAX_CMDLINE_PAYLOAD_LEN`], a prompt over +/// [`MAX_PROMPT_BUDGET`] is written whole to a file in `workspace` and the +/// agent is told to read it ([`spill`]); nothing of it is dropped. Then +/// high-priority context blocks (guards, standing rules, intake, asking, +/// roles) are preserved, while large informational blocks (discovered skills +/// catalog, auditor findings, recorder handoff) are truncated to fit the +/// remaining budget. An error only when the spill file cannot be written: +/// the turn is not started on a prompt the agent would never see whole. +/// +/// Called by the CLI backends that pass the prompt on a command line: +/// `agy.rs` and `codex.rs` (`full_prompt`) and `claude_code.rs` (`run_once`). +pub fn budget_for_spawn( + prompt: &str, + system_context: &[String], + workspace: &std::path::Path, +) -> Result<(String, String), AgentError> { let unbudgeted_context = system_context.join("\n\n"); let total_len = prompt.len() + unbudgeted_context.len(); if total_len <= MAX_CMDLINE_PAYLOAD_LEN { - return (prompt.to_string(), unbudgeted_context); + return Ok((prompt.to_string(), unbudgeted_context)); } - // Budget the prompt first - let prompt_budget = if prompt.len() > MAX_PROMPT_BUDGET { - MAX_PROMPT_BUDGET - } else { - prompt.len() - }; - // THE END OF A PROMPT IS THE INSTRUCTION. Every prompt this harness - // assembles puts its context first and what was asked last, so cutting - // from the end cut exactly the part the agent was meant to act on — - // measured: an agent handed only the head of an 18,084-character prompt - // answered by asking what the task was. The middle goes instead. - let budgeted_prompt = if prompt.len() > prompt_budget { - const MARKER: &str = "\n\n[...prompt truncated for OS command line limit...]\n\n"; - let head = crate::text::head_bytes(prompt, PROMPT_HEAD); - let room = prompt_budget.saturating_sub(head.len() + MARKER.len()); - let mut from = prompt.len().saturating_sub(room); - while !prompt.is_char_boundary(from) { - from += 1; - } - format!("{head}{MARKER}{}", &prompt[from..]) + // NO PART OF THE PROMPT IS DROPPED. It used to keep the first 2,000 and + // the last ~9,900 bytes and cut the middle, on the theory that the + // instruction is always short and last. Measured on Insi's lesson writer: + // its instruction comes FIRST and its prompt runs to 20 KB, so the model + // got the preamble's head and the tail of a diff, and answered by asking + // what it was meant to do — four lessons of fifteen lost. + let budgeted_prompt = if prompt.len() > MAX_PROMPT_BUDGET { + spill(prompt, workspace)? } else { prompt.to_string() }; @@ -394,7 +388,7 @@ pub fn budget_for_spawn(prompt: &str, system_context: &[String]) -> (String, Str .max(MIN_CONTEXT_BUDGET); if unbudgeted_context.len() <= context_budget { - return (budgeted_prompt, unbudgeted_context); + return Ok((budgeted_prompt, unbudgeted_context)); } // Partition blocks by priority: @@ -464,7 +458,40 @@ pub fn budget_for_spawn(prompt: &str, system_context: &[String]) -> (String, Str } } - (budgeted_prompt, result_blocks.join("\n\n")) + Ok((budgeted_prompt, result_blocks.join("\n\n"))) +} + +/// Write the whole of a prompt too long for a command line to the context +/// folder in `workspace` — the folder incode's memory hooks spill their +/// blocks to and prune (`crate::hooks::memory::CONTEXT`) — and return what the +/// agent is told instead: where the file is, and to read all of it first. +/// The name is the prompt's hash, so the same prompt is one file. +fn spill(prompt: &str, workspace: &std::path::Path) -> Result { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + prompt.hash(&mut hasher); + let name = format!("task-{:016x}.md", hasher.finish()); + let dir = workspace.join(crate::hooks::memory::CONTEXT); + let path = dir.join(&name); + std::fs::create_dir_all(&dir) + .and_then(|()| std::fs::write(&path, prompt)) + .map_err(|e| { + AgentError::new(format!( + "the task is {} bytes, more than a command line can carry, and could not \ + be written to {} for the agent to read: {e}; the turn was not started", + prompt.len(), + path.display() + )) + })?; + Ok(format!( + "This task is {} bytes, more than a command line can carry, so the whole of it \ + is in the file `{}/{name}` in your working folder ({}). Read that file in \ + full, from its first line to its last, before doing anything else: it is the \ + task, and what it asks is what you do.", + prompt.len(), + crate::hooks::memory::CONTEXT, + path.display() + )) } #[cfg(test)] @@ -669,9 +696,14 @@ mod handover_tests { "no-write tests".to_string(), "standing rules: follow style guide".to_string(), ]; - let (p, c) = budget_for_spawn(prompt, &context); + let dir = tempfile::tempdir().expect("dir"); + let (p, c) = budget_for_spawn(prompt, &context, dir.path()).expect("budgeted"); assert_eq!(p, prompt); assert_eq!(c, context.join("\n\n")); + assert!( + !dir.path().join(crate::hooks::memory::CONTEXT).exists(), + "a prompt that fits writes no file" + ); } #[test] @@ -683,45 +715,74 @@ mod handover_tests { "The owner's standing rules:\n1. do not delete".to_string(), large_skills, ]; - let (p, c) = budget_for_spawn(prompt, &context); + let dir = tempfile::tempdir().expect("dir"); + let (p, c) = budget_for_spawn(prompt, &context, dir.path()).expect("budgeted"); assert_eq!(p, prompt); assert!(p.len() + c.len() <= MAX_CMDLINE_PAYLOAD_LEN); assert!(c.contains("protected")); assert!(c.contains("standing rules")); } + /// The file an over-budget prompt was written to, read from what the + /// agent was told. + fn spilled(told: &str, workspace: &std::path::Path) -> String { + let at = told.find("task-").expect("names a task file"); + let name = &told[at..at + "task-".len() + 16 + ".md".len()]; + std::fs::read_to_string(workspace.join(crate::hooks::memory::CONTEXT).join(name)) + .expect("the task file exists") + } + + /// **AN OVER-BUDGET PROMPT REACHES THE AGENT WHOLE.** Both shapes a + /// harness sends: the instruction LAST (a task after its context) and + /// the instruction FIRST, then a long body (Insi's lesson writer, whose + /// instruction the old middle cut dropped). Each is written to a file in + /// full, the agent is told where it is and to read all of it, and the + /// command line stays within the OS limit. #[test] - fn budget_for_spawn_bounds_oversized_prompt() { - // Over this OS's limit, whichever it is: a fixed 40,000 was over the - // Windows limit and under the Unix one, so it tested nothing there. - let giant_prompt = "A".repeat(MAX_CMDLINE_PAYLOAD_LEN + 10_000); - let context = vec!["safe".to_string()]; - let (p, c) = budget_for_spawn(&giant_prompt, &context); - assert!(p.len() + c.len() <= MAX_CMDLINE_PAYLOAD_LEN); - assert!(p.contains("prompt truncated for OS command line limit")); - assert!(c.contains("safe")); + fn budget_for_spawn_spills_an_oversized_prompt_whole() { + let body = "context é ".repeat(MAX_CMDLINE_PAYLOAD_LEN / 10); + let last = format!("g\n\n{body}\n\nNow do this:\n\ncreate a.txt with hi"); + let first = + format!("Write one lesson as JSON with an issue field.\n\n{body}\nVERIFIED_FIX>>>"); + let context = vec!["safe".to_string(), "x".repeat(20_000)]; + for prompt in [last, first] { + let dir = tempfile::tempdir().expect("dir"); + let (p, c) = budget_for_spawn(&prompt, &context, dir.path()).expect("budgeted"); + assert!( + p.len() + c.len() <= MAX_CMDLINE_PAYLOAD_LEN, + "{}", + p.len() + c.len() + ); + assert!( + p.len() < 1_000, + "the agent is told where, not sent a cut: {p}" + ); + assert!(p.contains("Read that file in full"), "{p}"); + assert_eq!( + spilled(&p, dir.path()), + prompt, + "nothing of the prompt is dropped" + ); + assert!(c.contains("safe")); + } } - /// **AN OVER-BUDGET PROMPT KEEPS ITS INSTRUCTION.** Context comes first - /// and the instruction last, so the cut is taken from the middle: an - /// agent once got only the head and asked what it was meant to do. + /// **A PROMPT THAT CANNOT BE WRITTEN DOWN IS NOT SENT AT ALL.** The + /// workspace is a file, so the context folder cannot be made: the turn + /// stops with the reason instead of starting on a prompt the agent would + /// never see whole. #[test] - fn budget_for_spawn_keeps_the_instruction_at_the_end() { - let prompt = format!( - "g\n\n{}\n\nNow do this:\n\ncreate a.txt with hi", - "context é ".repeat(MAX_CMDLINE_PAYLOAD_LEN / 10) - ); - let context = vec!["x".repeat(20_000)]; - let (p, c) = budget_for_spawn(&prompt, &context); - assert!(p.len() + c.len() <= MAX_CMDLINE_PAYLOAD_LEN); - assert!(p.len() <= MAX_PROMPT_BUDGET, "{}", p.len()); + fn budget_for_spawn_refuses_when_the_task_file_cannot_be_written() { + let dir = tempfile::tempdir().expect("dir"); + let not_a_folder = dir.path().join("plain-file"); + std::fs::write(¬_a_folder, "x").expect("file"); + let prompt = "y".repeat(MAX_CMDLINE_PAYLOAD_LEN + 1); + let why = budget_for_spawn(&prompt, &[], ¬_a_folder).expect_err("refused"); + let why = why.to_string(); assert!( - p.ends_with("Now do this:\n\ncreate a.txt with hi"), - "{}", - &p[p.len() - 80..] + why.contains("could not be written") && why.contains("not started"), + "{why}" ); - assert!(p.starts_with("g")); - assert!(p.contains("prompt truncated for OS command line limit")); } } diff --git a/src/cli/run_ledger.rs b/src/cli/run_ledger.rs index 381ede8..ee245d7 100644 --- a/src/cli/run_ledger.rs +++ b/src/cli/run_ledger.rs @@ -340,6 +340,7 @@ pub(crate) fn run(ctx: &RunContext<'_>, cancel: &CancelToken, watch: Watch<'_>) cancel: cancel.clone(), opened: std::sync::atomic::AtomicUsize::new(0), engine, + review: crate::parallel::lanes_as_pull_requests(), }; let outcome = work( &mut ledger, @@ -575,9 +576,22 @@ pub(crate) struct CliLaneWorkers { /// The memory engine each lane's session is filed with, when there is /// one: every lane is a session of its own. pub engine: Option, + /// Lanes are opened as pull requests on GitHub rather than merged here + /// (`INCODE_LANES_AS=pr`), read once when the run starts. + pub review: bool, } impl crate::runner::LaneWorkers for CliLaneWorkers { + /// GitHub through `gh`, always: a task waiting on a pull request is + /// settled whether or not this run opens new ones. + fn forge(&self) -> Option<&dyn crate::tools::pull_request::Forge> { + Some(&crate::tools::pull_request::GitHubCli) + } + + fn open_for_review(&self) -> bool { + self.review + } + fn spawn_for(&self, workspace: &Path) -> Option> { let lane = self .opened @@ -867,6 +881,7 @@ mod tests { cancel: CancelToken::new(), opened: std::sync::atomic::AtomicUsize::new(0), engine: None, + review: false, }; let task = crate::ledger::Task::new("t-1", "r-1", "count the files").with_tier("implement"); // The lane's own agent, as `spawn_for` makes it, on a stand-in diff --git a/src/cli/slash_commands.rs b/src/cli/slash_commands.rs index 48c7df8..cb83cf0 100644 --- a/src/cli/slash_commands.rs +++ b/src/cli/slash_commands.rs @@ -1292,6 +1292,7 @@ fn roles_listing(admin: &crate::admin::AdminConfig) -> String { var } } + Source::Harness => format!("the harness in {}", crate::harness::HARNESS_ENV), Source::AdminFile => "your choice".to_string(), Source::Flag | Source::BuiltIn => "not set".to_string(), }; diff --git a/src/harness.rs b/src/harness.rs new file mode 100644 index 0000000..a1df243 --- /dev/null +++ b/src/harness.rs @@ -0,0 +1,227 @@ +//! The harness: a folder of instructions, routing and sub-agents that Insi's +//! improvement loop changes, named for one run by [`HARNESS_ENV`]. +//! +//! Insi scores a candidate harness by running incode with this variable set +//! to the candidate's folder. What incode takes from it: +//! +//! | File | What it changes | +//! | --- | --- | +//! | `instructions.md` | Standing instructions in the system context, after the project's own `CLAUDE.md`/`AGENTS.md` | +//! | `roles.json` | `{"roles": {...}, "tiers": {...}}`: per role and per tier, over `admin.json`; an `INCODE_ROLE_*` variable still wins | +//! | `agents/*.md` | Sub-agent definitions, between the project's and the operator's | +//! +//! Applied where they are READ, never folded into the loaded admin config: a +//! `/roles` change saves `admin.json`, and a harness must not end up written +//! into the operator's own settings. With the variable unset, nothing here +//! does anything. +//! +//! A harness file that is present but unreadable is an error, not an empty +//! file: a candidate whose routing cannot be read must fail its attempt, not +//! be scored as if it were the default ([`check`]). + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +/// The variable naming the harness folder. +pub const HARNESS_ENV: &str = "INSI_HARNESS"; +/// Standing instructions. +pub const INSTRUCTIONS_FILE: &str = "instructions.md"; +/// Role and tier routing. +pub const ROUTING_FILE: &str = "roles.json"; +/// Sub-agent definitions. +pub const AGENTS_DIR: &str = "agents"; +/// The most of `instructions.md` given inline, as for the project's own +/// instruction files. +pub const MAX_BYTES: usize = crate::instructions::MAX_BYTES; + +/// The harness folder this run was given, if any. +#[must_use] +pub fn dir() -> Option { + std::env::var_os(HARNESS_ENV) + .filter(|v| !v.is_empty()) + .map(PathBuf::from) +} + +/// Routing from a harness: per role and per tier, over `admin.json`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Routing { + #[serde(default)] + pub roles: BTreeMap>, + #[serde(default)] + pub tiers: BTreeMap>, +} + +/// The routing in `dir`; `Ok(None)` when it has no routing file. +pub fn routing_in(dir: &Path) -> Result, String> { + let path = dir.join(ROUTING_FILE); + match std::fs::read_to_string(&path) { + Ok(text) => serde_json::from_str(&text) + .map(Some) + .map_err(|e| format!("{}: {e}", path.display())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(format!("{}: {e}", path.display())), + } +} + +/// This run's harness routing, when there is a harness with routing that +/// reads. A problem is traced; [`check`] is what refuses a run over it. +#[must_use] +pub fn routing() -> Option { + let dir = dir()?; + match routing_in(&dir) { + Ok(routing) => routing, + Err(why) => { + crate::trace!("harness", "routing left out: {why}"); + None + } + } +} + +/// `roles` and `tiers` with the harness's entries over them, key by key. +#[must_use] +pub fn overlay( + roles: &BTreeMap>, + tiers: &BTreeMap>, + harness: Option<&Routing>, +) -> (BTreeMap>, BTreeMap>) { + let mut roles = roles.clone(); + let mut tiers = tiers.clone(); + if let Some(h) = harness { + roles.extend(h.roles.iter().map(|(k, v)| (k.clone(), v.clone()))); + tiers.extend(h.tiers.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + (roles, tiers) +} + +/// The harness's standing instructions as one context block, or `None` +/// when there is no harness or no instructions in it. +pub fn instructions_in(dir: &Path) -> Result, String> { + let path = dir.join(INSTRUCTIONS_FILE); + let text = match std::fs::read_to_string(&path) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(format!("{}: {e}", path.display())), + }; + let text = text.trim(); + if text.is_empty() { + return Ok(None); + } + let head = crate::text::head_bytes(text, MAX_BYTES); + let cut = if head.len() < text.len() { + format!( + "\n[… {} more bytes — read `{}` for the rest.]", + text.len() - head.len(), + path.display() + ) + } else { + String::new() + }; + Ok(Some(format!( + "\nStanding instructions for how to work, from the harness \ + this run was given.\n\n{head}{cut}\n" + ))) +} + +/// This run's harness instructions, if any; a problem is traced. +#[must_use] +pub fn instructions_block() -> Option { + let dir = dir()?; + instructions_in(&dir).unwrap_or_else(|why| { + crate::trace!("harness", "instructions left out: {why}"); + None + }) +} + +/// Whether this run's harness, if it was given one, can be used: the folder +/// exists and every file incode reads from it reads. A run refuses to start +/// on a harness it would have to half-ignore. +pub fn check() -> Result<(), String> { + let Some(dir) = dir() else { + return Ok(()); + }; + if !dir.is_dir() { + return Err(format!( + "{HARNESS_ENV} names {}, which is not a folder", + dir.display() + )); + } + routing_in(&dir)?; + instructions_in(&dir)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn folder() -> tempfile::TempDir { + tempfile::tempdir().expect("dir") + } + + /// **THE HARNESS'S ROUTING WINS PER KEY, AND ONLY WHERE IT SAYS.** + #[test] + fn harness_routing_overlays_the_admin_file_key_by_key() { + let dir = folder(); + std::fs::write( + dir.path().join(ROUTING_FILE), + r#"{"roles": {"plan": ["@fast"]}, "tiers": {"fast": ["model-b"]}}"#, + ) + .expect("write"); + let routing = routing_in(dir.path()).expect("reads").expect("present"); + let mut roles = BTreeMap::new(); + roles.insert("plan".to_string(), vec!["model-a".to_string()]); + roles.insert("review".to_string(), vec!["model-r".to_string()]); + let (roles, tiers) = overlay(&roles, &BTreeMap::new(), Some(&routing)); + assert_eq!(roles["plan"], ["@fast"]); + assert_eq!( + roles["review"], + ["model-r"], + "a role it does not name is kept" + ); + assert_eq!(tiers["fast"], ["model-b"]); + assert_eq!(routing_in(folder().path()).expect("absent"), None); + } + + /// **A HARNESS FILE THAT DOES NOT READ IS AN ERROR, NOT A DEFAULT.** + #[test] + fn a_broken_harness_file_is_reported() { + let dir = folder(); + std::fs::write(dir.path().join(ROUTING_FILE), "{not json").expect("write"); + assert!(routing_in(dir.path()).is_err()); + std::fs::write(dir.path().join(ROUTING_FILE), r#"{"rolez": {}}"#).expect("write"); + assert!( + routing_in(dir.path()).is_err(), + "a misspelt key is refused, not ignored" + ); + } + + #[test] + fn instructions_are_wrapped_and_bounded() { + let dir = folder(); + assert_eq!(instructions_in(dir.path()).expect("absent"), None); + std::fs::write(dir.path().join(INSTRUCTIONS_FILE), " \n").expect("write"); + assert_eq!(instructions_in(dir.path()).expect("blank"), None); + std::fs::write(dir.path().join(INSTRUCTIONS_FILE), "Run the tests first.").expect("w"); + let block = instructions_in(dir.path()) + .expect("reads") + .expect("present"); + assert!(block.starts_with("")); + assert!(block.contains("Run the tests first.")); + std::fs::write( + dir.path().join(INSTRUCTIONS_FILE), + "x".repeat(MAX_BYTES + 10), + ) + .expect("w"); + let block = instructions_in(dir.path()) + .expect("reads") + .expect("present"); + assert!( + block.contains("10 more bytes"), + "{}", + &block[block.len() - 200..] + ); + } +} diff --git a/src/hooks/memory.rs b/src/hooks/memory.rs index 561a1b1..1e44ad2 100644 --- a/src/hooks/memory.rs +++ b/src/hooks/memory.rs @@ -62,8 +62,10 @@ pub const TURN_INLINE: usize = 6_000; /// Where transcripts are written, under the workspace. pub(crate) const TRANSCRIPTS: &str = ".incode/transcripts"; -/// Where an oversized block is written whole, under the workspace. -const CONTEXT: &str = ".incode/context"; +/// Where an oversized block is written whole, under the workspace. Also where +/// `crate::agent::budget_for_spawn` writes a task too long for a command +/// line (src/agent/mod.rs `spill`), so `prune` keeps both bounded. +pub(crate) const CONTEXT: &str = ".incode/context"; /// How many files each of those two directories keeps. The newest survive. const KEEP: usize = 200; diff --git a/src/ledger.rs b/src/ledger.rs index 85c4a6d..8454396 100644 --- a/src/ledger.rs +++ b/src/ledger.rs @@ -133,6 +133,14 @@ pub struct Task { /// is no such branch, never "the branch is gone". #[serde(default)] pub unmerged_branch: Option, + /// The pull request this task's work is waiting in, by URL. + /// + /// Set when a lane's work was opened for review instead of merged here + /// (`INCODE_LANES_AS=pr`); the task is `Blocked` until a person merges or + /// closes it, and `crate::reviews::settle` reads which at the + /// start of the next run. `None` means no review is pending. + #[serde(default)] + pub pull_request: Option, /// The file this task must write, for work that produces information /// rather than code. /// @@ -170,6 +178,7 @@ impl Task { worked_by: String::new(), owner_note: String::new(), unmerged_branch: None, + pull_request: None, deliverable: None, } } @@ -502,6 +511,28 @@ impl Ledger { // it. Keeping the name would point a later reader — or a later // attempt — at a branch that no longer describes anything. task.unmerged_branch = None; + task.pull_request = None; + Ok(()) + } + + /// A lane's work opened as pull request `url` from `branch`: out of the + /// queue until a person merges or closes it. + /// + /// `Blocked`, because that is what it is — a decision outstanding with a + /// person — so nothing picks it up, its dependants stay `Pending`, and a + /// run that drains around it says it is waiting rather than finished. It + /// is NOT `Done`: the change is not in the tree the check runs over. + pub fn await_review(&mut self, id: &str, url: &str, branch: &str) -> Result<(), String> { + let task = self + .get_mut(id) + .ok_or_else(|| format!("no task `{id}` in the ledger"))?; + task.state = TaskState::Blocked; + task.note = format!( + "waiting for review: {url}. Merge it and run again to have it checked and \ + finished; close it to have the task done again." + ); + task.pull_request = Some(url.to_string()); + task.unmerged_branch = Some(branch.to_string()); Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 7b24d04..d04fe57 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod fleet; pub mod folders; pub mod goal; pub mod guards; +pub mod harness; pub mod hooks; pub mod identity; pub mod instructions; @@ -35,6 +36,7 @@ pub mod plan; pub mod recall; pub mod recorder; pub mod research; +pub mod reviews; pub mod roles; pub mod rules; pub mod runner; diff --git a/src/parallel.rs b/src/parallel.rs index b89c177..c575e19 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -215,8 +215,17 @@ pub enum Merged { Nothing, /// Two lanes touched the same lines. NOT resolved here. Conflict { files: Vec }, + /// The task's check failed in the lane's own checkout, so the work was + /// not merged: it stays on its branch and the task is done again in the + /// main tree. + CheckFailed { why: String }, /// The merge could not be attempted. Failed { why: String }, + /// Its check passed in its lane and it was opened as a pull request + /// ([`Landing::Review`]); nothing was merged here. Not landed: a person + /// merges it, and the next run checks the pulled tree before the task is + /// done. + Opened { url: String }, } impl Merged { @@ -241,13 +250,174 @@ impl Merged { files.join(", "), lane.branch ), + Merged::CheckFailed { why } => format!( + "`{}` failed its check in its own lane, so it was NOT merged. Its work is \ + kept on branch `{}` and the task goes back to the queue. The check said: {}", + lane.task_id, + lane.branch, + crate::text::head_bytes(why, 600) + ), Merged::Failed { why } => { format!("`{}` could not be merged: {why}", lane.task_id) } + Merged::Opened { url } => format!( + "`{}` passed its check in its own lane and is open for review at {url} \ + (branch `{}`). Nothing was merged here; once it is merged, the next run \ + checks the pulled tree and finishes the task.", + lane.task_id, lane.branch + ), } } } +/// Where a lane's work goes once its own check has passed. +pub enum Landing<'a> { + /// Merged into the tree the run started from. + Here, + /// Pushed and opened as a draft pull request onto `base`, the branch the + /// run is on; a person merges it. + Review { + forge: &'a dyn crate::tools::pull_request::Forge, + base: String, + }, +} + +/// The switch for [`lanes_as_pull_requests`]. +pub const LANES_AS_ENV: &str = "INCODE_LANES_AS"; + +/// Whether the operator asked for lanes to be opened as pull requests +/// (`INCODE_LANES_AS=pr`) rather than merged here, the default. +#[must_use] +pub fn lanes_as_pull_requests() -> bool { + std::env::var(LANES_AS_ENV) + .map(|v| v.trim().eq_ignore_ascii_case("pr")) + .unwrap_or(false) +} + +/// The branch lanes are opened onto as pull requests: the one checked out +/// at `repo_root`. Asked once, at the start of a run, so a run that cannot +/// open pull requests says so before any lane works rather than per lane. +/// +/// # Errors +/// +/// HEAD detached, no `origin`, the forge unreachable (`gh` missing or not +/// logged in), or the branch not on `origin` — a pull request needs a base +/// the forge has. +pub fn review_base( + repo_root: &Path, + forge: &dyn crate::tools::pull_request::Forge, +) -> Result { + let branch = git(repo_root, &["rev-parse", "--abbrev-ref", "HEAD"])?; + if branch == "HEAD" { + return Err("HEAD is detached, so there is no branch to open them onto".to_string()); + } + git(repo_root, &["remote", "get-url", "origin"]) + .map_err(|_| "this repository has no `origin` remote".to_string())?; + forge.default_branch(repo_root)?; + git( + repo_root, + &[ + "rev-parse", + "--verify", + "--quiet", + &format!("refs/remotes/origin/{branch}"), + ], + ) + .map_err(|_| format!("`{branch}` is not on origin; push it first"))?; + Ok(branch) +} + +/// Refused when HEAD has commits `origin/` lacks: lanes branch from +/// HEAD, so each pull request would carry those commits as well as its own. +/// Asked before each batch, since the run can move HEAD. +/// +/// # Errors +/// +/// The commits are unpushed, or git could not say. +pub fn nothing_unpushed(repo_root: &Path, base: &str) -> Result<(), String> { + git( + repo_root, + &[ + "merge-base", + "--is-ancestor", + "HEAD", + &format!("origin/{base}"), + ], + ) + .map_err(|_| { + format!( + "`{base}` has commits origin does not, and every pull request would carry them; \ + push `{base}` first" + ) + }) + .map(|_| ()) +} + +/// Seal a lane and open its branch as a draft pull request onto `base`, +/// through `/pr`'s own path and refusals (nothing new, a credential in the +/// diff). A task with no check is merged here instead: nothing could finish +/// it after the review, and the runner hands such a task to the owner. +pub fn open_for_review( + repo_root: &Path, + lane: &Lane, + task: &crate::ledger::Task, + forge: &dyn crate::tools::pull_request::Forge, + base: &str, +) -> Merged { + if task.verify.is_empty() { + return merge_lane(repo_root, lane); + } + match seal_lane(lane) { + Ok(false) => return Merged::Nothing, + Ok(true) => {} + Err(why) => return Merged::Failed { why }, + } + match crate::tools::pull_request::open_onto( + repo_root, + Some(&lane.branch), + Some(base), + true, + forge, + ) { + Ok(opened) => Merged::Opened { url: opened.url }, + Err(why) => Merged::Failed { + why: format!("it could not be opened for review: {why}"), + }, + } +} + +/// Whether each lane's check runs in the lane before its work is merged. +/// +/// On unless `INCODE_LANE_CHECK` is `0`, `false` or `off`. Off is for a +/// project whose check cannot run in a fresh checkout (dependencies installed +/// only in the main tree): there a lane's check fails for want of them, and +/// every lane falls back to being done again in the main tree. +#[must_use] +pub fn lane_check_wanted() -> bool { + !matches!( + std::env::var(LANE_CHECK_ENV) + .unwrap_or_default() + .trim() + .to_ascii_lowercase() + .as_str(), + "0" | "false" | "off" + ) +} + +/// The switch for [`lane_check_wanted`]. +pub const LANE_CHECK_ENV: &str = "INCODE_LANE_CHECK"; + +/// Where `workspace` sits inside its repository (`git rev-parse +/// --show-prefix`), so a lane's check runs in the same folder of the lane's +/// checkout as the run's check does in the main tree. Empty at the root, and +/// when git cannot say. +#[must_use] +pub fn workspace_prefix(workspace: &Path) -> PathBuf { + git(workspace, &["rev-parse", "--show-prefix"]) + .map(|p| PathBuf::from(p.trim_end_matches('/'))) + .unwrap_or_default() +} + /// Run git in a directory and return stdout, or the error it printed. fn git(dir: &Path, args: &[&str]) -> Result { let out = std::process::Command::new("git") @@ -411,19 +581,28 @@ pub struct LaneOutcome { /// /// # What this deliberately does NOT do /// -/// It does not verify, does not touch the ledger, and does not decide that -/// anything is done. Those belong to [`crate::runner`], which owns the law -/// that the only edge into `Done` runs through [`crate::verify`]. This owns -/// the two things the runner cannot do from inside a sequential loop: -/// concurrency, and git. +/// It does not touch the ledger and does not decide that anything is done. +/// Those belong to [`crate::runner`], which owns the law that the only edge +/// into `Done` runs through [`crate::verify`]. This owns the things the runner +/// cannot do from inside a sequential loop: concurrency, git, and the gate +/// in front of each merge. +/// +/// # The check runs twice: in the lane, and after the merge +/// +/// AFTER the merge, by the runner, on the merged tree: a lane's own tree holds +/// one agent's work, and two lanes can each pass alone and break the moment +/// they are put together. Only the merged tree answers whether the task is +/// done, so that is still where `Done` is decided. /// -/// # Why the check must run AFTER the merge, not in the lane +/// IN the lane, here, before the merge (`check_in`): without it, work that +/// fails its own check is merged into the tree every later task builds on, +/// and the post-merge failure only marks the task — the broken change stays +/// in `HEAD`. A lane whose check fails is not merged; its work is kept on its +/// branch ([`Merged::CheckFailed`]) and the task is done again in the main +/// tree. `None` skips this gate ([`lane_check_wanted`]). /// -/// A lane's own tree contains one agent's work. Proving that tree proves the -/// agent did not break its own lane — which is not the question. Two lanes can -/// each pass alone and break the moment they are put together, which is the -/// ordinary result of two people editing one codebase. So the report comes -/// back as a claim and the runner checks the MERGED tree, serially, after. +/// A lane past that gate is merged here, or opened as a pull request when +/// `landing` is [`Landing::Review`] ([`open_for_review`]). /// /// # Merges are serial even though the work was not /// @@ -434,6 +613,8 @@ pub struct LaneOutcome { pub fn run_batch( repo_root: &Path, tasks: &[crate::ledger::Task], + check_in: Option<&Path>, + landing: &Landing<'_>, make_worker: F, ) -> Vec where @@ -462,7 +643,12 @@ where // Scoped so the borrow of `lanes` and `make_worker` needs no Arc, and so // every thread is joined before anything is merged. - type LaneResult = (Result, crate::runtime::usage::TokenUsage); + // The agent's report, what it spent, and why its lane's check failed. + type LaneResult = ( + Result, + crate::runtime::usage::TokenUsage, + Option, + ); let mut reports: Vec = std::thread::scope(|scope| { let handles: Vec<_> = lanes .iter() @@ -475,7 +661,18 @@ where // Read HERE, on the lane's own thread, because the worker // is dropped the moment this closure ends and nothing // outside can ask it afterwards. - (report, worker.spent_since_last_call()) + let spent = worker.spent_since_last_call(); + // On the lane's thread too, so the lanes' checks run at + // the same time as each other. Only for work the agent + // finished and a task that has a check: a task with none + // is the runner's to escalate, not this gate's to judge. + let failed_check = match check_in { + Some(prefix) if report.is_ok() && !task.verify.is_empty() => { + crate::verify::verify(&task.verify, &lane.workspace.join(prefix)).err() + } + _ => None, + }; + (report, spent, failed_check) }) }) .collect(); @@ -493,6 +690,7 @@ where ( Err("the lane panicked".to_string()), crate::runtime::usage::TokenUsage::default(), + None, ) }) }) @@ -503,17 +701,22 @@ where .iter() .zip(tasks) .zip(reports.drain(..)) - .map(|((lane, task), (report, spent))| { + .map(|((lane, task), (report, spent, failed_check))| { // A lane whose agent FAILED is not merged. Its work stays on the // branch: half a change that could not finish is the last thing // the main tree should be given, and the branch keeps it for - // anyone who wants to look. - let merged = if report.is_ok() { - merge_lane(repo_root, lane) - } else { - Merged::Failed { + // anyone who wants to look. Nor is one whose own check failed. + let merged = match (&report, failed_check) { + (Err(_), _) => Merged::Failed { why: "the agent did not finish, so nothing was merged".to_string(), - } + }, + (Ok(_), Some(why)) => Merged::CheckFailed { why }, + (Ok(_), None) => match landing { + Landing::Here => merge_lane(repo_root, lane), + Landing::Review { forge, base } => { + open_for_review(repo_root, lane, task, *forge, base) + } + }, }; close_lane_or_report(repo_root, lane); LaneOutcome { @@ -932,7 +1135,7 @@ mod tests { } } - let outcomes = run_batch(&root, &[ready("t-1")], |workspace| { + let outcomes = run_batch(&root, &[ready("t-1")], None, &Landing::Here, |workspace| { Box::new(GivesUp(workspace.to_path_buf())) }); @@ -1048,7 +1251,7 @@ mod tests { let tasks = vec![ready("t-1"), ready("t-2")]; let gate = std::sync::Arc::new(std::sync::Barrier::new(tasks.len())); - let outcomes = run_batch(&root, &tasks, |workspace| { + let outcomes = run_batch(&root, &tasks, None, &Landing::Here, |workspace| { Box::new(Rendezvous { workspace: workspace.to_path_buf(), gate: std::sync::Arc::clone(&gate), @@ -1104,7 +1307,7 @@ mod tests { } let tasks = vec![ready("t-1")]; - let outcomes = run_batch(&root, &tasks, |workspace| { + let outcomes = run_batch(&root, &tasks, None, &Landing::Here, |workspace| { Box::new(Refuses(workspace.to_path_buf())) }); @@ -1129,6 +1332,254 @@ mod tests { close_lane(&root, &lanes[0]).expect("close"); } + /// **WORK THAT FAILS ITS OWN CHECK NEVER REACHES THE MAIN TREE.** Two + /// lanes finish; one task's check passes in its lane and is merged, the + /// other's fails and is not — its file is absent from the main tree, kept + /// on its branch, and the outcome says why. With the gate off the same + /// failing lane merges, as it did before the gate existed. + #[test] + fn a_lane_that_fails_its_check_is_not_merged() { + let Some((_guard, root)) = a_repo() else { + return; + }; + struct Writes(PathBuf); + impl crate::runner::TaskWorker for Writes { + fn work(&mut self, task: &crate::ledger::Task) -> Result { + std::fs::write(self.0.join(format!("{}.txt", task.id)), "done\n") + .map_err(|e| e.to_string())?; + Ok(format!("{} finished", task.id)) + } + } + // Real checks, run in the lane: `shared.txt` exists in every checkout + // of this repository; `missing.txt` exists in none. + let mut passes = ready("t-pass"); + passes.verify = ["git", "ls-files", "--error-unmatch", "shared.txt"] + .map(String::from) + .to_vec(); + let mut fails = ready("t-fail"); + fails.verify = ["git", "ls-files", "--error-unmatch", "missing.txt"] + .map(String::from) + .to_vec(); + + let outcomes = run_batch( + &root, + &[passes.clone(), fails.clone()], + Some(Path::new("")), + &Landing::Here, + |workspace| Box::new(Writes(workspace.to_path_buf())), + ); + assert_eq!(outcomes[0].merged, Merged::Clean); + assert!( + root.join("t-pass.txt").exists(), + "the passing lane did not land" + ); + assert!( + matches!(&outcomes[1].merged, Merged::CheckFailed { .. }), + "{:?}", + outcomes[1].merged + ); + assert!(!outcomes[1].merged.landed()); + assert!( + !root.join("t-fail.txt").exists(), + "work that failed its check reached the main tree" + ); + git( + &root, + &["show", &format!("{}:t-fail.txt", outcomes[1].branch)], + ) + .expect("the failed lane's work was not kept on its branch"); + let lane = Lane { + task_id: "t-fail".into(), + workspace: root.clone(), + branch: outcomes[1].branch.clone(), + }; + assert!(outcomes[1] + .merged + .summary(&lane) + .contains(&outcomes[1].branch)); + + // The gate off: the failing check is not run in the lane, and the + // lane merges as before (the runner's check still judges it). + let mut again = ready("t-ungated"); + again.verify = fails.verify.clone(); + let ungated = run_batch(&root, &[again], None, &Landing::Here, |workspace| { + Box::new(Writes(workspace.to_path_buf())) + }); + assert_eq!(ungated[0].merged, Merged::Clean); + } + + /// A workspace below the repository's root checks in the same folder of + /// the lane; at the root the prefix is empty. + #[test] + fn the_check_runs_in_the_lane_folder_matching_the_workspace() { + let Some((_guard, root)) = a_repo() else { + return; + }; + assert_eq!(workspace_prefix(&root), PathBuf::new()); + let sub = root.join("app").join("web"); + std::fs::create_dir_all(&sub).expect("dir"); + assert_eq!(workspace_prefix(&sub), Path::new("app").join("web")); + } + + #[test] + fn the_lane_check_is_on_unless_switched_off() { + // A variable of its own, so this cannot race the lane-count test. + std::env::remove_var(LANE_CHECK_ENV); + assert!(lane_check_wanted(), "on by default"); + for off in ["0", "false", "OFF"] { + std::env::set_var(LANE_CHECK_ENV, off); + assert!(!lane_check_wanted(), "{off}"); + } + std::env::set_var(LANE_CHECK_ENV, "1"); + assert!(lane_check_wanted()); + std::env::remove_var(LANE_CHECK_ENV); + } + + /// GitHub, stood in for: what would have been opened, and a fixed URL. + #[derive(Default)] + struct ReviewForge { + created: std::sync::Mutex>, + } + + impl crate::tools::pull_request::Forge for ReviewForge { + fn default_branch(&self, _: &Path) -> Result { + Ok("main".to_string()) + } + fn open_pr(&self, _: &Path, _: &str) -> Result, String> { + Ok(None) + } + fn create( + &self, + _: &Path, + new: &crate::tools::pull_request::NewPr, + ) -> Result { + self.created.lock().expect("lock").push(new.clone()); + Ok("https://github.com/o/r/pull/9".to_string()) + } + fn pr_status( + &self, + _: &Path, + _: &str, + ) -> Result, String> { + Ok(None) + } + fn merge( + &self, + _: &Path, + _: u64, + _: crate::tools::pull_request::MergeMethod, + _: &str, + ) -> Result { + Err("not asked".to_string()) + } + fn review(&self, _: &Path, _: &str) -> Result { + Err("not asked".to_string()) + } + } + + /// `a_repo` with a bare `origin` holding its branch, as a pull request + /// needs. The remote's tempdir is returned so it outlives the test. + fn a_repo_with_origin() -> Option<(tempfile::TempDir, tempfile::TempDir, PathBuf, String)> { + let (guard, root) = a_repo()?; + let remote = tempfile::tempdir().ok()?; + git(remote.path(), &["init", "--bare"]).ok()?; + git(&root, &["remote", "add", "origin", remote.path().to_str()?]).ok()?; + let branch = git(&root, &["rev-parse", "--abbrev-ref", "HEAD"]).ok()?; + git(&root, &["push", "origin", &branch]).ok()?; + git(&root, &["fetch", "origin"]).ok()?; + Some((guard, remote, root, branch)) + } + + /// **A LANE OPENED FOR REVIEW LEAVES THE MAIN TREE ALONE.** Its check + /// passed, its branch is on the remote, a draft pull request onto the + /// run's own branch was opened — and nothing was merged here, so the + /// runner cannot prove the task against a tree without the change. + #[test] + fn a_lane_opened_for_review_is_pushed_and_not_merged() { + let Some((_guard, remote, root, base)) = a_repo_with_origin() else { + return; + }; + struct Writes(PathBuf); + impl crate::runner::TaskWorker for Writes { + fn work(&mut self, task: &crate::ledger::Task) -> Result { + std::fs::write(self.0.join(format!("{}.txt", task.id)), "done\n") + .map_err(|e| e.to_string())?; + Ok(format!("{} finished", task.id)) + } + } + let mut task = ready("t-pr"); + task.verify = ["git", "ls-files", "--error-unmatch", "shared.txt"] + .map(String::from) + .to_vec(); + let forge = ReviewForge::default(); + let landing = Landing::Review { + forge: &forge, + base: base.clone(), + }; + + let outcomes = run_batch(&root, &[task], Some(Path::new("")), &landing, |w| { + Box::new(Writes(w.to_path_buf())) + }); + assert_eq!( + outcomes[0].merged, + Merged::Opened { + url: "https://github.com/o/r/pull/9".to_string() + } + ); + assert!(!outcomes[0].merged.landed()); + assert!(!root.join("t-pr.txt").exists(), "the lane was merged here"); + let created = forge.created.lock().expect("lock"); + assert_eq!(created.len(), 1); + assert_eq!(created[0].base, base); + assert!(created[0].draft); + assert_eq!(created[0].head, outcomes[0].branch); + git( + remote.path(), + &[ + "rev-parse", + "--verify", + &format!("refs/heads/{}", outcomes[0].branch), + ], + ) + .expect("the lane's branch was not pushed"); + } + + /// Lanes cannot be opened onto a branch the remote lacks, nor while the + /// branch has commits the remote lacks — each pull request would carry + /// them. + #[test] + fn a_review_base_must_be_on_the_remote_with_nothing_unpushed() { + let Some((_guard, _remote, root, base)) = a_repo_with_origin() else { + return; + }; + let forge = ReviewForge::default(); + assert_eq!(review_base(&root, &forge).as_deref(), Ok(base.as_str())); + assert!(nothing_unpushed(&root, &base).is_ok()); + + std::fs::write(root.join("local.txt"), "x").expect("write"); + git(&root, &["add", "-A"]).expect("add"); + git(&root, &["commit", "-m", "not pushed"]).expect("commit"); + assert!(nothing_unpushed(&root, &base) + .expect_err("unpushed") + .contains("push")); + + git(&root, &["checkout", "-b", "local-only"]).expect("branch"); + assert!(review_base(&root, &forge) + .expect_err("not on origin") + .contains("not on origin")); + } + + #[test] + fn lanes_are_merged_here_unless_pull_requests_are_asked_for() { + std::env::remove_var(LANES_AS_ENV); + assert!(!lanes_as_pull_requests()); + std::env::set_var(LANES_AS_ENV, "PR"); + assert!(lanes_as_pull_requests()); + std::env::set_var(LANES_AS_ENV, "merge"); + assert!(!lanes_as_pull_requests()); + std::env::remove_var(LANES_AS_ENV); + } + /// The requested count is read from the environment and clamped. #[test] fn the_requested_count_falls_back_to_one() { diff --git a/src/reviews.rs b/src/reviews.rs new file mode 100644 index 0000000..7a617b4 --- /dev/null +++ b/src/reviews.rs @@ -0,0 +1,317 @@ +//! Tasks waiting on a pull request, settled at the start of a run. +//! +//! With `INCODE_LANES_AS=pr` a lane whose check passed is opened as a pull +//! request instead of merged ([`crate::parallel::Landing::Review`]), and its +//! task waits `Blocked` with the URL recorded +//! ([`crate::ledger::Ledger::await_review`]). A person merges or closes it on +//! the forge; this reads which, at the start of the next run, and moves the +//! task on through the same edges every other task takes: +//! +//! - **Merged:** the branch it was merged into is fast-forwarded here from +//! `origin`, and the task's check runs on that tree — the single edge into +//! `Done`. A failing check is a failed attempt. +//! - **Closed unmerged:** a failed attempt; the task goes back to the queue +//! with its branch remembered, as a conflicted lane does. +//! - **Still open**, or the forge unreachable: left waiting. + +use std::path::Path; + +use crate::ledger::{Ledger, TaskState}; +use crate::tools::pull_request::{Forge, PrState}; + +/// Settle every task waiting on a pull request. Returns one line per task +/// that moved or could not be read, and the ids it completed. +pub fn settle( + ledger: &mut Ledger, + workspace: &Path, + forge: &dyn Forge, +) -> (Vec, Vec) { + let waiting: Vec<(String, String)> = ledger + .tasks + .iter() + .filter(|t| t.state == TaskState::Blocked) + .filter_map(|t| t.pull_request.clone().map(|url| (t.id.clone(), url))) + .collect(); + let mut notes = Vec::new(); + let mut completed = Vec::new(); + for (id, url) in waiting { + let review = match forge.review(workspace, &url) { + Ok(review) => review, + Err(why) => { + notes.push(format!( + "`{id}` is still waiting: {url} could not be read: {why}" + )); + continue; + } + }; + let note = match review.state { + PrState::Open => continue, + PrState::Closed => { + let why = format!("its pull request {url} was closed without being merged"); + match ledger.record_failure(&id, &why) { + Ok(TaskState::Abandoned) => { + format!("`{id}`: {why}, and it is out of attempts: set aside") + } + _ => format!("`{id}`: {why}; it goes back to the queue"), + } + } + PrState::Merged => match bring_in_merged(workspace, &review.base) { + Err(why) => { + // Still Blocked: the work is merged on the forge and only + // this checkout is behind, which a person can fix. + let why = format!( + "{url} was merged into `{}`, but this checkout could not be brought \ + up to it: {why}. Pull it, then run again.", + review.base + ); + if let Some(task) = ledger.get_mut_public(&id) { + task.note = why.clone(); + } + notes.push(format!("`{id}`: {why}")); + continue; + } + Ok(()) => { + let check = ledger + .get(&id) + .map(|t| t.verify.clone()) + .unwrap_or_default(); + match crate::verify::verify(&check, workspace) { + Ok(proof) => { + let _ = ledger.complete(&id, &proof); + completed.push(id.clone()); + format!("`{id}`: {url} was merged and its check passes here: done") + } + Err(why) => { + let why = format!( + "{url} was merged, but its check fails on the merged tree: {}", + crate::text::head_bytes(&why, 600) + ); + let _ = ledger.record_failure(&id, &why); + format!("`{id}`: {why}") + } + } + } + }, + }; + // Settled one way or the other: no review is pending any more. + if let Some(task) = ledger.get_mut_public(&id) { + task.pull_request = None; + } + notes.push(note); + } + if !notes.is_empty() { + let _ = ledger.save(workspace); + } + (notes, completed) +} + +/// Fast-forward this checkout to `origin/`, which a pull request was +/// just merged into. Only a fast-forward, and only on `base` itself: a +/// checkout on another branch, or with commits of its own, is left for a +/// person rather than merged into. +fn bring_in_merged(workspace: &Path, base: &str) -> Result<(), String> { + let git = |args: &[&str]| -> Result { + let out = std::process::Command::new("git") + .arg("-C") + .arg(workspace) + .args(args) + .output() + .map_err(|e| format!("could not run git: {e}"))?; + if out.status.success() { + Ok(String::from_utf8_lossy(&out.stdout).trim().to_string()) + } else { + Err(String::from_utf8_lossy(&out.stderr).trim().to_string()) + } + }; + let current = git(&["rev-parse", "--abbrev-ref", "HEAD"])?; + if current != base { + return Err(format!("it is on `{current}`, not `{base}`")); + } + git(&["fetch", "origin", base])?; + git(&["merge", "--ff-only", &format!("origin/{base}")])?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ledger::Task; + use crate::tools::pull_request::{MergeMethod, NewPr, PrStatus, Review}; + use std::path::PathBuf; + use std::process::Command; + + /// The forge, stood in for: every pull request is in `state`, onto `main`. + struct Settled(PrState); + + impl Forge for Settled { + fn default_branch(&self, _: &Path) -> Result { + Ok("main".to_string()) + } + fn open_pr(&self, _: &Path, _: &str) -> Result, String> { + Ok(None) + } + fn create(&self, _: &Path, _: &NewPr) -> Result { + Err("not asked".to_string()) + } + fn pr_status(&self, _: &Path, _: &str) -> Result, String> { + Ok(None) + } + fn merge(&self, _: &Path, _: u64, _: MergeMethod, _: &str) -> Result { + Err("not asked".to_string()) + } + fn review(&self, _: &Path, _: &str) -> Result { + Ok(Review { + state: self.0, + base: "main".to_string(), + }) + } + } + + fn sh(dir: &Path, args: &[&str]) { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .expect("git"); + assert!( + out.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// A checkout of a bare `origin` whose `main` has since gained + /// `merged.txt` — what merging a pull request on the forge leaves + /// behind — while this checkout's `main` has not. + fn merged_on_the_forge() -> (tempfile::TempDir, PathBuf) { + let dir = tempfile::tempdir().expect("dir"); + let work = dir.path().join("work"); + sh(dir.path(), &["init", "--bare", "-b", "main", "remote.git"]); + sh(dir.path(), &["init", "-b", "main", "work"]); + for (k, v) in [ + ("user.email", "t@example.invalid"), + ("user.name", "t"), + ("core.autocrlf", "false"), + ] { + sh(&work, &["config", k, v]); + } + std::fs::write(work.join("a.txt"), "a").expect("write"); + sh(&work, &["add", "."]); + sh(&work, &["commit", "-m", "base"]); + let remote = dir.path().join("remote.git"); + sh( + &work, + &["remote", "add", "origin", remote.to_str().expect("utf8")], + ); + sh(&work, &["push", "origin", "main"]); + sh(&work, &["checkout", "-b", "lane/t-1"]); + std::fs::write(work.join("merged.txt"), "m").expect("write"); + sh(&work, &["add", "."]); + sh(&work, &["commit", "-m", "the lane's change"]); + sh(&work, &["push", "origin", "lane/t-1:main"]); + sh(&work, &["checkout", "main"]); + (dir, work) + } + + /// One task waiting on a review, checked by `verify`. + fn waiting(verify: &[&str]) -> Ledger { + let mut ledger = Ledger { + version: 1, + tasks: vec![Task::new("t-1", "r-1", "add merged.txt").with_verify(verify.to_vec())], + }; + ledger + .await_review("t-1", "https://github.com/o/r/pull/9", "lane/t-1") + .expect("waiting"); + ledger + } + + /// **A MERGED REVIEW IS FINISHED ONLY BY ITS CHECK, ON THE PULLED TREE.** + #[test] + fn a_merged_pull_request_is_pulled_checked_and_done() { + let (_dir, work) = merged_on_the_forge(); + let mut ledger = waiting(&["git", "ls-files", "--error-unmatch", "merged.txt"]); + + let (notes, completed) = settle(&mut ledger, &work, &Settled(PrState::Merged)); + assert_eq!(completed, ["t-1"], "{notes:?}"); + let task = ledger.get("t-1").expect("task"); + assert_eq!(task.state, TaskState::Done); + assert_eq!(task.pull_request, None); + assert!(work.join("merged.txt").exists(), "the merge was not pulled"); + } + + /// Merged on the forge and still broken here: a failed attempt, not + /// `Done` — the review is not the proof. + #[test] + fn a_merged_pull_request_whose_check_fails_is_a_failed_attempt() { + let (_dir, work) = merged_on_the_forge(); + let mut ledger = waiting(&["git", "ls-files", "--error-unmatch", "absent.txt"]); + + let (notes, completed) = settle(&mut ledger, &work, &Settled(PrState::Merged)); + assert!(completed.is_empty()); + let task = ledger.get("t-1").expect("task"); + assert_eq!(task.state, TaskState::Ready, "{notes:?}"); + assert_eq!(task.attempts, 1); + assert_eq!(task.pull_request, None); + assert!(task.note.contains("check fails"), "{}", task.note); + } + + /// Closed without merging: the task is done again, pointed at the + /// branch that already holds the work. + #[test] + fn a_closed_pull_request_sends_the_task_back_with_its_branch() { + let dir = tempfile::tempdir().expect("dir"); + let mut ledger = waiting(&["true"]); + + let (notes, _) = settle(&mut ledger, dir.path(), &Settled(PrState::Closed)); + let task = ledger.get("t-1").expect("task"); + assert_eq!(task.state, TaskState::Ready, "{notes:?}"); + assert_eq!(task.attempts, 1); + assert_eq!(task.unmerged_branch.as_deref(), Some("lane/t-1")); + assert_eq!(task.pull_request, None); + } + + /// Still under review: nothing moves and nothing is said. + #[test] + fn an_open_pull_request_leaves_the_task_waiting() { + let dir = tempfile::tempdir().expect("dir"); + let mut ledger = waiting(&["true"]); + + let (notes, completed) = settle(&mut ledger, dir.path(), &Settled(PrState::Open)); + assert!(notes.is_empty() && completed.is_empty()); + let task = ledger.get("t-1").expect("task"); + assert_eq!(task.state, TaskState::Blocked); + assert!(task.pull_request.is_some()); + } + + /// Merged into `main` while this checkout is on another branch: nothing + /// is merged into it, and the task waits with a note saying what to do. + #[test] + fn a_checkout_on_another_branch_is_not_merged_into() { + let (_dir, work) = merged_on_the_forge(); + sh(&work, &["checkout", "-b", "elsewhere"]); + let mut ledger = waiting(&["git", "ls-files", "--error-unmatch", "merged.txt"]); + + let (notes, completed) = settle(&mut ledger, &work, &Settled(PrState::Merged)); + assert!(completed.is_empty()); + let task = ledger.get("t-1").expect("task"); + assert_eq!(task.state, TaskState::Blocked); + assert!(task.pull_request.is_some()); + assert!(notes[0].contains("Pull it"), "{notes:?}"); + assert!(!work.join("merged.txt").exists()); + } + + /// **WORK BUILT ON A CHANGE UNDER REVIEW WAITS FOR IT.** A dependant is + /// not started while its dependency is in review, and the run says both + /// are waiting rather than that the work is finished. + #[test] + fn the_dependants_of_a_task_in_review_are_not_started() { + let mut ledger = waiting(&["true"]); + ledger + .tasks + .push(Task::new("t-2", "r-1", "builds on it").with_dependencies(["t-1"])); + ledger.refresh_ready(); + assert_eq!(ledger.get("t-2").expect("t-2").state, TaskState::Pending); + assert!(ledger.next_ready().is_none()); + assert_eq!(ledger.waiting_ids(), ["t-1", "t-2"]); + } +} diff --git a/src/runner.rs b/src/runner.rs index 79272a1..e61e411 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -92,6 +92,21 @@ pub trait LaneWorkers: Sync { fn available(&self) -> bool { true } + + /// Where pull requests are opened and read, when there is a forge. + /// + /// Read at the start of every run whenever a task is waiting on one + /// ([`crate::reviews::settle`]), so a review opened under `INCODE_LANES_AS=pr` is + /// still settled by a run without it. `None`: no forge, nothing settled. + fn forge(&self) -> Option<&dyn crate::tools::pull_request::Forge> { + None + } + + /// Whether a lane whose check passed is opened as a pull request on + /// [`Self::forge`] instead of merged here. + fn open_for_review(&self) -> bool { + false + } } /// No lanes: one agent, one tree, today's behaviour. @@ -137,6 +152,7 @@ struct LaneReports { /// one lane, the workspace is not a git repository, fewer than two tasks are /// ready, or the pool declines to supply workers. Each leaves the sequential /// loop exactly as it was. +#[allow(clippy::too_many_arguments)] fn run_lanes_if_useful( ledger: &mut Ledger, workspace: &Path, @@ -145,6 +161,7 @@ fn run_lanes_if_useful( reports: &mut LaneReports, budget: &mut Budget, cancelled: &dyn Fn() -> bool, + review: Option<(&dyn crate::tools::pull_request::Forge, &str)>, ) { if !reports.ready.is_empty() || cancelled() { return; @@ -175,14 +192,43 @@ fn run_lanes_if_useful( return; } - let outcomes = crate::parallel::run_batch(repo_root, &batch, |lane_workspace| { - lanes.spawn_for(lane_workspace).unwrap_or_else(|| { - // The probe above said yes. A pool that then changes its mind - // must not take the batch down with it: this lane reports a - // failure and the task goes back to the queue. - Box::new(UnavailableWorker) - }) - }); + // Each lane's check runs in its own checkout, in the folder matching this + // workspace, before its work may be merged (`INCODE_LANE_CHECK=0` turns + // that gate off; the check after the merge runs either way). + let check_in = + crate::parallel::lane_check_wanted().then(|| crate::parallel::workspace_prefix(workspace)); + // Opened for review rather than merged, when the run was started for + // that — unless the branch has commits the remote lacks, which every pull + // request would then carry. Asked per batch, since the run can move HEAD. + let landing = match review { + Some((forge, base)) => match crate::parallel::nothing_unpushed(repo_root, base) { + Ok(()) => crate::parallel::Landing::Review { + forge, + base: base.to_string(), + }, + Err(why) => { + reports.notes.push(format!( + "this batch is merged here, not opened for review: {why}" + )); + crate::parallel::Landing::Here + } + }, + None => crate::parallel::Landing::Here, + }; + let outcomes = crate::parallel::run_batch( + repo_root, + &batch, + check_in.as_deref(), + &landing, + |lane_workspace| { + lanes.spawn_for(lane_workspace).unwrap_or_else(|| { + // The probe above said yes. A pool that then changes its mind + // must not take the batch down with it: this lane reports a + // failure and the task goes back to the queue. + Box::new(UnavailableWorker) + }) + }, + ); for outcome in outcomes { // CHARGED FOR EVERY LANE, including the ones that failed or could not @@ -205,14 +251,18 @@ fn run_lanes_if_useful( reports.notes.push(outcome.merged.summary(&lane)); // WHERE THE UNMERGED WORK IS, written onto the task itself so the - // retry can be pointed at it. Recorded only for a conflict: that is - // the case where a real change exists on a branch and the tree simply - // moved underneath it. A lane whose agent failed left no change worth - // reconciling, and telling the next attempt to go and merge one would - // send it after a branch with nothing on it. + // retry can be pointed at it. Recorded for a conflict — a real change + // on a branch, with the tree moved underneath it — and for a lane + // whose finished work failed its own check: a whole change the retry + // can read and correct rather than redo. A lane whose agent failed + // left no change worth reconciling, and telling the next attempt to + // go and merge one would send it after a branch with nothing on it. if let Some(entry) = ledger.get_mut_public(&outcome.task.id) { match &outcome.merged { - crate::parallel::Merged::Conflict { .. } if !outcome.branch.is_empty() => { + crate::parallel::Merged::Conflict { .. } + | crate::parallel::Merged::CheckFailed { .. } + if !outcome.branch.is_empty() => + { entry.unmerged_branch = Some(outcome.branch.clone()); } // Landed, so any branch remembered from an earlier round is @@ -224,6 +274,17 @@ fn run_lanes_if_useful( } } + // OPENED FOR REVIEW: not done and not failed. It waits on a person, + // out of the queue, and no report is cached — the runner must not + // check a tree that does not hold the change. `reviews::settle` picks + // it up once the pull request is merged or closed. + if let crate::parallel::Merged::Opened { url } = &outcome.merged { + if let Err(why) = ledger.await_review(&outcome.task.id, url, &outcome.branch) { + reports.notes.push(why); + } + continue; + } + // A LANE THAT DID NOT LAND IS A FAILED ATTEMPT, whatever the agent // said. Its work is on a branch, not in the tree the check is about // to run over — so reporting success here would have the runner prove @@ -1162,6 +1223,39 @@ fn work_the_ledger( eprintln!("{note}"); } + // PULL REQUESTS SETTLED FIRST: a task whose review was merged since the + // last run is checked and finished before anything is picked, so its + // dependants become ready in this run rather than the next. + if let Some(forge) = lanes.forge() { + let (notes, done) = crate::reviews::settle(ledger, workspace, forge); + for note in notes { + eprintln!("{note}"); + } + completed.extend(done); + } + + // OPENED FOR REVIEW, when the operator asked for that and lanes will run. + // Decided once, here, so a run that cannot open pull requests says why + // before any lane works — and merges as it always has. + let review_base = match (&isolation, lanes.forge()) { + (crate::parallel::Isolation::Worktrees { repo_root }, Some(forge)) + if lanes.open_for_review() && crate::parallel::requested_lanes() > 1 => + { + match crate::parallel::review_base(repo_root, forge) { + Ok(base) => Some(base), + Err(why) => { + eprintln!("Lanes will be merged here, not opened as pull requests: {why}."); + None + } + } + } + _ => None, + }; + let review = match (lanes.forge(), &review_base) { + (Some(forge), Some(base)) => Some((forge, base.as_str())), + _ => None, + }; + loop { progress.ledger(ledger); if cancelled() { @@ -1201,6 +1295,7 @@ fn work_the_ledger( &mut lane_reports, budget, cancelled, + review, ); for note in lane_reports.notes.drain(..) { crate::trace!("lanes", "{note}"); diff --git a/src/subagents.rs b/src/subagents.rs index 514e680..ce90a68 100644 --- a/src/subagents.rs +++ b/src/subagents.rs @@ -61,6 +61,8 @@ const MAX_NAME_CHARS: usize = 64; pub enum Scope { /// The workspace's `.incode/agents`. Project, + /// The `agents/` of the harness this run was given (`INSI_HARNESS`). + Harness, /// The operator's `~/.incode/agents`. User, } @@ -69,6 +71,7 @@ impl Scope { fn as_str(self) -> &'static str { match self { Self::Project => "project", + Self::Harness => "harness", Self::User => "user", } } @@ -163,20 +166,33 @@ impl Loaded { } } -/// Every sub-agent defined for `workspace`: the project's, then the -/// operator's that the project does not override. +/// Every sub-agent defined for `workspace`: the project's, then the run's +/// harness's, then the operator's — each only where an earlier one does not +/// already define the name. #[must_use] pub fn load(workspace: &Path) -> Loaded { - load_from( - &workspace.join(PROJECT_DIR), - &crate::admin::AdminConfig::default_dir().join(USER_DIR), - ) + let project = workspace.join(PROJECT_DIR); + let harness = crate::harness::dir().map(|d| d.join(crate::harness::AGENTS_DIR)); + let user = crate::admin::AdminConfig::default_dir().join(USER_DIR); + let mut dirs = vec![(project.as_path(), Scope::Project)]; + if let Some(h) = &harness { + dirs.push((h.as_path(), Scope::Harness)); + } + dirs.push((user.as_path(), Scope::User)); + load_dirs(&dirs) } -/// [`load`], from two named directories. +/// [`load`], from a project and an operator directory. +#[cfg(test)] fn load_from(project: &Path, user: &Path) -> Loaded { + load_dirs(&[(project, Scope::Project), (user, Scope::User)]) +} + +/// Definitions from `dirs`, earliest first; a name already defined is +/// shadowed, not an error. +fn load_dirs(dirs: &[(&Path, Scope)]) -> Loaded { let mut loaded = Loaded::default(); - for (dir, scope) in [(project, Scope::Project), (user, Scope::User)] { + for &(dir, scope) in dirs { for found in read_dir(dir, scope, &mut loaded.problems) { if loaded.find(&found.name).is_some() { // The project's own wins; the operator's is shadowed, not an @@ -781,6 +797,45 @@ mod tests { assert!(note.contains("- scout: finds"), "{note}"); } + /// A harness's definitions sit between the project's and the operator's: + /// over the operator's of the same name, under the project's. + #[test] + fn a_harness_definition_sits_between_project_and_operator() { + let dir = tempfile::tempdir().expect("dir"); + let (project, harness, user) = ( + dir.path().join("p"), + dir.path().join("h"), + dir.path().join("u"), + ); + write(&project, "reviewer.md", REVIEWER); + write( + &harness, + "reviewer.md", + "---\nname: reviewer\ndescription: the harness's\n---\nh", + ); + write( + &harness, + "scout.md", + "---\nname: scout\ndescription: the harness's scout\n---\nh", + ); + write( + &user, + "scout.md", + "---\nname: scout\ndescription: the operator's scout\n---\nu", + ); + let loaded = load_dirs(&[ + (&project, Scope::Project), + (&harness, Scope::Harness), + (&user, Scope::User), + ]); + assert_eq!(loaded.agents.len(), 2, "{:?}", loaded.agents); + assert_eq!( + loaded.find("reviewer").map(|a| a.scope), + Some(Scope::Project) + ); + assert_eq!(loaded.find("scout").map(|a| a.scope), Some(Scope::Harness)); + } + /// A directory with too many files is read only to its bound, and says so. #[test] fn a_directory_is_read_only_to_its_bound() { diff --git a/src/task_run.rs b/src/task_run.rs index b3112f7..73c4aae 100644 --- a/src/task_run.rs +++ b/src/task_run.rs @@ -279,6 +279,9 @@ fn run_task_reporting( format!("refusing to run: the protected-path policy could not be read ({e}). Running without it would leave the check editable by the thing being judged.") })?; let sentinel = crate::guards::Sentinel::over(workspace, &guards); + // A harness that cannot be read stops the run too: the run would be + // scored as that harness while running on the defaults. + crate::harness::check().map_err(|e| format!("refusing to run: {e}"))?; // The memory engine's session, fired by incode: the model underneath runs // with no hooks of its own. One instruction is one session — it starts, @@ -297,9 +300,11 @@ fn run_task_reporting( let turn = memory.prompt(instruction); let prompt = build_prompt(workspace, instruction, start.is_some(), turn.as_deref()); // What rides the system context rather than the prompt: the project's - // own instruction files, and the engine's session-start block. + // own instruction files, the harness's standing instructions, and the + // engine's session-start block. let system: Vec = crate::instructions::project_block(workspace) .into_iter() + .chain(crate::harness::instructions_block()) .chain(start) .collect(); let task = Task { @@ -478,8 +483,9 @@ fn drive( backend.choose_model(m); } else { let admin = crate::admin::AdminConfig::load_default(); - let plan_models = admin.roles.get("plan").cloned().unwrap_or_default(); - let (expanded, _) = crate::roles::expand_tiers(&plan_models, &admin.tiers); + let (roles, tiers) = admin.effective_routing(); + let plan_models = roles.get("plan").cloned().unwrap_or_default(); + let (expanded, _) = crate::roles::expand_tiers(&plan_models, &tiers); if !expanded.is_empty() { backend.set_models(&expanded); } diff --git a/src/toolbox/code_edits_tests.rs b/src/toolbox/code_edits_tests.rs index cf05fcd..be334cf 100644 --- a/src/toolbox/code_edits_tests.rs +++ b/src/toolbox/code_edits_tests.rs @@ -7,10 +7,13 @@ use serde_json::{json, Value}; use super::*; +/// How a [`Recorder`] answers each call. +type Answer = Box Result + Send + Sync>; + /// A ledger that records every call and answers from `answer`. struct Recorder { calls: Mutex>, - answer: Box Result + Send + Sync>, + answer: Answer, } impl EditLedger for Recorder { diff --git a/src/tools/mod.rs b/src/tools/mod.rs index bc184cb..d688980 100644 --- a/src/tools/mod.rs +++ b/src/tools/mod.rs @@ -20,6 +20,7 @@ pub mod file_ops; pub mod git_context; pub mod pull_request; pub mod search; +pub mod secret_scan; pub(crate) mod shell_choice; pub mod tool_status; pub mod worktree; diff --git a/src/tools/pull_request.rs b/src/tools/pull_request.rs index 393bc1e..cb533e2 100644 --- a/src/tools/pull_request.rs +++ b/src/tools/pull_request.rs @@ -4,15 +4,20 @@ //! //! Pushing publishes work and a pull request is seen by other people, so //! nothing here runs on its own: `/pr` is typed by the operator, at the REPL -//! or over `--acp`, and no tool the model can call reaches it. +//! or over `--acp`, and no tool the model can call reaches it. The one other +//! caller is a run the operator started with `INCODE_LANES_AS=pr`, which +//! opens each lane whose check passed as a draft pull request through +//! [`open_onto`] — with the same refusals ([`crate::parallel::Landing`]). //! //! # What it refuses, before anything leaves the machine //! //! A branch name git would not accept, a detached HEAD, the default branch //! itself (that is a push, not a pull request), a current branch with //! uncommitted changes (they would not be in the PR the operator thinks they -//! are opening), a branch with no commits its base lacks, and a missing -//! `origin`. Each is refused by name and nothing is pushed. +//! are opening), a branch with no commits its base lacks, a missing +//! `origin`, and a branch that adds something credential-shaped +//! ([`crate::tools::secret_scan`]). Each is refused by name and nothing is +//! pushed. //! //! # GitHub through `gh` //! @@ -56,6 +61,24 @@ pub trait Forge { method: MergeMethod, head: &str, ) -> Result; + /// Where pull request `pr` (its URL) stands, open or not — the question + /// [`Self::pr_status`] cannot answer, since it sees only open ones. + fn review(&self, repo: &Path, pr: &str) -> Result; +} + +/// Whether a pull request is still open, was merged, or was closed unmerged. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PrState { + Open, + Merged, + Closed, +} + +/// A pull request's state and the branch it was opened onto. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Review { + pub state: PrState, + pub base: String, } /// How `/pr merge` lands a pull request on its base. @@ -268,6 +291,37 @@ impl Forge for GitHubCli { ], ) } + + fn review(&self, repo: &Path, pr: &str) -> Result { + // Read back from the ledger, a file on disk: only a URL is passed on, + // so nothing there can reach `gh` as an option. + if !pr.starts_with("https://") { + return Err(format!("\"{pr}\" is not a pull request URL")); + } + parse_review(&Self::run( + repo, + &["pr", "view", pr, "--json", "state,baseRefName"], + )?) + } +} + +/// [`Review`] from `gh pr view --json state,baseRefName`. A state other than +/// the three GitHub documents is an error, not a guess. +pub fn parse_review(json: &str) -> Result { + let v: serde_json::Value = + serde_json::from_str(json).map_err(|e| format!("gh printed unreadable JSON: {e}"))?; + let state = match v["state"].as_str() { + Some("OPEN") => PrState::Open, + Some("MERGED") => PrState::Merged, + Some("CLOSED") => PrState::Closed, + other => return Err(format!("gh reported a pull request state of {other:?}")), + }; + let base = v["baseRefName"] + .as_str() + .filter(|b| !b.is_empty()) + .ok_or("gh reported no base branch for the pull request")? + .to_string(); + Ok(Review { state, base }) } /// [`PrStatus`] from `gh pr view --json …`; `None` for one that is not open. @@ -479,6 +533,32 @@ pub fn merge( }) } +/// Refused when what `range` adds, or its commit messages, hold something +/// credential-shaped: pushing is publishing, and a key in a pull request +/// has to be revoked, not just deleted. The refusal names each place and +/// never the value. +fn refuse_credentials(repo: &Path, range: &str) -> Result<(), String> { + use crate::tools::secret_scan::{scan_diff, scan_messages}; + let diff = git(repo, &["diff", "--no-color", "--no-ext-diff", "-U0", range])?; + let messages = git(repo, &["log", "--format=%B", range])?; + let mut found = scan_diff(&diff); + found.extend(scan_messages(&messages)); + if found.is_empty() { + return Ok(()); + } + let listed: Vec = found.iter().take(10).map(|f| format!(" {f}")).collect(); + Err(format!( + "nothing was pushed: the branch adds what looks like a credential\n{}{}\n\ + take it out of the history (not only the last commit), and revoke it if it is real", + listed.join("\n"), + if found.len() > listed.len() { + format!("\n … and {} more", found.len() - listed.len()) + } else { + String::new() + } + )) +} + fn short(sha: &str) -> &str { &sha[..sha.len().min(7)] } @@ -488,6 +568,19 @@ pub fn open( branch: Option<&str>, draft: bool, forge: &dyn Forge, +) -> Result { + open_onto(repo, branch, None, draft, forge) +} + +/// [`open`], onto `base` when given rather than the forge's default branch: +/// a lane branched from a feature branch is opened onto that branch, or its +/// pull request would carry every commit the feature branch has. +pub fn open_onto( + repo: &Path, + branch: Option<&str>, + base: Option<&str>, + draft: bool, + forge: &dyn Forge, ) -> Result { let (branch, current, _) = resolve_branch(repo, branch)?; if branch == current { @@ -502,7 +595,10 @@ pub fn open( git(repo, &["remote", "get-url", "origin"]) .map_err(|_| "this repository has no `origin` remote to push to".to_string())?; - let base = forge.default_branch(repo)?; + let base = match base { + Some(base) => base.to_string(), + None => forge.default_branch(repo)?, + }; if branch == base { return Err(format!( "\"{branch}\" is the default branch; a pull request needs a branch of its own \ @@ -523,6 +619,7 @@ pub fn open( "\"{branch}\" has no commits that {base_ref} lacks; nothing to open a pull request for" )); } + refuse_credentials(repo, &range)?; git( repo, @@ -603,6 +700,9 @@ mod tests { .push((number, method, head.to_string())); Ok("0123456789abcdef".to_string()) } + fn review(&self, _: &Path, _: &str) -> Result { + Err("not asked in these tests".to_string()) + } } fn forge(existing: Option<&str>) -> FakeForge { @@ -742,6 +842,61 @@ mod tests { assert_eq!(parse_status(closed).expect("json"), None); } + /// **MERGED AND CLOSED ARE TOLD APART**, which `parse_status` cannot do: + /// one finishes a task, the other sends it back. + #[test] + fn a_pull_requests_state_and_base_are_read() { + for (state, want) in [ + ("OPEN", PrState::Open), + ("MERGED", PrState::Merged), + ("CLOSED", PrState::Closed), + ] { + let json = format!(r#"{{"state":"{state}","baseRefName":"dev"}}"#); + assert_eq!( + parse_review(&json).expect("read"), + Review { + state: want, + base: "dev".to_string() + } + ); + } + assert!(parse_review(r#"{"state":"DRAFTY","baseRefName":"dev"}"#).is_err()); + assert!(parse_review(r#"{"state":"OPEN"}"#).is_err()); + } + + /// **A PULL REQUEST ONTO THE BRANCH IT CAME FROM** carries only its own + /// commit, not everything that branch has beyond the default one — and + /// the forge is not asked for its default branch at all. + #[test] + fn a_branch_is_opened_onto_the_base_it_is_given() { + let (_dir, _remote, work) = repo(); + sh(&work, &["checkout", "-b", "dev"]); + commit(&work, "d.txt", "work already on dev"); + sh(&work, &["push", "origin", "dev"]); + sh(&work, &["fetch", "origin"]); + sh(&work, &["checkout", "-b", "lane/t-1"]); + commit(&work, "l.txt", "the lane's change"); + sh(&work, &["checkout", "dev"]); + let mut forge = forge(None); + forge.default = "never-asked".to_string(); + + let opened = open_onto(&work, Some("lane/t-1"), Some("dev"), true, &forge).expect("opened"); + assert_eq!(opened.base, "dev"); + let created = forge.created.borrow(); + assert_eq!(created[0].base, "dev"); + assert!(created[0].draft); + assert!( + created[0].body.contains("the lane's change"), + "{}", + created[0].body + ); + assert!( + !created[0].body.contains("work already on dev"), + "{}", + created[0].body + ); + } + fn sh(dir: &Path, args: &[&str]) { let ok = Command::new("git") .args(args) @@ -863,6 +1018,35 @@ mod tests { .unwrap_or(false)); } + /// **A KEY NEVER LEAVES THE MACHINE.** A branch that adds a + /// credential-shaped string is refused before the push: the remote never + /// sees the branch, no PR is created, and the refusal names the file and + /// line without the value. + #[test] + fn a_branch_that_adds_a_credential_is_refused_before_the_push() { + let (_dir, remote, work) = repo(); + sh(&work, &["checkout", "-b", "feature/leak"]); + let token = format!("ghp_{}", "Q".repeat(36)); + commit(&work, "config.js", &format!("const t = \"{token}\";")); + let forge = forge(None); + + let why = open(&work, None, false, &forge).expect_err("refused"); + assert!(why.contains("config.js:1: a GitHub token"), "{why}"); + assert!(!why.contains(&token), "the value was repeated: {why}"); + assert!(forge.created.borrow().is_empty()); + assert!(Command::new("git") + .args([ + "rev-parse", + "--verify", + "--quiet", + "refs/heads/feature/leak" + ]) + .current_dir(&remote) + .output() + .map(|o| !o.status.success()) + .unwrap_or(false)); + } + #[test] fn the_url_is_read_from_what_gh_printed() { let out = "Creating pull request for x into main\n\nhttps://github.com/o/r/pull/12\n"; diff --git a/src/tools/secret_scan.rs b/src/tools/secret_scan.rs new file mode 100644 index 0000000..17e0559 --- /dev/null +++ b/src/tools/secret_scan.rs @@ -0,0 +1,377 @@ +//! Credentials and personal data in what is about to leave the machine. +//! +//! `/pr` pushes a branch and opens a pull request that other people read. +//! Before anything is pushed, the lines the branch ADDS and its commit +//! messages are checked for shapes that are credentials, and files that are +//! credentials by name; the added lines are also checked for personal data +//! ([`pii_in`]). A finding names the file and line and what it looked like — +//! never the value. +//! +//! Personal data is held to the same precision: a card number written like +//! one that passes Luhn, a US social security number in an issued range, an +//! email address on a real domain (not `example.*`, `.test`, `.invalid`, +//! `noreply`, or a `git@` remote). Commit messages are not checked for it: a +//! `Signed-off-by` trailer carries an email on purpose. +//! +//! Only high-precision shapes are used: a prefixed key, a cloud key id, a +//! JWT, a private-key block, a bearer token. The generic `token: ` +//! shape that `crate::agent::stderr_tail` redacts in error output is left +//! out on purpose: in source code it matches `token: String` on every other +//! line, and a check that cries wolf gets overridden. + +use std::sync::OnceLock; + +use regex::Regex; + +/// One credential-shaped thing in a diff or a message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Finding { + /// The file, or `commit message`. + pub place: String, + /// The line in the new file (1-based); 0 for a whole file or a message. + pub line: usize, + /// What it looked like: `a GitHub token`, `a private key file`. + pub what: &'static str, +} + +impl std::fmt::Display for Finding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.line == 0 { + write!(f, "{}: {}", self.place, self.what) + } else { + write!(f, "{}:{}: {}", self.place, self.line, self.what) + } + } +} + +/// Credential shapes, each with what it is called in a finding. +const SHAPES: &[(&str, &str)] = &[ + (r"\bsk-[A-Za-z0-9_-]{20,}", "an API key (sk-…)"), + (r"\bgh[pousr]_[A-Za-z0-9]{20,}", "a GitHub token"), + (r"\bgithub_pat_[A-Za-z0-9_]{20,}", "a GitHub token"), + (r"\bxox[abprs]-[A-Za-z0-9-]{10,}", "a Slack token"), + (r"\bAKIA[0-9A-Z]{16}\b", "a cloud access key id"), + (r"\bAIza[0-9A-Za-z_-]{35}\b", "a Google API key"), + ( + r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}", + "a JSON web token", + ), + (r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "a private key"), + (r"(?i)\bbearer\s+[A-Za-z0-9._~+/=-]{20,}", "a bearer token"), +]; + +fn shapes() -> &'static [(Regex, &'static str)] { + static COMPILED: OnceLock> = OnceLock::new(); + COMPILED.get_or_init(|| { + SHAPES + .iter() + .filter_map(|(p, what)| Regex::new(p).ok().map(|re| (re, *what))) + .collect() + }) +} + +/// What a file is, when its name alone makes it a credential. +fn credential_file(path: &str) -> Option<&'static str> { + let name = path.rsplit('/').next().unwrap_or(path); + let lower = name.to_ascii_lowercase(); + // `.env`, `.env.production` — but the templates people commit on + // purpose are not credentials. + if lower == ".env" + || (lower.starts_with(".env.") + && ![".env.example", ".env.sample", ".env.template", ".env.dist"] + .contains(&lower.as_str())) + { + return Some("an environment file"); + } + if ["id_rsa", "id_dsa", "id_ecdsa", "id_ed25519"].contains(&lower.as_str()) { + return Some("a private key file"); + } + if [".pem", ".p12", ".pfx"] + .iter() + .any(|ext| lower.ends_with(ext)) + { + return Some("a key or certificate store"); + } + None +} + +fn shape_in(text: &str) -> Option<&'static str> { + shapes() + .iter() + .find(|(re, _)| re.is_match(text)) + .map(|(_, what)| *what) +} + +/// What personal data `line` holds the shape of, if any. +pub fn pii_in(line: &str) -> Option<&'static str> { + if card_number(line) { + Some("a payment card number") + } else if us_ssn(line) { + Some("a US social security number") + } else if personal_email(line) { + Some("an email address") + } else { + None + } +} + +fn luhn(digits: &[u32]) -> bool { + let sum: u32 = digits + .iter() + .rev() + .enumerate() + .map(|(i, &d)| match (i % 2 == 1, d * 2) { + (true, doubled) if doubled > 9 => doubled - 9, + (true, doubled) => doubled, + (false, _) => d, + }) + .sum(); + sum.is_multiple_of(10) +} + +/// Four groups of four digits, or a bare 15–16 digits with an issuer's +/// prefix (4, 51–55, 2221–2720, 34, 37, 6011, 65) — passing Luhn either way. +fn card_number(line: &str) -> bool { + static RUN: OnceLock = OnceLock::new(); + let run = RUN.get_or_init(|| { + Regex::new( + r"(?:^|[^0-9A-Za-z])(\d{4}[ -]\d{4}[ -]\d{4}[ -]\d{4}|\d{15,16})(?:$|[^0-9A-Za-z])", + ) + .expect("a valid pattern") + }); + run.captures_iter(line).any(|c| { + let text = &c[1]; + let digits: Vec = text.chars().filter_map(|ch| ch.to_digit(10)).collect(); + let prefix = |n: usize| text[..n].parse::().unwrap_or(0); + let issuer = text.contains([' ', '-']) + || text.starts_with('4') + || (51..=55).contains(&prefix(2)) + || (2221..=2720).contains(&prefix(4)) + || [34, 37, 65].contains(&prefix(2)) + || prefix(4) == 6011; + let same = digits.windows(2).all(|w| w[0] == w[1]); + issuer && !same && luhn(&digits) + }) +} + +/// `ddd-dd-dddd` alone, in a range that is issued: not area 000, 666 or +/// 900–999, not group 00, not serial 0000. +fn us_ssn(line: &str) -> bool { + static SSN: OnceLock = OnceLock::new(); + let ssn = SSN.get_or_init(|| { + Regex::new(r"(?:^|[^0-9A-Za-z-])(\d{3})-(\d{2})-(\d{4})(?:$|[^0-9A-Za-z-])") + .expect("a valid pattern") + }); + ssn.captures_iter(line).any(|c| { + let area: u32 = c[1].parse().unwrap_or(0); + area != 0 && area != 666 && area < 900 && &c[2] != "00" && &c[3] != "0000" + }) +} + +/// An email address on a real domain: not a reserved or placeholder one, +/// not a `noreply` address, not a `git@host` remote. +fn personal_email(line: &str) -> bool { + static EMAIL: OnceLock = OnceLock::new(); + let email = EMAIL.get_or_init(|| { + Regex::new(r"([A-Za-z0-9._%+-]+)@([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.([A-Za-z]{2,}))\b") + .expect("a valid pattern") + }); + email.captures_iter(line).any(|c| { + let local = c[1].to_ascii_lowercase(); + let domain = c[2].to_ascii_lowercase(); + let tld = c[3].to_ascii_lowercase(); + let reserved = ["example.com", "example.org", "example.net"] + .iter() + .any(|d| domain == *d || domain.ends_with(&format!(".{d}"))) + || ["example", "invalid", "test", "localhost", "local"].contains(&tld.as_str()) + || domain.ends_with("users.noreply.github.com"); + let placeholder = local == "git" || local.contains("noreply") || local.contains("no-reply"); + !reserved && !placeholder + }) +} + +/// Every credential in what a unified diff ADDS: its `+` lines, read with +/// their line numbers in the new file, and every file it adds whose name +/// makes it a credential. Removed lines are not looked at: deleting a +/// secret is the fix, not the leak. +pub fn scan_diff(diff: &str) -> Vec { + let mut found = Vec::new(); + let mut file = String::new(); + let mut new_line = 0usize; + for line in diff.lines() { + if let Some(path) = line.strip_prefix("+++ ") { + file = path.strip_prefix("b/").unwrap_or(path).to_string(); + if file != "/dev/null" { + if let Some(what) = credential_file(&file) { + found.push(Finding { + place: file.clone(), + line: 0, + what, + }); + } + } + continue; + } + if line.starts_with("--- ") || line.starts_with("diff --git") { + continue; + } + if let Some(rest) = line.strip_prefix("@@ ") { + // `@@ -a,b +c,d @@`: the new file's lines start at c. + new_line = rest + .split_whitespace() + .find_map(|part| part.strip_prefix('+')) + .and_then(|start| start.split(',').next()) + .and_then(|n| n.parse().ok()) + .unwrap_or(0); + continue; + } + if let Some(added) = line.strip_prefix('+') { + if let Some(what) = shape_in(added).or_else(|| pii_in(added)) { + found.push(Finding { + place: file.clone(), + line: new_line, + what, + }); + } + new_line += 1; + } else if line.starts_with(' ') { + new_line += 1; + } + } + found +} + +/// Credentials in commit messages, which become the pull request's title +/// and body. +pub fn scan_messages(messages: &str) -> Vec { + shape_in(messages) + .map(|what| Finding { + place: "a commit message".to_string(), + line: 0, + what, + }) + .into_iter() + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + // Each fixture is SHAPED like a credential and is not one: the right + // prefix and length, built at run time, so nothing real is written + // down and no scanner reading this file is fooled. + fn fake(prefix: &str, n: usize) -> String { + format!("{prefix}{}", "Q".repeat(n)) + } + + #[test] + fn every_shape_compiles() { + assert_eq!(shapes().len(), SHAPES.len()); + } + + /// **A KEY IN AN ADDED LINE IS FOUND, AT ITS LINE.** The finding names + /// the file, the line in the new file and the kind — never the value. + #[test] + fn an_added_credential_is_found_with_its_file_and_line() { + let token = fake("ghp_", 36); + let diff = format!( + "diff --git a/src/config.ts b/src/config.ts\n\ + --- a/src/config.ts\n\ + +++ b/src/config.ts\n\ + @@ -10,2 +10,3 @@ export const config = {{\n\ + \x20 name: \"app\",\n\ + + github: \"{token}\",\n\ + \x20 debug: false,\n" + ); + let found = scan_diff(&diff); + assert_eq!( + found, + [Finding { + place: "src/config.ts".into(), + line: 11, + what: "a GitHub token" + }] + ); + assert!(!found[0].to_string().contains(&token), "the value leaked"); + } + + /// **WHAT IS NOT A LEAK IS NOT FLAGGED.** A removed key (the fix), code + /// that merely names a token, and an `.env.example` template. + #[test] + fn removals_ordinary_code_and_templates_are_not_flagged() { + let token = fake("sk-", 32); + let diff = format!( + "--- a/src/a.rs\n+++ b/src/a.rs\n@@ -1,3 +1,2 @@\n\ + -let key = \"{token}\";\n\ + +let token: String = read_token();\n\ + +let password = std::env::var(\"PASSWORD\")?;\n\ + --- /dev/null\n+++ b/.env.example\n@@ -0,0 +1 @@\n+API_KEY=\n" + ); + assert_eq!(scan_diff(&diff), []); + } + + #[test] + fn credential_files_and_messages_are_found() { + let diff = "--- /dev/null\n+++ b/.env\n@@ -0,0 +1 @@\n+MODE=dev\n\ + --- /dev/null\n+++ b/deploy/id_ed25519\n@@ -0,0 +1 @@\n+x\n\ + --- /dev/null\n+++ b/certs/server.pem\n@@ -0,0 +1 @@\n+x\n"; + let places: Vec = scan_diff(diff).into_iter().map(|f| f.place).collect(); + assert_eq!(places, [".env", "deploy/id_ed25519", "certs/server.pem"]); + + let pem = format!("-----BEGIN {}PRIVATE KEY-----", "RSA "); + assert_eq!( + scan_diff(&format!("+++ b/k.txt\n@@ -0,0 +1 @@\n+{pem}\n"))[0].what, + "a private key" + ); + assert_eq!( + scan_messages(&format!("fix auth\n\nused {}", fake("AKIA", 16)))[0].what, + "a cloud access key id" + ); + assert!(scan_messages("fix: read the token from the keychain").is_empty()); + } + + /// **PERSONAL DATA IN AN ADDED LINE IS FOUND; WHAT ONLY LOOKS LIKE IT + /// IS NOT.** Test card numbers (Luhn-valid, never issued), an + /// issued-range SSN and a real-domain email are found; epoch + /// milliseconds, versions, dates, placeholder emails, git remotes and + /// npm specs are not; and a commit message's sign-off is not scanned. + #[test] + fn personal_data_is_found_and_lookalikes_are_not() { + let visa = ["4111", "1111", "1111", "1111"]; + for (line, what) in [ + (format!("card: {}", visa.join(" ")), "a payment card number"), + (format!("pan={}", visa.concat()), "a payment card number"), + ( + format!("ssn {}-{}-{}", "123", "45", "6789"), + "a US social security number", + ), + ( + format!("mail {}@{}", "jane.doe", "gmail.com"), + "an email address", + ), + ] { + assert_eq!(pii_in(&line), Some(what), "{line}"); + } + for line in [ + "const now = 1696212345678;", + "id = 4111111111111112", + "let n = 1234 5678 9012 3456;", + "date 2026-10-02", + "ssn: 000-12-3456", + "author: Jo ", + "Co-Authored-By: bot ", + "url = git@github.com:o/r.git", + "npm i @scope/pkg@1.2.3", + "send to admin@service.test", + ] { + assert_eq!(pii_in(line), None, "{line}"); + } + let diff = format!( + "+++ b/fixtures/users.json\n@@ -0,0 +1 @@\n+{{\"email\": \"{}@{}\"}}\n", + "jane", "gmail.com" + ); + assert_eq!(scan_diff(&diff)[0].what, "an email address"); + assert!(scan_messages("fix\n\nSigned-off-by: Jo ").is_empty()); + } +}