diff --git a/README.md b/README.md index 0e7d0aa..d7941b9 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,35 @@ # aatxe -> Statistical microbenchmark + regression-gate harness for TypeScript, Go, and Rust. +> Catch performance regressions on every pull request — with statistics, not vibes. -**Aatxe** [/ˈaːtʃe/] is the red-bull spirit of Basque mythology that emerges -from caves at night to identify and punish wrongdoers. The name is fitting: -this tool benches your code on every PR, statistically compares head against -base, posts a sticky comment, and gates CI when something regressed. +**aatxe** benches your code on each PR, statistically compares the change against +its base, and posts a single sticky comment that gates CI when something *actually* +regressed. It speaks **TypeScript, Go, and Rust** through one shared JSON report +format, runs as a **reusable GitHub Actions workflow** downstream repos call in one +line, and ships as a single static binary (`curl … | sh`, or `cargo install`). +```mermaid +flowchart LR + PR([PR pushed]) --> H["bench HEAD
(GitHub Actions)"] + PR --> B["bench base
(GitHub Actions)"] + H --> C["compare
median Δ · Mann–Whitney U · noise gate"] + B --> C + C --> M["sticky PR comment
(updated in place)"] ``` - ┌──────────────────────┐ -PR pushed ─┬──▶│ GH Actions: HEAD │ ─┐ - │ └──────────────────────┘ │ ┌────────────┐ ┌──────────────┐ - │ ┌──────────────────────┐ ├─▶ Compare ─▶ Sticky │ ─▶ │ GH PR comment │ - └──▶│ GH Actions: base │ ─┘ (median Δ + PR comment │ (in-place) │ - └──────────────────────┘ MW-U + noise gate) └──────────────┘ -``` -No LLM. No magic instrumentation. Statistics, not opinions. Verdicts come -from numbers, not text generation. +**Why you can trust the verdict.** No LLM, no magic instrumentation. A change is +only flagged when the median shift is large enough, statistically significant under +a non-parametric test, *and* clears a noise gate — three independent signals, all +required. Numbers, not opinions. ([methodology](#methodology)) + +**Plus an optional [agent council](#agent-council).** A mixture-of-agents LLM PR +reviewer layers semantic review on top — its own sticky comment, its own exit-code +gate, the perf gate untouched. It eats its own dogfood: the first published baseline +is **0.857 critical-F1 at 2.4 false positives per case** on a 24-case labeled corpus +([real-LLM baselines](#real-llm-baselines)). -A separate, **opt-in** subsystem — the [agent council](#agent-council) — -layers semantic PR review on top, with its own sticky marker and its own -exit-code gate. The perf gate above is untouched by it. +> **aatxe** [/ˈaːtʃe/] — the red-bull spirit of Basque mythology that emerges from +> caves at night to identify and punish wrongdoers. Fitting, for a regression detector. ## Why this rebuild @@ -36,9 +44,11 @@ with three sharper goals: * **GitHub Actions first-class.** Workflows ship in `.github/workflows/`, including a reusable `aatxe.yml` that downstream services can call. * **Testable end-to-end.** The core (`aatxe-core`) is pure — no IO, no - globals — and ships 52 unit + integration tests covering stats, the + globals — every side effect sits behind a trait so tests inject an + in-memory filesystem and git. The full workspace suite (stats, the three-signal verdict, markdown rendering, the affected-set graph across - all three languages, and the GitHub URL/header protocol. + all three languages, the GitHub protocol) runs on every PR via + `make check`. ## Hacking on aatxe @@ -228,11 +238,17 @@ producer can emit complete reports without the Rust binary in the loop. ``` aatxe/ -├── crates/ -│ ├── aatxe-core/ # types · stats · compare · report · affected · github (pure) +├── crates/ # six library crates + the CLI, layered so the brain exists once +│ ├── aatxe-core/ # the brain: types · stats · compare · report · affected · github URLs (pure) +│ ├── aatxe-ast/ # tree-sitter symbol/scope extraction for TS/Go/Rust (pure) │ ├── aatxe-council/ # MoA proposer→judge LLM PR-reviewer (pure) -│ ├── aatxe-evals/ # eval harness — scores council + stats end to end -│ └── aatxe/ # CLI binary + language adapters + reqwest GH client +│ ├── aatxe-learn/ # bounded, self-healing per-repo learning corpus (pure) +│ ├── aatxe-evals/ # eval harness — scores the council + stats engine end to end (pure) +│ ├── aatxe-ui/ # local realtime dashboard (axum + an embedded Svelte build) +│ └── aatxe/ # the CLI binary, organised internally as: +│ # commands/ subcommand impls adapter/ per-language bench runners +│ # llm/ council backends github/ ureq REST client + PR-diff fetch +│ # ast/ AST-scope + import glue cli.rs clap surface ├── sdk/ │ ├── ts/ # @aatxe/bench npm package (bench API + runner) │ ├── go/ # aatxe Go module (Bench + Suite) diff --git a/crates/aatxe/src/adapter/mod.rs b/crates/aatxe/src/adapter/mod.rs index 65dbc90..41f1a63 100644 --- a/crates/aatxe/src/adapter/mod.rs +++ b/crates/aatxe/src/adapter/mod.rs @@ -13,7 +13,7 @@ //! All three normalise the output through [`aatxe_core::stats::summarize_samples`] //! so downstream consumers see identical statistics regardless of language. -use crate::ast_import_extractor::AstImportExtractor; +use crate::ast::ast_import_extractor::AstImportExtractor; use aatxe_core::affected::{resolve_affected, AffectedOptions}; use aatxe_core::types::{AffectedScope as CoreScope, Language, RunReport}; use anyhow::{Context, Result}; diff --git a/crates/aatxe/src/ast_import_extractor.rs b/crates/aatxe/src/ast/ast_import_extractor.rs similarity index 100% rename from crates/aatxe/src/ast_import_extractor.rs rename to crates/aatxe/src/ast/ast_import_extractor.rs diff --git a/crates/aatxe/src/ast_scope.rs b/crates/aatxe/src/ast/ast_scope.rs similarity index 100% rename from crates/aatxe/src/ast_scope.rs rename to crates/aatxe/src/ast/ast_scope.rs diff --git a/crates/aatxe/src/ast/mod.rs b/crates/aatxe/src/ast/mod.rs new file mode 100644 index 0000000..731e633 --- /dev/null +++ b/crates/aatxe/src/ast/mod.rs @@ -0,0 +1,9 @@ +//! CLI glue over the `aatxe-ast` crate. +//! +//! * [`ast_scope`] — builds the structural-metadata block injected into +//! council proposer prompts. +//! * [`ast_import_extractor`] — tree-sitter-backed import graph feeding the +//! `--affected` resolver. + +pub mod ast_import_extractor; +pub mod ast_scope; diff --git a/crates/aatxe/src/commands/affected.rs b/crates/aatxe/src/commands/affected.rs index 9ad159b..0eecb53 100644 --- a/crates/aatxe/src/commands/affected.rs +++ b/crates/aatxe/src/commands/affected.rs @@ -1,7 +1,7 @@ //! `aatxe affected` — print the affected bench files for a given diff base. use crate::adapter::real_fs::{RealFs, RealGit}; -use crate::ast_import_extractor::AstImportExtractor; +use crate::ast::ast_import_extractor::AstImportExtractor; use crate::cli::AffectedArgs; use aatxe_core::affected::{resolve_affected, AffectedOptions}; use anyhow::Result; diff --git a/crates/aatxe/src/commands/comment.rs b/crates/aatxe/src/commands/comment.rs index 14fed50..a191727 100644 --- a/crates/aatxe/src/commands/comment.rs +++ b/crates/aatxe/src/commands/comment.rs @@ -1,7 +1,7 @@ //! `aatxe comment` — post / update the sticky PR comment. use crate::cli::CommentArgs; -use crate::github_http::UreqClient; +use crate::github::github_http::UreqClient; use aatxe_core::github::{detect_context, validate_sticky, GithubContext}; use aatxe_core::secret::Secret; use anyhow::{anyhow, Context, Result}; diff --git a/crates/aatxe/src/commands/council.rs b/crates/aatxe/src/commands/council.rs index 3800a5e..ae43403 100644 --- a/crates/aatxe/src/commands/council.rs +++ b/crates/aatxe/src/commands/council.rs @@ -2,24 +2,24 @@ //! render the sticky markdown body, and optionally post it. //! //! Wires together: -//! * [`crate::gh_diff::fetch_pr_diff`] — pulls the unified diff over +//! * [`crate::github::gh_diff::fetch_pr_diff`] — pulls the unified diff over //! `Accept: application/vnd.github.v3.diff`. -//! * [`crate::pi_proxy::PiAgentClient`] — spawns the local `pi` coding +//! * [`crate::llm::pi_proxy::PiAgentClient`] — spawns the local `pi` coding //! agent per LLM call so proposers can `read`/`grep`/`find`/`ls` the //! repo under review. Uses `KIMI_API_KEY` (forwarded to the child). //! * [`aatxe_council::pipeline::run_council`] — the proposer→judge //! pipeline lives in the pure crate. -//! * [`crate::github_http::UreqClient`] — same sticky-comment client the +//! * [`crate::github::github_http::UreqClient`] — same sticky-comment client the //! perf gate uses, with the council's own marker. -use crate::claude_code::{ClaudeCodeClient, ClaudeCodeConfig}; use crate::cli::{BackendArg, CouncilArgs}; use crate::commands::Outcome; -use crate::gemini_http::{self, GeminiClient, GeminiConfig}; -use crate::gh_diff::fetch_pr_diff; -use crate::github_http::UreqClient; -use crate::pi_proxy::{PiAgentClient, PiConfig}; -use crate::stub_client::{stub_enabled, StubKimi}; +use crate::github::gh_diff::fetch_pr_diff; +use crate::github::github_http::UreqClient; +use crate::llm::claude_code::{ClaudeCodeClient, ClaudeCodeConfig}; +use crate::llm::gemini_http::{self, GeminiClient, GeminiConfig}; +use crate::llm::pi_proxy::{PiAgentClient, PiConfig}; +use crate::llm::stub_client::{stub_enabled, StubKimi}; use aatxe_core::github::{detect_context, validate_sticky, GithubContext}; use aatxe_council::diff::parse_unified_diff; use aatxe_council::events::{CouncilEvent, EventSink, NullSink}; diff --git a/crates/aatxe/src/commands/evals.rs b/crates/aatxe/src/commands/evals.rs index b85c893..49ee759 100644 --- a/crates/aatxe/src/commands/evals.rs +++ b/crates/aatxe/src/commands/evals.rs @@ -2,8 +2,8 @@ //! //! 1. Run the stats eval (deterministic synthetic A/B pairs). //! 2. Run the council eval over every case in the corpus directory. -//! LLM client is the deterministic [`crate::stub_client::StubKimi`] by -//! default; `--council-real-llm` swaps in the [`crate::pi_proxy::PiAgentClient`] +//! LLM client is the deterministic [`crate::llm::stub_client::StubKimi`] by +//! default; `--council-real-llm` swaps in the [`crate::llm::pi_proxy::PiAgentClient`] //! (requires `KIMI_API_KEY` so the spawned `pi` child can reach //! Moonshot). //! 3. Serialise the result to JSON. @@ -11,12 +11,12 @@ //! 5. Optionally diff against a baseline; exit 2 on regression past //! tolerance. -use crate::claude_code::{ClaudeCodeClient, ClaudeCodeConfig}; use crate::cli::{BackendArg, EvalsArgs}; use crate::commands::Outcome; -use crate::gemini_http::{self, GeminiClient, GeminiConfig}; -use crate::pi_proxy::{PiAgentClient, PiConfig}; -use crate::stub_client::StubKimi; +use crate::llm::claude_code::{ClaudeCodeClient, ClaudeCodeConfig}; +use crate::llm::gemini_http::{self, GeminiClient, GeminiConfig}; +use crate::llm::pi_proxy::{PiAgentClient, PiConfig}; +use crate::llm::stub_client::StubKimi; use aatxe_council::llm::LlmClient; use aatxe_council::pipeline::{run_council_with_files, CouncilOptions}; use aatxe_evals::council::{ @@ -114,7 +114,8 @@ pub fn execute(args: EvalsArgs) -> Result { .into_iter() .map(|f| f.path) .collect(); - let ast_scope = crate::ast_scope::build_scope_for_review(&files_map, &changed_paths); + let ast_scope = + crate::ast::ast_scope::build_scope_for_review(&files_map, &changed_paths); eprintln!( " • case {} → {} ({} file fixtures, AST scope: {} bytes)", case.name, diff --git a/crates/aatxe/src/commands/learn.rs b/crates/aatxe/src/commands/learn.rs index 82698c3..b9cfc16 100644 --- a/crates/aatxe/src/commands/learn.rs +++ b/crates/aatxe/src/commands/learn.rs @@ -9,7 +9,7 @@ use crate::cli::{LearnArgs, LearnCommand, LearnCompactArgs, LearnHarvestArgs, LearnShowArgs}; use crate::commands::Outcome; -use crate::github_http::UreqClient; +use crate::github::github_http::UreqClient; use aatxe_core::github::{detect_context, GithubContext}; use aatxe_council::types::{CouncilReport, Severity}; use aatxe_learn::harvest::{ShippedFindingRef, DEFAULT_TRUSTED_ASSOCIATIONS}; diff --git a/crates/aatxe/src/gh_diff.rs b/crates/aatxe/src/github/gh_diff.rs similarity index 100% rename from crates/aatxe/src/gh_diff.rs rename to crates/aatxe/src/github/gh_diff.rs diff --git a/crates/aatxe/src/github_http.rs b/crates/aatxe/src/github/github_http.rs similarity index 100% rename from crates/aatxe/src/github_http.rs rename to crates/aatxe/src/github/github_http.rs diff --git a/crates/aatxe/src/github/mod.rs b/crates/aatxe/src/github/mod.rs new file mode 100644 index 0000000..2325da1 --- /dev/null +++ b/crates/aatxe/src/github/mod.rs @@ -0,0 +1,9 @@ +//! GitHub REST helpers used by the CLI (the pure URL/header logic lives in +//! `aatxe_core::github`). +//! +//! * [`github_http`] — `ureq`-backed client that posts/updates the sticky PR +//! comment and lists comments + reactions for the learning corpus. +//! * [`gh_diff`] — fetches a PR's unified diff (`Accept: vnd.github.v3.diff`). + +pub mod gh_diff; +pub mod github_http; diff --git a/crates/aatxe/src/claude_code.rs b/crates/aatxe/src/llm/claude_code.rs similarity index 99% rename from crates/aatxe/src/claude_code.rs rename to crates/aatxe/src/llm/claude_code.rs index 7cb9e88..107569e 100644 --- a/crates/aatxe/src/claude_code.rs +++ b/crates/aatxe/src/llm/claude_code.rs @@ -1,6 +1,6 @@ //! Claude Code proxy — `claude` CLI as an [`LlmClient`] backend. //! -//! Modelled byte-for-byte on [`crate::pi_proxy`]; the only meaningful +//! Modelled byte-for-byte on [`crate::llm::pi_proxy`]; the only meaningful //! differences are the spawn argv (Claude Code's `--print` surface) and //! the output parser (Claude Code emits `--output-format json` which //! carries the final assistant turn plus per-call usage tokens). @@ -44,7 +44,7 @@ //! occasionally wraps JSON answers in a markdown fence even when told //! not to) plus the usage tokens for cost telemetry. -use crate::subprocess_llm::{ +use crate::llm::subprocess_llm::{ join_with_blank_lines, partition_messages, sanitize_text_output, spawn_and_wait, }; use aatxe_council::llm::{ChatRequest, ChatResponse, LlmClient, LlmError}; @@ -300,7 +300,7 @@ struct ClaudeUsage { #[cfg(all(test, unix))] mod tests { use super::*; - use crate::subprocess_llm::test_fixture::{fake_binary, fake_sleeping_binary}; + use crate::llm::subprocess_llm::test_fixture::{fake_binary, fake_sleeping_binary}; use aatxe_council::llm::ChatMessage; use std::fs; diff --git a/crates/aatxe/src/gemini_http.rs b/crates/aatxe/src/llm/gemini_http.rs similarity index 99% rename from crates/aatxe/src/gemini_http.rs rename to crates/aatxe/src/llm/gemini_http.rs index b47a73e..1b11ae0 100644 --- a/crates/aatxe/src/gemini_http.rs +++ b/crates/aatxe/src/llm/gemini_http.rs @@ -7,7 +7,7 @@ //! and can `read`/`grep`/`glob` the repo under review. Gemini has no such //! agent binary, so this backend is a *direct* blocking HTTP client over //! [`ureq`] — the same dependency the sticky-comment poster -//! ([`crate::github_http`]) already uses. Gemini sees exactly the +//! ([`crate::github::github_http`]) already uses. Gemini sees exactly the //! pre-packed prompt the pipeline builds (diff + AST scope + related-file //! context) and nothing else; it has no repo tool access. That makes it //! the "pre-packed context, no tools" arm of the backend experiment — and @@ -39,7 +39,7 @@ //! a wrapping markdown fence from the content (`response_format` should //! prevent it, but models occasionally add one anyway). -use crate::subprocess_llm::sanitize_text_output; +use crate::llm::subprocess_llm::sanitize_text_output; use aatxe_council::llm::{ChatRequest, ChatResponse, LlmClient, LlmError, Role}; use serde::Deserialize; use std::env; diff --git a/crates/aatxe/src/llm/mod.rs b/crates/aatxe/src/llm/mod.rs new file mode 100644 index 0000000..b331cd6 --- /dev/null +++ b/crates/aatxe/src/llm/mod.rs @@ -0,0 +1,17 @@ +//! LLM backends for the agent council. +//! +//! Every backend produces the same `Finding[]` JSON the council pipeline +//! consumes; they differ only in transport and tool access: +//! +//! * [`pi_proxy`] / [`claude_code`] — shell out to a local agent CLI that +//! runs the model + a read-only repo tool loop. Both share the spawn, +//! timeout, and output-sanitisation plumbing in [`subprocess_llm`]. +//! * [`gemini_http`] — a direct blocking HTTP client; no tools, pre-packed +//! prompt only. +//! * [`stub_client`] — deterministic canned responses for offline tests + CI. + +pub mod claude_code; +pub mod gemini_http; +pub mod pi_proxy; +pub mod stub_client; +pub mod subprocess_llm; diff --git a/crates/aatxe/src/pi_proxy.rs b/crates/aatxe/src/llm/pi_proxy.rs similarity index 99% rename from crates/aatxe/src/pi_proxy.rs rename to crates/aatxe/src/llm/pi_proxy.rs index 590fee7..34c785a 100644 --- a/crates/aatxe/src/pi_proxy.rs +++ b/crates/aatxe/src/llm/pi_proxy.rs @@ -47,7 +47,7 @@ //! pipeline already parallelises proposers across personas, so wall-clock //! is roughly the slowest single agent call rather than 4×. -use crate::subprocess_llm::{ +use crate::llm::subprocess_llm::{ join_with_blank_lines, partition_messages, sanitize_text_output, spawn_and_wait, }; use aatxe_core::secret::Secret; @@ -218,7 +218,7 @@ impl LlmClient for PiAgentClient { #[cfg(all(test, unix))] mod tests { use super::*; - use crate::subprocess_llm::test_fixture::{fake_binary, fake_sleeping_binary}; + use crate::llm::subprocess_llm::test_fixture::{fake_binary, fake_sleeping_binary}; use aatxe_council::llm::ChatMessage; use std::fs; diff --git a/crates/aatxe/src/stub_client.rs b/crates/aatxe/src/llm/stub_client.rs similarity index 100% rename from crates/aatxe/src/stub_client.rs rename to crates/aatxe/src/llm/stub_client.rs diff --git a/crates/aatxe/src/subprocess_llm.rs b/crates/aatxe/src/llm/subprocess_llm.rs similarity index 99% rename from crates/aatxe/src/subprocess_llm.rs rename to crates/aatxe/src/llm/subprocess_llm.rs index 4be1621..e51dbdd 100644 --- a/crates/aatxe/src/subprocess_llm.rs +++ b/crates/aatxe/src/llm/subprocess_llm.rs @@ -1,6 +1,6 @@ //! Shared helpers for [`LlmClient`] backends that drive a child CLI. //! -//! Both [`crate::pi_proxy`] and [`crate::claude_code`] follow the same +//! Both [`crate::llm::pi_proxy`] and [`crate::llm::claude_code`] follow the same //! shape: build an argv, spawn the binary, stream the user payload on //! stdin, wait with a wall-clock deadline, then sanitise the model's //! stdout. The pieces shared between the two live here so a bug fix or diff --git a/crates/aatxe/src/main.rs b/crates/aatxe/src/main.rs index bed7466..1b11f48 100644 --- a/crates/aatxe/src/main.rs +++ b/crates/aatxe/src/main.rs @@ -6,18 +6,12 @@ //! * `2` — regressions detected and `--fail-on-regression` was passed. mod adapter; -mod ast_import_extractor; -mod ast_scope; -mod claude_code; +mod ast; mod cli; mod commands; mod curator; -mod gemini_http; -mod gh_diff; -mod github_http; -mod pi_proxy; -mod stub_client; -mod subprocess_llm; +mod github; +mod llm; use std::process::ExitCode;