diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index a14d797..f494a74 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -5,7 +5,8 @@ # uses: CMaintz/foundry/.github/workflows/java.yml@ # with: # spotbugs: true # opt-in bytecode analysis -# no_var: true # opt-in diff-scoped no-`var` rule +# +# no-`var` is not an input: it's folded into `lint` (mise.toml), enforced by the gate. # # Runs the same six verbs a developer runs locally (`mise run gate`). If this and # a laptop ever disagree, that's a bug in the setup, not the code. Toolchain (JDK) @@ -40,10 +41,6 @@ on: description: Gradle task for SpotBugs. type: string default: "spotbugsMain" - no_var: - description: Enforce the diff-scoped no-`var` rule on changed source (opt-in). - type: boolean - default: false osv_scanner_version: type: string default: "2.5.1" @@ -91,17 +88,24 @@ jobs: # gets a targeted fix. Devs still run the composite `mise run gate` locally. - name: 'gate: lint' id: lint + # FOUNDRY_BASE_REF is the ratchet base for lint's no-`var` check (see mise.toml): + # the PR base SHA on PRs (origin/main isn't a reliable ref in a PR checkout), + # empty on dispatch ⇒ novar falls back to origin/main. fetch-depth: 0 (above) + # supplies the history the merge-base needs. + env: + FOUNDRY_BASE_REF: ${{ github.event.pull_request.base.sha }} run: mise run lint - name: How to fix (lint) if: failure() && steps.lint.outcome == 'failure' run: | { - echo "## ❌ Gate failed at \`lint\` — formatting (Spotless)" + echo "## ❌ Gate failed at \`lint\` — formatting (Spotless) or no-\`var\`" echo "" - echo "This is **100% mechanical**. Run it, commit, push — done:" + echo "**Formatting** is 100% mechanical — run it, commit, push:" echo '```' echo "mise run fix # = ./gradlew spotlessApply" echo '```' + echo "**no-\`var\`** is not auto-fixable: replace \`var\` with the explicit type, or tag a justified use \`// foundry-allow-var: reason\`. It only flags \`var\` on files this PR changed." echo "(Structural smells are a separate 'Structural smells' job, not this one.) The offending files are in the **gate: lint** step log above." } >> "$GITHUB_STEP_SUMMARY" - name: 'gate: typecheck' @@ -231,50 +235,8 @@ jobs: - uses: gradle/actions/setup-gradle@94bac82a5b62952e304b4f7a45a90e19c46c7e50 # v4 - run: ./gradlew ${{ inputs.spotbugs_task }} - no-var: - name: No unjustified var - if: inputs.no_var && github.event_name == 'pull_request' - runs-on: ubuntu-latest - # Runs from the repo root: `git diff` prints root-relative paths, so pmd's -d - # must resolve from root too (a non-root working-directory would double the - # prefix). The project dir is applied as a prefix below instead. - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - fetch-depth: 0 - - name: Cache PMD - uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4 - with: - path: ~/.local/opt/pmd-bin-${{ inputs.pmd_version }} - key: pmd-${{ inputs.pmd_version }} - - name: Install PMD - run: | - if [ ! -x "$HOME/.local/opt/pmd-bin-${{ inputs.pmd_version }}/bin/pmd" ]; then - mkdir -p "$HOME/.local/opt" - curl -fsSL -o /tmp/pmd.zip \ - "https://github.com/pmd/pmd/releases/download/pmd_releases/${{ inputs.pmd_version }}/pmd-dist-${{ inputs.pmd_version }}-bin.zip" - unzip -q /tmp/pmd.zip -d "$HOME/.local/opt" - fi - echo "$HOME/.local/opt/pmd-bin-${{ inputs.pmd_version }}/bin" >> "$GITHUB_PATH" - # Diff from the merge-base (3-dot), not the raw base tip: pull_request events - # don't re-fire when main advances, so base.sha goes stale and a 2-dot diff - # would drag in files this PR never touched. Enforce no-`var` only on changed - # main source, so pre-existing uses are untouched until touched. - - name: Check changed Java files for unjustified var - env: - BASE: ${{ github.event.pull_request.base.sha }} - HEAD: ${{ github.event.pull_request.head.sha }} - WD: ${{ inputs.working_directory }} - run: | - set -euo pipefail - BASE=$(git merge-base "$BASE" "$HEAD") - prefix="" - if [ "$WD" != "." ]; then prefix="${WD%/}/"; fi - mapfile -t files < <(git diff --name-only --diff-filter=d "$BASE" "$HEAD" \ - | grep -E "^${prefix}src/main/java/.*\.java$" || true) - if [ "${#files[@]}" -eq 0 ]; then - echo "No main Java files changed — nothing to check."; exit 0 - fi - printf 'Checking %d changed file(s):\n' "${#files[@]}"; printf ' %s\n' "${files[@]}" - pmd check "${files[@]/#/-d=}" -R "${prefix}config/pmd/no-var.xml" \ - -f text --no-progress --suppress-marker foundry-allow-var + # no-`var` used to be a separate job here. It's now folded into `lint` (mise.toml), + # so the deterministic gate enforces it in one place — locally and in CI — and it's + # no longer opt-in. The gate job's checkout uses fetch-depth: 0, and it passes + # FOUNDRY_BASE_REF (the PR base SHA) so novar diff-scopes correctly. See + # designs/verb-tiers.md and designs/changed-scope-gate.md. diff --git a/CONTRACT.md b/CONTRACT.md index e8532dd..9fa187e 100644 --- a/CONTRACT.md +++ b/CONTRACT.md @@ -19,6 +19,10 @@ Every repo, regardless of language, exposes these six verbs: A repo with no meaningful work for a verb still defines it as a no-op that exits 0. Absence is not permitted — a caller must never have to ask whether a verb exists. +## Auxiliary tasks + +A repo may define additional `mise` tasks beyond the six (e.g. `spotbugs`, `novar`, `eval`, `setup:pmd`). These are **repo-local and not part of the contract**: no caller may assume they exist, and `gate` need not run them. They exist for work that is either not universal across repos or deliberately advisory (report-only). A capability graduates to a seventh contract verb only if it is **universal** — nearly every repo has real work for it — *and* cannot fit inside an existing verb. Until both hold, extend a verb's *composition* (what `lint`/`typecheck`/`test`/`audit` already run) or add an auxiliary task; do not grow the six. + ## Rules for callers **Callers invoke verbs, never tools.** A skill says `mise run lint`. It never says `eslint`, `phpcs`, or `ruff`. This is the whole reason a single skill library can serve a Kotlin repo and a React repo. diff --git a/designs/arch-fitness.md b/designs/arch-fitness.md new file mode 100644 index 0000000..6d1cea3 --- /dev/null +++ b/designs/arch-fitness.md @@ -0,0 +1,130 @@ +# Spec — Architecture fitness functions + +**Status:** In progress · **Owner:** CMaintz · **Date:** 2026-09-19 +**Home:** foundry (rule templates + presets + guard) · **Weight:** ⚖️ TS in-loop, JVM pre-push/CI + +**Implemented:** `presets/arch/dependency-cruiser.cjs` — generic, architecture-agnostic +layer+cycle rules generated from an editable `LAYERS`/`ALLOW` map (executed here: rule +generation verified for hexagonal + modular maps); `presets/arch/ArchitectureTest.java` +— ArchUnit template (layered + framework-freedom + cycles, wrapped in `FreezingArchRule`); +`presets/arch/guides/{layering-violation,dependency-cycle}.md`; `ruleset_guard.py` gains +a `lines` kind for the ArchUnit freeze store (tested; dep-cruiser baseline reuses `snooze`); +`mise/ts.toml` adds an opt-in `arch` task. **Not yet:** live ArchUnit/dep-cruiser run on a +real project (no JVM/node project here); Python `.importlinter` preset; foundry-init +vendoring of the arch presets. + +## Problem + +AutoApplicant's backend is textbook hexagonal — `adapter/ port/ usecase/ domain/ +config/` — and the docs *state* the rules (domain is framework-free, usecase must not +reach into adapter, no dependency cycles). Nothing *enforces* them. These are +module-graph-level structural properties, invisible to habit-hooks (which sees +function-level smells), and they're precisely what rots a documented architecture into +a big ball of mud one "just this once" import at a time. Neither ArchUnit nor +dependency-cruiser is present in AutoApplicant today — this is genuinely unbuilt. + +## Design + +Deterministic architecture rules, **folded into existing verbs** (tier-(b) verb +composition — no new contract verb; see the verb-tiers note): + +- **Java → ArchUnit**, expressed as JUnit tests → runs under `test`. +- **TS → dependency-cruiser** (cycles + forbidden cross-layer imports) → under `lint`. +- **Python → import-linter** → under `lint`. + +The placement asymmetry (arch-as-test on Java, arch-as-lint on TS) is invisible to +callers, who only ever call verbs — the verb interface earning its keep exactly as +CONTRACT.md intends. + +### Architecture-agnostic by construction + +The mechanism is **not hexagonal-specific** — hexagonal is one preset. A consumer +describes *their* architecture as a small model and every rule is generated from it: + +- **LAYERS** — a name → path/package pattern map (what code belongs to each layer). +- **ALLOW** — per layer, which other layers it may import; anything else is forbidden. +- **no cycles** — always enforced. +- **framework-freedom** — optional "layer X must not import package regex Y". + +foundry ships this as an editable preset per stack (`presets/arch/`), plus **coaching +guides** (`presets/arch/guides/{layering-violation,dependency-cycle}.md`) that render on +failure. The consumer fills the map; the layer *names* and edges are theirs. + +Four worked example maps ship in the presets (swap one in, or write your own): +- **Hexagonal / ports-and-adapters** (AutoApplicant): domain ← port ← usecase ← adapter/config; domain framework-free. +- **Classic layered (n-tier):** web → service → data, one direction only. +- **Clean / onion:** entities ← usecases ← interfaces ← frameworks. +- **Modular / feature-sliced:** each feature may import only a shared kernel, never another feature (module isolation). + +AutoApplicant's hexagonal rule set is just the default instance: +- `domain` imports no other layer (no Spring, no `adapter`, no `usecase`); `usecase` → + `domain`+`port`, never `adapter`; `adapter` implements `port`; no package cycles. + +## Per-stack wiring + +### Java (deep) +- Add an **ArchUnit test source set / package `…/arch/`** so the rules live in their + own place and are *always run* — changed-scope test selection (see changed-scope + spec) must never silently skip them (they're cheap and global). Wire into + `backend:test` (or a `backend:arch` auxiliary task that `test` depends on). +- Ratchet via `FreezingArchRule` backed by a committed violation store directory + (verify the exact store path/API at implementation — do not hard-code here). + +### TypeScript (deep) +- `.dependency-cruiser.js` with `forbidden` rules for cycles + layer boundaries, run in + `lint`. Known pre-existing violations recorded in dependency-cruiser's + known-violations baseline (verify the exact `--ignore-known` flag/file at impl). + +### Other stacks (recipe) +- **Python:** `.importlinter` contracts (layers/forbidden); run in `lint`. +- **Kotlin:** Konsist or ArchUnit-for-Kotlin, as `test`. **PHP:** deptry/phpat as + `lint`. **dotnet:** NetArchTest as `test`. + +## Ratchet mechanics + +- **Java:** `FreezingArchRule` violation store — existing violations frozen, store may + **only shrink**. Retrofits onto a dirty codebase without a red day-one wall. +- **TS:** dependency-cruiser known-violations baseline — same shrink-only property. +- Both are committed baseline files, same doctrine as eslint-suppressions/snooze. + +## ruleset-guard changes + +Two mechanisms, matching what each artifact is: + +| Artifact | Guard mechanism | Loosening (needs `ruleset-change`) | +|---|---|---| +| ArchUnit freeze store (`archunit_store/*.txt`) | **`lines` kind** (new; multiset of frozen-violation lines) — done | a store file gains a line | +| dep-cruiser known-violations (`.dependency-cruiser-known-violations.json`) | existing **`snooze`** kind (value_counts over the JSON) | a violation added | +| rule config (`.dependency-cruiser.cjs`, ArchUnit rule classes, `.importlinter`) | existing **ruleset-file watch** (any change + source ⇒ label) | `ALLOW` widened / a rule removed | + +The count classifiers catch *baseline* growth; the ruleset-file watch catches *rule* +weakening (widening `ALLOW`, deleting a rule), which isn't count-based — it rides the +existing "touched a ruleset file + source ⇒ needs a human label" control. + +## Placement + +| Placement | TS (dep-cruiser) | Java (ArchUnit) | +|---|---|---| +| In-loop | ✅ fast (parses source) | ❌ needs compiled classpath (JVM asymmetry, DESIGN §7.4) | +| Pre-push | ✅ | ✅ | +| CI | ✅ | ✅ | + +## If-funded tier + +None — fully deterministic. That's a *strength*: architecture enforcement never needs +a model, so it's pure oracle. + +## Acceptance criteria + +1. A PR making `domain` import Spring fails `test` (Java) / `lint` (TS). +2. An existing violation recorded in the frozen store does **not** fail the gate. +3. Removing a violation + pruning the store passes `ruleset-guard` **without** a label. +4. Adding a new forbidden cross-layer import fails, with a coaching guide printed. +5. Introducing a package cycle fails. + +## Backport split + +- **foundry:** rule templates, starter configs, coaching guides, the three + `ruleset_guard.py` classifiers, per-template verb wiring. +- **consumer:** the layer→folder map and which rules are enabled (AutoApplicant is the + reference; its hexagonal map ships as the worked example). diff --git a/designs/changed-scope-gate.md b/designs/changed-scope-gate.md new file mode 100644 index 0000000..913976a --- /dev/null +++ b/designs/changed-scope-gate.md @@ -0,0 +1,194 @@ +# Spec — Changed-scope gate (`FOUNDRY_SINCE`) + +**Status:** In progress · **Owner:** CMaintz · **Date:** 2026-09-19 +**Home:** foundry (per-`mise` template) · **Placement weight:** shift-left (local only) + +**Implemented:** `mise/ts.toml` — `lint` + `test` honour `FOUNDRY_SINCE` (validated +across unset / valid-ref / empty / unresolvable-ref); `presets/agent-loop.md` documents +the activation (fast scoped inner-loop runs, whole-tree final gate); `mise/java.toml` — +honest "Gradle already scopes it" note, `novar` folded into `lint` and made +ratchet-scoped via `FOUNDRY_BASE_REF`; reusable `java.yml` — separate `no-var` job + +`no_var` input removed, gate lint step passes `FOUNDRY_BASE_REF` (TOML/YAML/shell +validated). Pre-push hook and CI confirmed correctly whole-tree — unchanged. +**Remaining:** other-stack recipes (php/kotlin/dotnet/python). AutoApplicant inherits +the Java fold via its inline→foundry CI migration (separate work-stream). + +## Problem + +In-loop and pre-push both run the full gate over the *whole tree*. On a 300-word +edit the agent still pays for a whole-repo lint + the full test suite + a project-wide +type check. Slow feedback is the DX gap that remains once `/ship` and `/feature` +exist: fewer loop iterations per unit time, and a Stop hook that's slow enough to +tempt deferral to CI. The fix is not a different rule set — it's the *same* rules, +scoped to what changed, at the cheap placements only. + +## Two "scopes" — do not conflate them + +The instinct "quality checks should only look at changed files, everywhere" is right +for one meaning of scope and a **correctness bug** for the other. There are two: + +- **Ratchet-scope — *which findings fail the build*.** Only *new* violations fail; + legacy debt is accepted in a shrink-only baseline. This is safe and correct in + **every placement including CI**, and foundry **already does it everywhere**: + eslint-suppressions, `.habit-hooks/snooze.json`, Spotless `ratchetFrom origin/main`, + Semgrep `--baseline-commit`, the no-`var` PMD rule (CI checks only changed files). + So "point only at new work" — in the sense of *not failing on old debt* — is already + the whole design. +- **Execute-scope — *which files the checker even runs over*.** Restricting the tool + to the changed file set. Safe for **file-local** checks (a finding depends only on + that one file); **wrong for whole-program** checks, because a changed file can break + an *unchanged* one: + - Change a function signature in `A`; caller `B` (untouched) no longer compiles. + `tsc` run on `A` alone never sees it — type analysis is inherently whole-program. + - Add an import that forms a cycle; detecting it needs the whole module graph. + - Change a util; the test that exercises it lives in an unchanged file. + +**So the rule is:** ratchet-scope always (fail only on new); execute-scope only for +file-local checks, and only at the fast placements — with **whole-tree CI as the +authoritative backstop** for whole-program checks. `FOUNDRY_SINCE` is the +execute-scope signal for the fast placements; it is *not* set in CI, precisely so a +cross-file break a scoped local run misses is caught one push later. + +**Two scopes, two signals — don't reuse one for the other:** + +- **`FOUNDRY_SINCE`** — the *execute-scope* speed signal. Set at the fast placements + (in-loop, `/ship` pre-flight), **unset in CI**. Narrows which files a file-local verb + runs over. Optional; unset ⇒ whole-tree. +- **`FOUNDRY_BASE_REF`** — the *ratchet-scope* base for diff-scoped rules that must fail + only on *new* violations (e.g. Java `novar` folded into `lint`). Defaults to + `origin/main`; **CI sets it to the PR base SHA** because `origin/main` isn't a reliable + ref in a PR checkout. It is set *everywhere the rule runs, including CI* — that's the + difference from `FOUNDRY_SINCE`. An unresolvable base ⇒ the rule skips (CI's + full-history run is authoritative). + +The two are orthogonal: a diff-scoped rule can honour `FOUNDRY_BASE_REF` (which commits +count as "new") while its host verb is also execute-scoped by `FOUNDRY_SINCE` (which +files to bother running). Conflating them — e.g. using `FOUNDRY_SINCE` as the ratchet +base — would make a rule pass in CI (where it's unset) that fails locally. + +### Which checks are which — yes, this covers PMD, SpotBugs, everything + +| Check | Kind | Execute-scope safe? | How foundry scopes it | +|---|---|---|---| +| Formatting (prettier, Spotless) | file-local | ✅ | Spotless `ratchetFrom`; prettier on changed | +| Single-file lint, no-`var` (PMD) | file-local | ✅ | changed-file list (CI already does no-`var`) | +| habit-hooks smells (PMD/phpmd/eslint/…) | file-local | ✅ | in-loop `--branch`; snooze ratchet | +| SpotBugs / Semgrep (SAST) | file-local *findings* | ✅ (filter findings to changed) | Semgrep `--baseline-commit`; SpotBugs report filtered to diff | +| Secret scan (gitleaks) | **history, not diff** | ❌ | full history by design — a secret 5 commits back isn't in the diff | +| Type-check (tsc, compileJava) | whole-program | ❌ | whole-tree; ratchet N/A | +| Tests | whole-program (cross-file deps) | ⚠️ dep-graph only | `vitest --changed` follows the import graph locally; whole-suite in CI | +| Architecture (ArchUnit, dep-cruiser) | whole-program (graph) | ❌ | whole-graph; FreezingArchRule/known-violations ratchet | + +**PMD and SpotBugs specifically:** yes — PMD-backed smells are file-local and already +`--branch`-scoped in-loop and snooze-ratcheted; the no-`var` PMD rule is already +changed-file-scoped in CI. SpotBugs needs a full compile (so it can't cheaply +execute-scope), but its *findings* can and should be filtered to the changed set so it +only nags about new bugs. The taxonomy above is the general answer: **every checker +declares its kind, and scoping follows from the kind** — not an ad-hoc per-tool choice. + +### Is this what CI does today? + +For the **file-local** checks, yes — Spotless, no-`var`, Semgrep and habit-hooks are +all diff-scoped in CI already. For the **whole-program** checks (types, tests, and the +new arch rules) CI runs whole-tree **on purpose**, and should stay that way: that +whole-tree CI run is exactly the backstop that lets the fast local placements cut +corners safely. So "look at changed files and nothing else" is already true where it's +*safe*, and deliberately not true where it would miss cross-file breaks. + +## Design + +Introduce one scope signal: an environment variable **`FOUNDRY_SINCE=`** +(default when set but empty: merge-base with `origin/main`). Contract semantics: + +- A verb **MAY** honour `FOUNDRY_SINCE` by restricting its work to files changed + since that ref. A verb that cannot scope (e.g. project-wide type inference) simply + ignores it and runs whole-tree — correctness is never traded for speed. +- A **caller MUST NOT require** it. Unset = whole-tree, the current behaviour. This + keeps the "a caller never asks whether a verb exists / behaves specially" property. +- **CI never sets it.** Whole-tree CI stays the authoritative backstop, which is + exactly what makes local scoping safe: a scoped local run can miss a cross-file + break, and CI catches it one push later. + +The in-loop Stop hook and the git pre-push hook set `FOUNDRY_SINCE` (to the +branch merge-base); `/ship` sets it for its fast pre-flight and unsets it for the +final authoritative `gate`. This is a **tier-(b) change** (verb *composition*, +repo-local) — no new contract verb, no change to the six verb names. + +### Why an env var (not args or parallel tasks) + +- **Trailing args** (`mise run test -- --since`) force every task to pass args + through to a specific tool and leak tool syntax into callers. +- **Parallel tasks** (`test:changed`) double the task surface and make callers choose + which to invoke — breaking the "verbs are uniform across repos" contract. +- **Env var** is invisible to any caller that doesn't set it, opt-in per verb, and + costs zero contract growth. One signal, read where it helps, ignored where it can't. + +## Per-stack wiring + +### TypeScript (deep) +- `lint` → `eslint $(git diff --name-only --diff-filter=ACMR "$FOUNDRY_SINCE" -- '*.ts' '*.tsx')` + when set; else current glob. (eslint-suppressions ratchet unaffected.) +- `test` → `vitest --changed "$FOUNDRY_SINCE"` (vitest's built-in changed-since). +- `typecheck` → **non-scopeable, documented.** `tsc` is project-global; `tsc-files` + on changed files drops cross-file errors, so keep whole-tree. This is the honest + asymmetry — typecheck stays full even in-loop; it's fast enough on MDB-sized repos. + +### Java (deep) +- `lint` (Spotless) → **already changed-scoped** via `ratchetFrom origin/main`. No work. +- `typecheck` (`compileJava`) → Gradle incremental compilation already scopes this; + `FOUNDRY_SINCE` is a no-op. Document it. +- `test` → Gradle has no native changed-file test selection. Wrapper computes changed + test classes from the diff and runs `./gradlew :backend:test --tests `; + when the diff touches only non-test main classes, fall back to whole module (safe). + **Honest note:** on the JVM the build dominates and Gradle's up-to-date checks + already skip unaffected work, so the marginal win here is smaller than on TS. This + spec does not pretend otherwise. + +### Other stacks (recipe, 3–5 lines each) +- **PHP:** `phpstan analyse $(git diff …)`, `phpcs` on changed files; PHPUnit + `--filter` by changed test classes. +- **Python:** ruff is fast enough whole-tree (leave it); `pytest --testmon` or `-k` + for changed-scoped tests. +- **Kotlin/dotnet:** ktlint/dotnet-format on changed files; test filtering by + changed test class — same shape as Java, same JVM/build caveat. + +## Ratchet mechanics + +None — this is not a baseline feature. The safety property is instead: **whole-tree +CI is unconditional**, so scoping can only ever make local runs *faster*, never +weaker. That invariant is the spec's load-bearing guarantee. + +## ruleset-guard changes + +None. No new watched files. + +## Placement + +| Placement | `FOUNDRY_SINCE` | Scope | +|---|---|---| +| In-loop (Stop hook) | set = merge-base | changeset only (fast) | +| Pre-push (`/ship` pre-flight) | set = merge-base | changeset; final `gate` unset = whole-tree | +| CI | **never set** | whole-tree (authoritative) | + +## If-funded tier + +N/A — pure local optimisation. + +## Acceptance criteria + +1. With `FOUNDRY_SINCE` set, `mise run lint`/`test` operate only on changed files; + a timing measurement shows a scoped run materially faster than whole-tree on a + one-file change. +2. With `FOUNDRY_SINCE` unset (CI), behaviour is byte-identical to today. +3. A deliberately induced cross-file type break is *missed* by a scoped local run but + *caught* by whole-tree CI — proving the backstop and documenting the asymmetry. +4. Non-scopeable verbs (`typecheck` on TS, `compileJava`) are documented as such and + run whole-tree regardless. + +## Backport split + +- **foundry:** the `FOUNDRY_SINCE` convention, the merge-base diff snippet, and the + per-template wiring for each stack. A one-paragraph note in CONTRACT.md defining the + MAY/MUST-NOT semantics. +- **consumer:** repo-specific test-selection quirks (e.g. AutoApplicant's + changed-test-class mapping) if they exceed the shared snippet. diff --git a/designs/flake-triager.md b/designs/flake-triager.md new file mode 100644 index 0000000..9a1023e --- /dev/null +++ b/designs/flake-triager.md @@ -0,0 +1,103 @@ +# Spec — Deterministic flake triager + +**Status:** In progress · **Owner:** CMaintz · **Date:** 2026-09-19 +**Home:** foundry (wrapper + baseline format + guard) · **Weight:** pre-push/CI + +**Implemented & tested (in isolation):** `scripts/foundry-flaky` — `gate` mode +post-processes a test report (JUnit XML *and* vitest JSON) against `flaky-baseline.json` +and fails only if a NON-quarantined test failed (quarantined flakes are printed, never +hidden); `classify` mode reruns failed tests N times to separate flakes (flip) from real +failures (always fail) and surfaces prune candidates (quarantined-but-now-stable). Both +modes tested across formats + quarantined/real/no-baseline cases. `presets/flaky-baseline.example.json` +(schema); baseline reuses the guard's `snooze` kind — verified add=loosening (needs +label), remove=tightening (free), no new guard code. `mise/ts.toml` gains an opt-in +`test:flaky` integration task. **Not yet:** live rerun on a real runner (no JVM/node +project here — only the parsing/decision/classify logic is validated); Gradle `--tests` +rerun recipe wiring; auto-prune of stable entries (needs a green-streak counter — a +follow-on that can piggyback the loop-telemetry log). + +## Problem + +DESIGN §4.2 names three probabilistic roles; **Fixer and Reviewer are built, Triager +isn't.** A flaky test that fails the gate is worse than no test: it trains everyone — +human and agent — to ignore red, and it burns loop iterations chasing a ghost. The +deterministic 80% of triage (is this failure reproducible?) needs no model at all. + +## Design + +On a `test` failure, a wrapper does deterministic-first triage: + +1. **Rerun only the failed tests, N times** (via runner filters — not the whole suite). +2. A test that **flips** (passes on rerun) is a flake candidate → recorded in a + committed `flaky-baseline.json` (test id, first-seen ref, rerun evidence). +3. **Quarantined tests still run and still report, but don't gate.** This is done by + **post-processing the runner's report** against the baseline in the wrapper — *not* + by runner plugins or `@Disabled`. Don't fight the runner: let it run everything, + then the wrapper computes the gating exit code = fail iff a *non-quarantined* test + failed. +4. A quarantined test that passes **M consecutive** runs is **auto-pruned** (re-armed). + +This keeps the oracle honest: a real, 100%-reproducible failure still fails the gate; +only genuinely non-deterministic tests are set aside, and only visibly. + +## Per-stack wiring + +### Java (deep) +- Parse `build/test-results/**/*.xml` (JUnit XML) → failed test ids. +- Rerun: `./gradlew :backend:test --tests ` × N. +- Gating decision: wrapper re-reads the XML, subtracts quarantined ids, sets exit. + +### TypeScript (deep) +- vitest JSON reporter (`--reporter=json`) → failed test names. +- Rerun: `vitest run -t ""` × N. (Use explicit reruns for *classification* — + not vitest `--retry`, which silently hides flakes instead of recording them.) +- Same post-processing of the JSON report against the baseline. + +### Other stacks (recipe) +- **PHP:** parse PHPUnit JUnit XML; rerun `--filter`. **Python:** pytest JUnit XML / + `pytest-json-report`; rerun `-k`. **Kotlin/dotnet:** JUnit-XML / TRX, same shape. + +## Ratchet mechanics + +`flaky-baseline.json` **only shrinks**: entries are auto-pruned on sustained green, and +adding an entry is a loosening. Never regenerated wholesale to make a build pass. + +## ruleset-guard changes + +Watch `flaky-baseline.json`. Classifier: + +| Change | Verdict | +|---|---| +| Entries only removed (auto-prune) | tightening — no label | +| Entry added (new quarantine) | loosening — needs `ruleset-change` (or a dedicated `flaky-quarantine`) label | + +"This test is flaky, trust me" *should* be a deliberate, labelled human decision — the +guard makes it one. + +## Placement + +| Placement | Runs? | Why | +|---|---|---| +| In-loop | ❌ | rerun cost too high for the <30s Stop budget | +| Pre-push | ✅ | catches flakes before they hit CI | +| CI | ✅ | authoritative; quarantine keeps a known flake from blocking merge | + +## If-funded tier + +**Model Triager:** read stack traces to classify flaky-vs-real, dedupe repeat +failures, and auto-file a ticket that `/feature` can later pick up. Advisory only — +the deterministic rerun decides quarantine; the model just routes and explains. + +## Acceptance criteria + +1. A test injected to fail ~50% of the time is quarantined after N reruns and stops + gating; it still appears in the report. +2. A test that fails 100% still fails the gate (never quarantined). +3. A quarantined test made stable is auto-pruned after M green runs. +4. Adding a quarantine entry without a label fails `ruleset-guard`; removing one passes. + +## Backport split + +- **foundry:** the rerun-and-classify wrapper, the baseline JSON format, per-stack + report parsers, the `ruleset_guard.py` classifier. +- **consumer:** the baseline file contents (its own known flakes). diff --git a/designs/loop-telemetry.md b/designs/loop-telemetry.md new file mode 100644 index 0000000..8515f25 --- /dev/null +++ b/designs/loop-telemetry.md @@ -0,0 +1,103 @@ +# Spec — Local loop telemetry + +**Status:** In progress · **Owner:** CMaintz · **Date:** 2026-09-19 +**Home:** foundry (schema + wrapper) + cmaintz-skills (summarizer) · **Weight:** local-first + +**Implemented & tested (in isolation):** `scripts/foundry-verb-wrap` — times a verb, +appends one JSONL line (`ts, ms, verb, scope, placement, exit, duration_ms`), always +propagates the verb's exit (telemetry failure can't change the outcome; validated across +success/fail/unwritable-path). `scripts/foundry-loop-report` — offline summariser: per-verb +time/fails/**regressions (thrash)**, time-sink, latest-red, placement/scope breakdown; +handles missing/empty/torn logs. Wired into `mise/ts.toml` + `mise/java.toml` (gate wraps +each verb; `FOUNDRY_TELEMETRY` + `scripts/` on PATH via `{{config_root}}`; array form kept +so the CONTRACT's sequential-not-`depends` rule holds). `foundry-init` vendors both scripts ++ gitignores `.foundry/`. `/loop-report` skill added to cmaintz-skills. +**Not yet:** end-to-end run on a live `mise` project (only the scripts are validated here — +no mise runtime in this env); smell-count emission from the habit-hooks Stop hook (the +smell-trend convergence signal — a follow-on); `gh run` CI mining lives in the skill. + +## Problem + +"Why did this run fail, is the loop converging or thrashing, where does the wall-clock +go, how many gate rounds to green" is invisible today. Thrash — the agent fixes the +in-loop-flagged smell, CI flags another, it regresses the first — is *felt*, never +*measured*. The `mise` verbs are the single chokepoint every placement runs through, +so the cheapest place to make the loop legible is right there. + +## Design + +**Emit:** a small portable-sh wrapper `foundry-verb-wrap -- ` that each +of the six top-level verbs in a `mise` template is prefixed with. It runs the command, +times it, captures the exit code, and appends **one JSONL line** to a gitignored +`.foundry/telemetry.jsonl`: + +```json +{"ts":"","verb":"lint","scope":"changed|whole","exit":0, + "duration_ms":812,"placement":"hook|ship|ci|manual","smells":{"high-complexity":3}} +``` + +- **One wrapper, not per-task edits.** Wrapping only the six verb entry points (three + extra characters of prefix per template) captures every invocation — direct, + composed-via-`gate`, hook-driven — without touching leaf tasks. `smells` is + populated by parsing habit-hooks' normalized JSON when it ran; omitted otherwise. +- **Placement** is inferred from env (`CI`, a hook-set `FOUNDRY_PLACEMENT`, else + `manual`). `scope` reads whether `FOUNDRY_SINCE` was set (ties to the changed-scope + spec). + +**Summarize:** a `/loop-report` skill (cmaintz-skills) reads the JSONL and answers: +- **Convergence vs thrash** — is the per-smell count series monotonically falling, or + oscillating? A smell that returns after being cleared is a thrash signal. This + operationalises the "don't spin past two no-progress rounds" rule in `agent-loop.md`. +- **Time sinks** — wall-clock per verb, cold vs warm. +- **Rounds to green** — how many `gate` invocations from first-run to green. +- **CI mining (extension)** — `gh run list --json …` for CI job outcomes/durations + (no repo instrumentation needed; answers "which job is the tall pole"). + +## Per-stack wiring + +Schema, wrapper, and skill are **language-agnostic** (foundry / cmaintz-skills). Every +template just prefixes its verbs with the wrapper — identical for TS and Java. `smells` +parsing rides on habit-hooks' already-normalized output, so it's stack-independent. +Other stacks: same prefix, no per-stack code. + +## Ratchet mechanics + +None — telemetry is local observability, `.foundry/` is gitignored, nothing committed. +(Deliberately not a baseline: committing run traces would be noise and churn.) + +## ruleset-guard changes + +None (gitignored, non-source). + +## Placement + +| Placement | Emits? | Summarizer | +|---|---|---| +| In-loop | yes (via wrapper) | — | +| Pre-push | yes | `/loop-report` after `/ship` | +| CI | yes (local file discarded) + `gh run` mining | `/loop-report --ci` | + +## If-funded tier + +- **Token/cost accounting:** if a metered API is ever in the loop, log token counts per + verb/round and correlate spend against convergence — "this refactor cost 40k tokens + and 6 rounds." Aspirational; no spend today. +- **Hosted dashboard:** ship the JSONL to a sink for cross-session trends. Not needed + while single-maintainer; the skill answers the same questions locally. + +## Acceptance criteria + +1. After a `/ship` run, `.foundry/telemetry.jsonl` has one line per verb invocation + with a plausible `duration_ms` and correct `exit`. +2. `/loop-report` prints rounds-to-green and per-verb wall-clock for a real session. +3. When a fix→regress thrash is *induced* (clear a smell, reintroduce it), the report + flags it as non-converging. +4. `/loop-report --ci` lists recent CI job durations via `gh run` with zero repo + instrumentation. + +## Backport split + +- **foundry:** JSONL schema, `foundry-verb-wrap`, per-template prefix wiring, + `.gitignore` entry. +- **cmaintz-skills:** the `/loop-report` summarizer skill. +- **consumer:** nothing beyond adopting the wrapper prefix (comes with the template). diff --git a/designs/prompt-eval.md b/designs/prompt-eval.md new file mode 100644 index 0000000..48ac4c3 --- /dev/null +++ b/designs/prompt-eval.md @@ -0,0 +1,117 @@ +# Spec — Prompt / agent-output regression eval + +**Status:** In progress · **Owner:** CMaintz · **Date:** 2026-09-19 +**Home:** consumer (AutoApplicant) → generalize to foundry · **Weight:** local; CI replays only + +**Implemented:** `presets/prompt-eval/` — a reusable harness generalising AutoApplicant's +`PromptEvalHarnessTest`: `prompt-eval.test.ts` (vitest) + `PromptEvalHarnessTest.java` +(JUnit `@TestFactory`) apply the **ranking** assertions (good ≥ FLOOR, weak ≤ CEILING, +gap ≥ SEPARATION) + optional prompt-block checks against a *consumer-supplied* scorer +(foundry ships no scorer). `fixtures/example.json` + `manifest.json` (the ratcheted +surface). `ruleset_guard.py` gains a **`coverage` kind** — INVERSE of a debt baseline: +removing a fixture = loosening (needs label), adding = free; tested (comment edits don't +false-flag). Ranking logic verified with a toy scorer. Runs under `test` (fixtures are +tests — no verb change). **Scope note:** narrowest of the five — only AI-output repos need +it; this is a clean backport of AA's proven pattern so future AI repos inherit it. +**Not yet:** the consumer half (AA keeps its `DocumentQualityEvaluator` + real fixtures; +wiring the generalised template back into AA is follow-on); the if-funded LLM-judge tier +(documented below, needs an API key — not built, no spend). + +## Problem + +Prompt or output-shaping changes can silently regress quality — the one class of +change the deterministic gate is blind to, because the output is probabilistic. +AutoApplicant **already solves this offline and deterministically**: +`PromptEvalHarnessTest` scores fixtures with `DocumentQualityEvaluator` (a +model-free scorer) and asserts on the *ranking* of good vs weak samples, plus asserts +the composed prompt carries the blocks it must. The task is to **generalize the +pattern into foundry** — not rewrite the scorer, which is domain-specific and stays in +the consumer. + +## Design + +The generalizable contract, distilled from what the real harness asserts: + +- **Fixtures** — JSON `(input context, good sample, weak sample)` in a resources dir. +- **A repo-provided deterministic scorer** — stays in the consumer + (`DocumentQualityEvaluator` is AutoApplicant's, not foundry's). +- **Ranking assertions** (the trustworthy signal): `good ≥ FLOOR`, `weak ≤ CEILING`, + and `good − weak ≥ MIN_SEPARATION`. Absolute cutoffs on hand-written samples are + arbitrary; *separation* is what makes a scorer trustworthy as a regression metric. +- **Structural prompt assertions** — the composed prompt contains required blocks + (e.g. language directive, banned-phrases section, market conventions). Catches a + prompt regression at composition time, not weeks later in a bad output. +- **Runs offline, as ordinary tests → under `test`.** No new contract verb: prompt-eval + fails the universality test (only AI-output repos have this work), so it lives in + `test` for the deterministic tier, with an optional auxiliary `eval` task (tier-(c)) + for the funded LLM-judge tier. + +foundry ships: the fixture schema, harness templates (JUnit `@TestFactory` for Java, +`describe.each`/`it.each` for TS-vitest), and the floor/ceiling/separation assertion +helpers. The scorer and fixtures are the consumer's. + +## Per-stack wiring + +### Java (deep) +- Generalize AutoApplicant's `PromptEvalHarnessTest` shape into a foundry template: a + `@TestFactory` that loads fixtures from `src/test/resources/prompt-eval/` and applies + the ranking + structural assertions against a repo-supplied `QualityScorer` interface. + Already under `backend:test` — zero verb change. + +### TypeScript (deep) +- vitest template: `it.each(fixtures)` applying the same ranking assertions against a + repo-supplied scorer function; fixtures as JSON alongside the test. Under `test`. + +### Other stacks (recipe) +- **Python:** pytest `@pytest.mark.parametrize` over fixture files; scorer is a repo + function. **PHP:** PHPUnit data provider. Same three assertions, same fixture schema. + +## Ratchet mechanics + +**Ratchet the fixture *count*, not the scores.** Scores are not monotone, and +ratcheting them upward invites overfitting to the scorer — the exact game-the-proxy +failure `agent-loop.md` warns against. Instead: the **fixture set may only grow** (you +never delete a regression case), and the FLOOR/CEILING/SEPARATION thresholds are +*ruleset* (guarded), not a ratchet. + +## ruleset-guard changes + +Watch the fixtures dir + the threshold constants. Classifier: + +| Change | Verdict | +|---|---| +| Fixture added | tightening (more coverage) — no label | +| Fixture removed | loosening — needs label | +| `FLOOR`↓ / `CEILING`↑ / `SEPARATION`↓ | loosening — needs label | + +## Placement + +| Placement | Deterministic tier | LLM-judge tier | +|---|---|---| +| In-loop | ✅ (offline, just tests) | — | +| Pre-push | ✅ | (if funded) advisory PR comment | +| CI | ✅ replays recorded fixtures + re-runs assertions | ❌ never calls a model | + +## If-funded tier + +- **LLM-judge on real generations:** run the app's generation path, feed the output to + *both* the deterministic scorer and a judge model; post a score-delta comment. + Reviewer-class — **advisory, never blocks** (DESIGN §4.2). +- **Fixture mining:** use the judge to propose new fixtures from production near-misses, + growing the (ratcheted) fixture set. +- No spend today; the deterministic tier is the whole real-world deliverable. + +## Acceptance criteria + +1. A prompt change that drops a required block fails `test` at composition time. +2. A scorer that can't separate good from weak fails the separation assertion. +3. Adding a fixture passes `ruleset-guard`; lowering `FLOOR` needs a label. +4. The generalized template runs in a TS/vitest repo as well as in AutoApplicant/JUnit, + proving it's a foundry pattern and not an AutoApplicant one-off. + +## Backport split + +- **foundry:** fixture schema, JUnit + vitest harness templates, assertion helpers, the + `ruleset_guard.py` classifier, the `QualityScorer` interface shape. +- **consumer:** the scorer implementation (`DocumentQualityEvaluator`), the fixtures, + and the threshold values. diff --git a/designs/tooling-brainstorm.md b/designs/tooling-brainstorm.md new file mode 100644 index 0000000..69f106a --- /dev/null +++ b/designs/tooling-brainstorm.md @@ -0,0 +1,140 @@ +# Foundry — dev-tooling / eval / observability / DX brainstorm + +**Status:** Brainstorm (agreed to spec all five) · **Owner:** CMaintz · **Date:** 2026-09-19 + +High-leverage additions across dev tooling, evaluation, observability, developer +experience, and CI/CD. Five ideas, filtered against what already exists, then a +ranked value/effort shortlist. + +> **Constraints note.** The original brainstorm was written under three constraints +> that have since been **relaxed** for the spec phase: +> - *Six frozen verbs* → new verbs are now allowed where they earn their place; the +> existing structure may be edited to improve it. +> - *No API key in CI* → still the real-world default (no paid API/product spend), +> but each idea may explore an optional "if-funded" tier as an aspirational note. +> - *Actions-minutes scarcity* → no longer a design constraint. Optimise for +> end-product quality and how well the system produces good code, not minutes. +> +> The redundancy analysis below still holds and still governs the specs. + +## Not proposed — already covered + +Coupling/hotspot → `hotspot-rec`. Duplication → `jscpd`. Auto-fix on PR → +`autofix.yml`. Baseline-movement reporting → `ratchet-report.yml`. Ticket→PR → +`/feature`. Agent-done → green orchestration → `/ship` + `/feature` + the `gate-ok` +aggregate pattern (OVERVIEW §13). New work extends these where there's a real gap; +it does not re-skin them. + +--- + +## 1. Architecture fitness functions + +**Problem.** DESIGN.md and the CONTEXT/domain docs *describe* layering +(adapter→usecase, domain→framework-free) and forbid cycles, but nothing *enforces* +it. Layering violations and dependency cycles are structural at the module-graph +level — invisible to habit-hooks, which sees function-level smells — and they're the +ones that quietly rot a documented architecture into a big ball of mud. + +**Fit.** The deterministic-oracle thesis applied to architecture: turn a documented +rule into a pass/fail exit code, per stack. +- TS → `dependency-cruiser` (or `madge` for cycles-only). +- Java → **ArchUnit**, which runs *as JUnit tests*. +- Python → `import-linter`. + +**Ratchet, day one.** ArchUnit's `FreezingArchRule` (violation store that only +shrinks) and dependency-cruiser's `--ignore-known` baseline both retrofit without a +red day-one wall and only tighten. Rule configs join `ruleset-guard`'s watched set — +arch rules are the first thing an agent loosens to make a violation pass. + +**Effort.** Medium. TS path is fast (parses source, good in-loop). Java path needs a +compiled classpath → CI/pre-push only, in-loop skipped (same JVM-latency asymmetry as +DESIGN §7.4). + +--- + +## 2. Local loop telemetry — make the agent loop legible + +**Problem.** "Why did this run fail, is the loop converging, where do time/tokens/CI +minutes go" is invisible. Thrash is felt, not measured — including the agent failure +mode of fix-flagged / CI-flags-other / regress-first, which is a convergence problem. + +**Fit.** The `mise` verbs are the single chokepoint every placement runs through, so +instrument there: each verb appends one JSONL line (`{verb, exit, duration_ms, +smell_counts, ts}`) to a gitignored `.foundry/telemetry.jsonl`. A summarizer skill +answers: is smell-count monotonically falling (converging) or oscillating +(thrashing)? Which verb eats wall-clock? How many gate rounds to green? This turns +the "don't spin past two no-progress rounds" prose in `agent-loop.md` into something +measurable. Same summarizer can mine `gh run list --json` (free of minutes) for CI +outcomes/durations → one skill answers "is my loop converging" and "where do my CI +minutes go". + +**Effort.** Low–Medium. Emit is trivial; value is in the summarizer's questions. + +--- + +## 3. Prompt / agent-output regression eval + +**Problem.** jobbuddy has an offline prompt-eval harness that dies in that repo. When +a prompt or output-shaping change lands, nothing catches a silent quality +regression — the one class of change the deterministic gate is blind to. + +**Fit (split per DESIGN §4).** +- **Deterministic assertions gate** — valid JSON / schema match / required fields / + regex. Reproducible, no model. Maps into `test` (or a dedicated verb — see specs). +- **LLM-judged scoring is Reviewer-class — advisory, never blocking.** Runs + local/pre-push (API key); emits a score delta as a PR comment. CI replays recorded + fixtures and re-runs the deterministic assertions only. +- **Ratchet:** a committed score baseline that may only rise; joins `ruleset-guard`. + +**Effort.** Medium–High; most consumer-specific. Worth it because jobbuddy proved the +shape — the work is generalizing, not inventing. + +--- + +## 4. Changed-scope gate — make the loop fast + +**Problem.** `/ship` + `/feature` + `gate-ok` already close most of the agent-done → +green gap. What's left is **gate speed**: in-loop/pre-push run the full gate over the +whole tree, so the agent waits and does fewer iterations per unit time. Slow feedback +*is* the DX gap now. + +**Fit.** The *same* rule set, scoped by placement: a `--since ` scope so +in-loop/pre-push run lint+typecheck+test over the changeset (merge-base diff) while CI +stays whole-tree authoritative. OVERVIEW §12 already does this once (in-loop scope is +`--branch`); generalize it into every `mise` template. + +**Effort.** Low–Medium per stack; compounds on every future loop iteration. + +--- + +## 5. Deterministic flake triager + +**Problem.** DESIGN §4.2 names three probabilistic roles; Fixer and Reviewer are +built, **Triager isn't.** A flaky test failing the gate is worse than no test — it +trains everyone to ignore red and burns iterations chasing a ghost. + +**Fit.** Do the deterministic 80% without a model: on a `test` failure, re-run the +failed tests N times; a test that flips is quarantined into a committed +`flaky-baseline.json` (ratchet: only shrinks; a 20-green quarantined test is pruned +and re-armed). Quarantined tests still run and report but don't gate. The baseline +joins `ruleset-guard` — adding a flaky exemption is a deliberate, labelled decision. +Model Triager (dedupe, route-to-ticket) is a later advisory add. + +**Effort.** Medium. Re-run-and-classify is a wrapper around `mise run test`; per-stack +test selection reuses #4's plumbing. + +--- + +## Shortlist — ranked by value/effort + +| # | Idea | Value/Effort | Home | Shift-left? | +|---|---|---|---|---| +| **1** | **Changed-scope gate** (`--since`) | Highest — compounds on every loop | foundry (per-`mise` template) | ✅ pure local | +| **2** | **Local loop telemetry** + `gh run` mining | High — observability, feeds convergence | foundry (schema+skill); consumer emits | ✅ local-first | +| **3** | **Architecture fitness functions** | High — flagship document→enforce | foundry (presets+wiring); consumer picks layers | ⚖️ TS in-loop, JVM pre-push/CI | +| **4** | **Deterministic flake triager** | Medium — fills a named gap | foundry (wrapper); consumer owns baseline | ✅ pre-push | +| **5** | **Prompt/agent-output eval** | Medium — biggest new capability, most consumer-specific | consumer (jobbuddy) → generalize to foundry | ✅ local; CI replays only | + +**Sequencing:** #1 and #2 first (fast + legible loop is the real DX gap once `/ship` +and `/feature` exist). #3 designed now (layer names are the hard part, already +half-written in the docs), built next. #4 and #5 follow. diff --git a/designs/verb-tiers.md b/designs/verb-tiers.md new file mode 100644 index 0000000..d8b250b --- /dev/null +++ b/designs/verb-tiers.md @@ -0,0 +1,70 @@ +# Spec — Verb tiers & the six-verb decision + +**Status:** Agreed (keep six + CONTRACT addendum) · **Owner:** CMaintz · **Date:** 2026-09-19 +**Home:** foundry + cmaintz-skills (CONTRACT.md, kept identical) + +## The decision + +Keep the **six contract verbs frozen**; extend functionality *below* the contract, not +by growing it. The number matters as a *contract-stability* property — every caller may +assume exactly `fix/lint/typecheck/test/audit/gate` exist in every repo and never has to +ask — not as a limit on what the system can check. + +## Three tiers + +| Tier | What | Frozen? | New work goes here when… | +|---|---|---|---| +| **(a) Contract verbs** | the six | **yes** — a change is a major bump on both repos | (never grows in practice) | +| **(b) Verb composition** | what `lint`/`typecheck`/`test`/`audit`/`gate` *run* | no — repo-local | the concern fits an existing verb's meaning (arch → `lint`/`test`; flake → inside `test`) | +| **(c) Auxiliary tasks** | extra `mise` tasks (`novar`, `spotbugs`, `eval`, `setup:pmd`) | no — repo-local, non-contract | the concern isn't universal, or is deliberately advisory/report-only | + +**The test a concern must pass to earn a seventh contract verb:** it must be +(1) *universal* — nearly every repo has real work for it — **and** (2) genuinely unable +to fit inside an existing verb. None of the five brainstorm features passes: arch fits +`lint`/`test`; prompt-eval isn't universal (only AI repos) and its deterministic tier is +already tests; flake is a `test` wrapper. The cost of a seventh contract verb is paid by +*all* repos (each must define it, even as a no-op) to serve a concern only *some* have. + +Codified as the **Auxiliary tasks** section now added to CONTRACT.md (both copies). + +## Disposition of AutoApplicant's current auxiliary tasks + +Answering "should any of these hook into an existing verb?" — case by case: + +- **`novar` → folded into `lint` (tier b). Done in foundry.** It *is* a lint rule; it + lived outside `lint` only because whole-tree locally would fail on legacy `var`. It is + **ratchet-scoped, not execute-scoped** — it diffs from `FOUNDRY_BASE_REF` (the ratchet + base, default `origin/main`; CI sets the PR base SHA), *not* `FOUNDRY_SINCE` (the speed + signal). So folding it into `lint` never fails on legacy `var` in any placement. + Implemented in `mise/java.toml` (`lint = [spotlessCheck, novar]`) and the reusable + `java.yml` (the separate `no-var` job and `no_var` input removed; the gate's lint step + passes `FOUNDRY_BASE_REF`). A consumer inherits it by calling foundry's `java.yml` and + folding `novar` into its own `mise.toml` `lint`; **AutoApplicant picks this up via its + inline→foundry CI migration, a separate work-stream — not edited here.** +- **`spotbugs` → in AutoApplicant it is *already* blocking + ratcheted; its cleanup is + folding into `audit`.** Correction to an earlier draft: AA does not run SpotBugs + report-only. `quality.yml` runs `mise run backend:spotbugs` as a **blocking** job with + a `config/spotbugs/exclude.xml` ratchet ("the current tree is clean"). So the general + claim "report-only until ratcheted" is the right *default for a fresh repo*, but AA has + already done the ratchet+tune. What remains for AA is purely a verb-tier tidy: fold + `backend:spotbugs` into `backend:audit` (its semantic home — bug-pattern SAST) so it + rides the gate composite instead of a standalone job. foundry's `java.yml` template + keeps SpotBugs opt-in/report-only as the safe default for repos that haven't ratcheted. +- **`setup:pmd` → stays auxiliary (tier c).** It's provisioning (a `postinstall` hook), + not a check. Never a verb member. **Keep as-is.** + +## Acceptance + +1. CONTRACT.md (both repos, identical) carries the Auxiliary-tasks section. +2. foundry `mise/java.toml` `lint` runs `novar` (ratchet-scoped via `FOUNDRY_BASE_REF`); + the reusable `java.yml` no longer has a separate `no-var` job or `no_var` input, and + its gate lint step passes `FOUNDRY_BASE_REF`. ✓ done. +3. `setup:pmd` stays auxiliary; `gate` does not run it. `spotbugs` stays opt-in in the + foundry template; in AA it's blocking and slated to fold into `audit`. + +## Backport split + +- **foundry / cmaintz-skills:** the CONTRACT.md addendum (identical); the `novar`→`lint` + fold in `mise/java.toml` + `java.yml`; `FOUNDRY_BASE_REF` convention. +- **consumer (AutoApplicant):** folding `novar` into its own `mise.toml` `lint` and the + `spotbugs`→`audit` tidy — carried by the inline→foundry CI migration work-stream. diff --git a/mise/java.toml b/mise/java.toml index 17e85c9..34f5689 100644 --- a/mise/java.toml +++ b/mise/java.toml @@ -2,6 +2,14 @@ # Commands assume Gradle + Spotless; structural smells come from habit-hooks' # Java/PMD plugin, not the Gradle build. Adjust for Maven or your plugins. # Proven on a Spring Boot repo (AutoApplicant); see OVERVIEW.md. +# +# No FOUNDRY_SINCE execute-scoping here, unlike mise/ts.toml — and that's deliberate, +# not an omission (see designs/changed-scope-gate.md). On the JVM the Gradle build +# dominates wall-clock, and Gradle already scopes the fast placements for free: +# Spotless `spotlessCheck` ratchets from origin/main (only changed files), incremental +# compilation scopes `typecheck`, and up-to-date checks skip unaffected `test` work. +# Bolting a changed-file file-list on top would save little and fight the build tool. +# The one ratchet-scoped check that needs an explicit diff is `novar`, below. [tools] java = "21" @@ -19,7 +27,12 @@ java = "21" # with no manual install, identically local and in CI. [env] PMD_VERSION = "7.27.0" -_.path = ["{{env.HOME}}/.local/opt/pmd-bin-{{env.PMD_VERSION}}/bin"] +# PMD on PATH, plus the repo's scripts/ so the gate can call `foundry-verb-wrap` by name. +_.path = ["{{env.HOME}}/.local/opt/pmd-bin-{{env.PMD_VERSION}}/bin", "{{config_root}}/scripts"] + +# Loop telemetry: each gate verb appends one JSONL line here (see designs/loop-telemetry.md). +# Gitignore `.foundry/`. Read it with `foundry-loop-report`. foundry-init installs the wrapper. +FOUNDRY_TELEMETRY = "{{config_root}}/.foundry/telemetry.jsonl" [hooks] postinstall = "mise run setup:pmd" @@ -41,8 +54,12 @@ description = "Auto-format (Spotless; use ratchetFrom('origin/main') to avoid a run = "./gradlew spotlessApply" [tasks.lint] -description = "Format check (Spotless). Structural smells run via habit-hooks (PMD)." -run = "./gradlew spotlessCheck" +description = "Format check (Spotless) + no-`var` on changed source. Smells run via habit-hooks (PMD)." +# lint is 'style + smells' (CONTRACT.md), not only auto-fixable formatting: Spotless +# is mechanical (`mise run fix`), but no-`var` is a real rule — fix it or tag a +# justified use. Both are non-mutating here. novar is ratchet-scoped (see below), so +# folding it in never fails on legacy `var`. +run = ["./gradlew spotlessCheck", "mise run novar"] [tasks.typecheck] description = "Compilation is Java's type check" @@ -61,11 +78,36 @@ run = [ "osv-scanner --lockfile=gradle.lockfile", ] +# no-`var` is a ratchet-scoped check: it must fail only on *new* `var`, never on legacy. +# It diffs from the merge-base of a base ref with HEAD — pre-existing `var` is untouched +# until its file is edited. The base ref is FOUNDRY_BASE_REF (the ratchet base, distinct +# from FOUNDRY_SINCE's speed signal), defaulting to origin/main for local runs; CI sets +# it to the PR base SHA, since origin/main isn't a reliable ref in a PR checkout. An +# unresolvable base skips (CI enforces from a full-history checkout, so it's authoritative). +# Folded into `lint` above — no separate CI job needed. [tasks.novar] -description = "Report `var` uses. Enforce diff-scoped in CI. Tag exceptions `// foundry-allow-var: reason`." +description = "no-`var` on changed Java (ratchet base = FOUNDRY_BASE_REF, default origin/main). Tag exceptions `// foundry-allow-var: reason`." depends = ["setup:pmd"] -run = "pmd check -d src/main/java -R config/pmd/no-var.xml -f text --suppress-marker foundry-allow-var" +run = ''' +set -eu +mb=$(git merge-base "${FOUNDRY_BASE_REF:-origin/main}" HEAD 2>/dev/null || true) +if [ -z "$mb" ]; then echo "novar: cannot resolve base '${FOUNDRY_BASE_REF:-origin/main}'; skipping"; exit 0; fi +files=$(git diff --name-only --diff-filter=ACMR "$mb" 2>/dev/null | grep -E '^src/main/java/.*\.java$' || true) +if [ -z "$files" ]; then echo "novar: no changed main Java files"; exit 0; fi +set -- +for f in $files; do set -- "$@" "-d=$f"; done +pmd check "$@" -R config/pmd/no-var.xml -f text --no-progress --suppress-marker foundry-allow-var +''' [tasks.gate] description = "The oracle. Sequential — mise `depends` runs in parallel with no ordering." -run = ["mise run lint", "mise run typecheck", "mise run test", "mise run audit"] +# Each verb wrapped by `foundry-verb-wrap`: times it, appends a line to FOUNDRY_TELEMETRY, +# propagates the exit code unchanged (telemetry never changes the outcome; a failing verb +# still stops the sequence). Read with `foundry-loop-report`. Wrapper on PATH via scripts/ +# ([env]); foundry-init installs it. Not part of the six-verb contract. +run = [ + "foundry-verb-wrap lint -- mise run lint", + "foundry-verb-wrap typecheck -- mise run typecheck", + "foundry-verb-wrap test -- mise run test", + "foundry-verb-wrap audit -- mise run audit", +] diff --git a/mise/ts.toml b/mise/ts.toml index 1949716..345afe8 100644 --- a/mise/ts.toml +++ b/mise/ts.toml @@ -5,8 +5,24 @@ node = "22.14.0" # pin to whatever the repo actually uses [env] -# Put local binaries on PATH so tasks call `eslint`, not `npx eslint`. -_.path = ["./node_modules/.bin"] +# Put local binaries on PATH so tasks call `eslint`, not `npx eslint`; and the repo's +# scripts/ so the gate can call `foundry-verb-wrap` by name (foundry-init installs it). +_.path = ["./node_modules/.bin", "{{config_root}}/scripts"] + +# Loop telemetry: each gate verb appends one JSONL line here (see designs/loop-telemetry.md). +# Gitignore `.foundry/`. Read it with `foundry-loop-report`. The gate calls +# `foundry-verb-wrap` (installed by foundry-init, on PATH via scripts/ above). +FOUNDRY_TELEMETRY = "{{config_root}}/.foundry/telemetry.jsonl" + +# ── FOUNDRY_SINCE — changed-scope for the fast placements ──────────────────── +# A git ref (empty ⇒ origin/main). When SET, the file-local verbs (`lint`, `test`) +# restrict work to files changed since its merge-base with HEAD — for the in-loop +# Stop hook and pre-push, where speed matters. When UNSET (CI, and the final +# `/ship` gate), they run whole-tree. Whole-program checks (`typecheck`) ignore it +# and always run whole-tree — a changed file can break an unchanged one. This is +# execute-scope, safe only because whole-tree CI is the authoritative backstop. +# See designs/changed-scope-gate.md. An unresolvable ref falls back to whole-tree +# (never skips). The signal is opt-in per verb; callers MUST NOT require it. [tasks.fix] description = "Auto-fix what is mechanically fixable; never opinionated" @@ -23,7 +39,21 @@ description = "Style and smells; non-mutating" # Fails on new violations AND on stale suppressions, so the baseline cannot rot. # eslint-suppressions.json holds the accepted debt — it is committed, and it may # only ever shrink. Never add to it by hand; never delete it to make lint pass. -run = "eslint ." +# Honours FOUNDRY_SINCE (see [env]): scoped to changed files at the fast placements, +# whole-tree in CI. eslint findings are file-local, so scoping is safe; the +# whole-tree CI run still catches stale suppressions repo-wide. +run = ''' +set -eu +if [ -z "${FOUNDRY_SINCE+x}" ]; then exec eslint .; fi +base="${FOUNDRY_SINCE:-origin/main}" +mb=$(git merge-base "$base" HEAD 2>/dev/null || true) +if [ -z "$mb" ]; then echo "changed-scope lint: cannot resolve '$base'; whole-tree"; exec eslint .; fi +changed=$(git diff --name-only --diff-filter=ACMR "$mb" -- '*.ts' '*.tsx' '*.js' '*.cjs' '*.mjs' 2>/dev/null || true) +untracked=$(git ls-files --others --exclude-standard -- '*.ts' '*.tsx' '*.js' '*.cjs' '*.mjs' 2>/dev/null || true) +files=$(printf '%s\n%s\n' "$changed" "$untracked" | sort -u | sed '/^$/d') +if [ -z "$files" ]; then echo "changed-scope lint: no changed JS/TS since $mb"; exit 0; fi +printf '%s\n' "$files" | xargs -d '\n' eslint +''' [tasks.typecheck] description = "Static types" @@ -33,20 +63,70 @@ run = "tsc -b" [tasks.test] description = "Tests with coverage floor" -run = "vitest run --coverage" +# FOUNDRY_SINCE unset (CI / final gate): full suite + coverage floor — the oracle. +# FOUNDRY_SINCE set (in-loop / pre-push): only tests affected by the changeset, via +# vitest's module graph, and WITHOUT the coverage floor — coverage is a whole-tree +# property, so it's enforced on the unset path only. Unresolvable ref ⇒ full suite. +run = ''' +set -eu +if [ -z "${FOUNDRY_SINCE+x}" ]; then exec vitest run --coverage; fi +base="${FOUNDRY_SINCE:-origin/main}" +mb=$(git merge-base "$base" HEAD 2>/dev/null || true) +if [ -z "$mb" ]; then echo "changed-scope test: cannot resolve '$base'; full suite"; exec vitest run --coverage; fi +exec vitest run --changed "$mb" +''' [tasks.audit] description = "Dependency vulnerabilities. Local subset; CI Tier 0 adds gitleaks" run = "npm audit --audit-level=critical" +# Architecture fitness (opt-in, tier-c auxiliary until you adopt it). Enforces YOUR +# layer map + no cycles via dependency-cruiser (needs `dependency-cruiser` installed). +# Whole-graph, so NOT changed-scoped — a cycle can be formed by an unrelated edge. +# To adopt: fill the layer map in .dependency-cruiser.cjs (from presets/arch/), generate +# the shrink-only baseline once, then fold `mise run arch` into `lint`. See +# designs/arch-fitness.md. Architecture-agnostic — hexagonal is just the default map. +[tasks.arch] +description = "Architecture fitness: no cross-layer imports, no cycles (dependency-cruiser)" +run = ''' +set -eu +cfg=".dependency-cruiser.cjs" +base=".dependency-cruiser-known-violations.json" +if [ ! -f "$cfg" ]; then echo "arch: no $cfg yet — copy presets/arch/ and fill your layer map"; exit 0; fi +if [ -f "$base" ]; then + exec depcruise src --config "$cfg" --ignore-known "$base" +fi +exec depcruise src --config "$cfg" +''' + +# Flaky-test triage (opt-in, tier-c auxiliary). Runs tests emitting a machine report, +# then lets `foundry-flaky` drop QUARANTINED flakes (flaky-baseline.json) from the gate +# decision while still printing them — a real failure still fails. Fold into `test` when +# adopted; keep coverage on the main `test`. See designs/flake-triager.md. `foundry-flaky` +# is on PATH via scripts/ (foundry-init installs it). +[tasks."test:flaky"] +description = "Tests, with quarantined flakes reported but not gating (flaky-baseline.json)" +run = ''' +set -eu +mkdir -p .foundry +rep=".foundry/vitest-report.json" +vitest run --reporter=json --outputFile="$rep" || true +foundry-flaky gate "$rep" flaky-baseline.json +''' + [tasks.gate] description = "The authoritative composite — the oracle. Nothing merges without this green." # Array form runs sequentially. Do NOT use `depends`: mise runs dependencies in # parallel with no ordering guarantee, which makes gate output unreadable and # hides which check failed first. +# Each verb is wrapped by `foundry-verb-wrap`, which times it, appends one JSONL line +# to FOUNDRY_TELEMETRY, then propagates the verb's exit code unchanged — telemetry +# never changes the outcome, and a failing verb still stops the sequence. Read the log +# with `foundry-loop-report`. The wrapper is required (foundry-init installs it, on +# PATH via scripts/ — see [env]); it is NOT on the six-verb contract. run = [ - "mise run lint", - "mise run typecheck", - "mise run test", - "mise run audit", + "foundry-verb-wrap lint -- mise run lint", + "foundry-verb-wrap typecheck -- mise run typecheck", + "foundry-verb-wrap test -- mise run test", + "foundry-verb-wrap audit -- mise run audit", ] diff --git a/presets/agent-loop.md b/presets/agent-loop.md index 8cb1365..ba599eb 100644 --- a/presets/agent-loop.md +++ b/presets/agent-loop.md @@ -45,6 +45,24 @@ possible — trust the in-loop signal, don't defer to CI: Same rules in each — no "passes locally, fails in CI." +### Fast iterative runs — `FOUNDRY_SINCE` + +While you're *iterating* — running the oracle over and over on one change — scope the +file-local verbs to what you touched so each round is fast. Set `FOUNDRY_SINCE` to your +branch base: + +``` +FOUNDRY_SINCE=origin/main mise run gate # lint + test only over the changeset +``` + +`lint` and `test` then run on changed files only (tests via the module graph); +`typecheck` still runs whole-tree, because a change can break an unchanged file's types +and scoping that would be a false green. **Your final verification is always a clean, +whole-tree `mise run gate` with `FOUNDRY_SINCE` unset** — the scoped runs are for speed +during the loop, not for declaring done. CI never sets it, so whole-tree CI stays the +backstop. Unset it and you're back to the full local gate; an unresolvable ref falls +back to whole-tree on its own. + ## Outer loop — campaigns (`repo-align`) Paying down debt is the same discipline, scaled — but **bounded**: pick a target up diff --git a/presets/arch/ArchitectureTest.java b/presets/arch/ArchitectureTest.java new file mode 100644 index 0000000..47e9a6b --- /dev/null +++ b/presets/arch/ArchitectureTest.java @@ -0,0 +1,81 @@ +// Foundry architecture fitness — Java (ArchUnit). +// +// ArchUnit rules ARE JUnit tests, so this runs under `mise run test` with no verb +// change. Drop it in src/test/java//arch/ and change the package + base-package +// below. Requires the ArchUnit test dependency: +// testImplementation("com.tngtech.archunit:archunit-junit5:1.3.0") // pin the version +// +// ARCHITECTURE-AGNOSTIC. Hexagonal (AutoApplicant's shape) is the default; swap the +// layer/access lines for classic-layered, clean/onion, or modular — see the examples +// at the bottom and designs/arch-fitness.md. +// +// RATCHET: every rule is wrapped in FreezingArchRule.freeze(...). On first run it +// records today's violations into a store (default: archunit_store/, committed) and +// passes; thereafter only NEW violations fail, and the store may only shrink. +// ruleset-guard watches the store files with the `lines` kind. +package com.example.arch; // <- your base package + ".arch" + +import com.tngtech.archunit.core.importer.ImportOption; +import com.tngtech.archunit.junit.AnalyzeClasses; +import com.tngtech.archunit.junit.ArchTest; +import com.tngtech.archunit.lang.ArchRule; +import com.tngtech.archunit.library.freeze.FreezingArchRule; + +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.noClasses; +import static com.tngtech.archunit.library.Architectures.layeredArchitecture; +import static com.tngtech.archunit.library.dependencies.SlicesRuleDefinition.slices; + +@AnalyzeClasses( + packages = "com.example", // <- your base package + importOptions = ImportOption.DoNotIncludeTests.class) +class ArchitectureTest { + + // --- Layer boundaries (hexagonal example) ------------------------------- + // `whereLayer(X).mayOnlyBeAccessedByLayers(...)` reads "who may depend on X". + @ArchTest + static final ArchRule layerBoundaries = FreezingArchRule.freeze( + layeredArchitecture().consideringOnlyDependenciesInLayers() + .layer("Domain").definedBy("..domain..") + .layer("Port").definedBy("..port..") + .layer("Usecase").definedBy("..usecase..") + .layer("Adapter").definedBy("..adapter..") + .layer("Config").definedBy("..config..") + // Domain and Port are the core: nothing below them may be reached upward. + .whereLayer("Adapter").mayNotBeAccessedByAnyLayer() + .whereLayer("Usecase").mayOnlyBeAccessedByLayers("Adapter", "Config") + .whereLayer("Port").mayOnlyBeAccessedByLayers("Usecase", "Adapter", "Config") + .whereLayer("Domain").mayOnlyBeAccessedByLayers("Port", "Usecase", "Adapter", "Config")); + + // --- Framework-freedom: the domain stays plain -------------------------- + @ArchTest + static final ArchRule domainIsFrameworkFree = FreezingArchRule.freeze( + noClasses().that().resideInAPackage("..domain..") + .should().dependOnClassesThat() + .resideInAnyPackage("org.springframework..", "jakarta..", "..adapter..", "..config..")); + + // --- No package cycles -------------------------------------------------- + @ArchTest + static final ArchRule noCycles = FreezingArchRule.freeze( + slices().matching("com.example.(*)..") // <- your base package + .should().beFreeOfCycles()); +} + +// ---- example architectures (replace the layerBoundaries rule) -------------- +// +// Classic layered (n-tier): controller -> service -> repository +// layeredArchitecture().consideringOnlyDependenciesInLayers() +// .layer("Web").definedBy("..controller..", "..web..") +// .layer("Service").definedBy("..service..") +// .layer("Data").definedBy("..repository..", "..dao..") +// .whereLayer("Web").mayNotBeAccessedByAnyLayer() +// .whereLayer("Service").mayOnlyBeAccessedByLayers("Web") +// .whereLayer("Data").mayOnlyBeAccessedByLayers("Service"); +// +// Clean / onion: entities <- usecases <- interfaces <- frameworks +// .layer("Entities").definedBy("..entities..") ... .whereLayer("Entities").mayOnlyBeAccessedByLayers("Usecases","Interfaces","Frameworks") ... +// +// Modular / feature isolation: features never depend on each other, only a shared kernel. +// Use noClasses() per feature instead of layeredArchitecture(): +// noClasses().that().resideInAPackage("..features.billing..") +// .should().dependOnClassesThat().resideInAPackage("..features.auth..") +// or slices().matching("..features.(*)..").should().notDependOnEachOther(). diff --git a/presets/arch/dependency-cruiser.cjs b/presets/arch/dependency-cruiser.cjs new file mode 100644 index 0000000..e4fb9c3 --- /dev/null +++ b/presets/arch/dependency-cruiser.cjs @@ -0,0 +1,107 @@ +// Foundry architecture fitness — TypeScript (dependency-cruiser). +// +// Deterministic, whole-graph enforcement of YOUR architecture: forbids cross-layer +// imports and dependency cycles. Runs under `lint` (fold in when your layer map is +// filled — see mise/ts.toml `arch` task). It is whole-program, so it always runs +// whole-tree, never changed-scoped (a cycle can be formed by an unrelated edge). +// +// ARCHITECTURE-AGNOSTIC. You describe *your* layers once, in LAYERS + ALLOW; every +// rule is generated from that. Hexagonal is only the default example — swap in any of +// the maps at the bottom (classic-layered, clean/onion, modular) or write your own. +// +// A "layer" is a set of source paths (regex, matched against the module path). +// ALLOW[x] lists the layers x MAY import; any layer not listed is forbidden. +// Cycles are always forbidden. Framework-freedom is a FORBIDDEN_IMPORTS rule. +// +// Ratchet: generate today's violations as a baseline that may only shrink, and fail +// only on NEW ones (see the `options` note). ruleset-guard watches the baseline. + +/** layer name -> path regex (matched against the resolved module path). */ +const LAYERS = { + domain: "^src/domain/", + usecase: "^src/(usecase|application)/", + adapter: "^src/adapter/", + infra: "^src/(infra|config)/", +}; + +/** allowed *internal* dependencies per layer; omit or [] => may import nothing internal. */ +const ALLOW = { + domain: [], // framework-free core: depends on no other layer + usecase: ["domain"], + adapter: ["usecase", "domain"], + infra: ["adapter", "usecase", "domain"], +}; + +/** layers that must not import a given third-party package regex (framework-freedom). */ +const FORBIDDEN_IMPORTS = [ + // { layers: ["domain"], packages: "^(@angular|@nestjs|express|typeorm)" }, +]; + +// --------------------------------------------------------------------------- +// Generation — you should not need to edit below this line. +// --------------------------------------------------------------------------- +const names = Object.keys(LAYERS); + +const layerRules = names + .map((name) => { + const allowed = new Set([name, ...(ALLOW[name] || [])]); + const forbidden = names.filter((n) => !allowed.has(n)); + if (forbidden.length === 0) return null; + return { + name: `layer-${name}`, + comment: `${name} may import only {${[...allowed].join(", ")}} (+ externals). See presets/arch/guides/layering-violation.md`, + severity: "error", + from: { path: LAYERS[name] }, + to: { path: forbidden.map((n) => LAYERS[n]) }, + }; + }) + .filter(Boolean); + +const frameworkRules = FORBIDDEN_IMPORTS.map((r, i) => ({ + name: `no-framework-in-${r.layers.join("-")}-${i}`, + comment: `${r.layers.join("/")} must stay framework-free. See presets/arch/guides/layering-violation.md`, + severity: "error", + from: { path: r.layers.map((l) => LAYERS[l]) }, + to: { path: r.packages, dependencyTypes: ["npm", "npm-dev", "npm-peer"] }, +})); + +module.exports = { + forbidden: [ + { + name: "no-circular", + comment: "Dependency cycles make the module graph impossible to reason about or test in isolation. See presets/arch/guides/dependency-cycle.md", + severity: "error", + from: {}, + to: { circular: true }, + }, + ...layerRules, + ...frameworkRules, + ], + options: { + doNotFollow: { path: "node_modules" }, + tsConfig: { fileName: "tsconfig.json" }, + // Ratchet (verify exact flags for your dependency-cruiser version): generate a + // known-violations baseline once, commit it, and run against it so only NEW + // violations fail. The baseline may only shrink; ruleset-guard watches it with + // the `snooze` kind (value_counts over the JSON). Example: + // depcruise src --config presets/arch/dependency-cruiser.cjs \ + // --output-type baseline > .dependency-cruiser-known-violations.json + // depcruise src --config presets/arch/dependency-cruiser.cjs \ + // --ignore-known .dependency-cruiser-known-violations.json + }, +}; + +// ---- example maps for other architectures (replace LAYERS + ALLOW above) ---- +// +// Classic layered (n-tier): +// LAYERS = { web: "^src/(controller|web)/", service: "^src/service/", data: "^src/(repo|dao)/" } +// ALLOW = { web: ["service"], service: ["data"], data: [] } +// +// Clean / onion: +// LAYERS = { entities:"^src/entities/", usecases:"^src/usecases/", ifaces:"^src/interfaces/", fw:"^src/frameworks/" } +// ALLOW = { entities:[], usecases:["entities"], ifaces:["usecases","entities"], fw:["ifaces","usecases","entities"] } +// +// Modular / feature-sliced (feature isolation — features never import each other, +// only a shared kernel): give each feature its own layer whose ALLOW is ["shared"]: +// LAYERS = { shared:"^src/shared/", billing:"^src/features/billing/", auth:"^src/features/auth/" } +// ALLOW = { shared: [], billing: ["shared"], auth: ["shared"] } // billing !-> auth, auth !-> billing diff --git a/presets/arch/guides/dependency-cycle.md b/presets/arch/guides/dependency-cycle.md new file mode 100644 index 0000000..640e602 --- /dev/null +++ b/presets/arch/guides/dependency-cycle.md @@ -0,0 +1,29 @@ +# dependency-cycle — two or more modules import each other + +**What it's telling you.** Module A imports B, and B (directly or through a chain) imports +back to A. A cycle means the modules are really one tangled unit: you can't understand, +test, build, or reuse either without the other, and the "boundary" between them is +fiction. Cycles are where refactors go to die. + +**Fix toward breaking the loop, not hiding it.** +- **Find the edge that points the wrong way.** Usually one import in the cycle violates + the intended direction. Reverse it by depending on an abstraction: extract the shared + contract (an interface/type) into a module both can depend on, so the concrete + dependency points one way only. +- **Extract the shared piece.** If A and B both need X, X wants to be its own module that + both import — not something one owns and lends to the other. +- **Split a module that wears two hats.** A cycle often means a module has two + responsibilities, one of which belongs on the other side of the boundary. Move it. + +**Don't game it.** +- Making the import dynamic/lazy (a deferred `require`, a runtime lookup, a string path) + so the static detector stops seeing the edge does **not** break the cycle — the runtime + coupling is still there, now invisible. This is strictly worse. +- Deleting the no-circular rule or freezing the cycle into the baseline for *new* code + hides a structural defect. Baselining is only for pre-existing debt you'll pay down. +- Merging A and B into one file to "remove" the cross-module edge trades a visible cycle + for a bigger blob — the smell moves, it doesn't leave. + +The rule is a proxy for "modules have a direction and can stand alone." Break the loop by +introducing the missing abstraction; that's the change that makes the graph reasoning- +friendly again. diff --git a/presets/arch/guides/layering-violation.md b/presets/arch/guides/layering-violation.md new file mode 100644 index 0000000..1480465 --- /dev/null +++ b/presets/arch/guides/layering-violation.md @@ -0,0 +1,33 @@ +# layering-violation — a module reached across an architectural boundary + +**What it's telling you.** Your architecture declares who may depend on whom (see the +layer map in the arch config). This import crosses a boundary the wrong way — e.g. the +domain reached into an adapter, or a use case imported a framework. Left alone, these +edges accumulate until the "layers" are decorative and nothing can be changed or tested +in isolation. + +**Fix toward the dependency rule, not the symptom.** +- **Depend inward, never outward.** The core (domain/entities) must not know about the + things that call it (adapters, controllers, frameworks). If the core "needs" an + adapter, it actually needs an *interface* the core owns and the adapter implements — + invert the dependency (a port). +- **Move the code, don't move the line.** If a class sits in the wrong layer for what it + does, relocate it to the layer whose rules it obeys — don't keep it where it is and + widen the layer map to allow the edge. +- **A framework leak in the core** means business logic is entangled with a library. + Extract the pure logic; keep the framework call in the adapter/config layer. + +**Don't game it.** +- Widening the layer map (adding the forbidden edge to `ALLOW`, or deleting a rule) to + make the check pass is a **loosening** — it needs the `ruleset-change` label and a + human, precisely because it silently redefines the architecture. That's the opposite + of a fix. +- Casting to `any`/`Object`, importing via a string/dynamic path, or routing through a + third module to dodge the detector hides the coupling instead of removing it — worse + than the original violation. +- Adding the violation to the frozen baseline is only honest for *pre-existing* debt. + New code that trips this rule should be fixed, not frozen. + +The rule is a proxy for "each layer can be understood and replaced on its own." Clearing +it by weakening the rule defeats the purpose; clearing it by inverting the dependency is +the fix that makes the next change easier. diff --git a/presets/flaky-baseline.example.json b/presets/flaky-baseline.example.json new file mode 100644 index 0000000..8cc35f8 --- /dev/null +++ b/presets/flaky-baseline.example.json @@ -0,0 +1,15 @@ +{ + "//": "Foundry flaky-test quarantine baseline. Committed; may ONLY shrink. Growing it (adding an entry) is a ratchet loosening — ruleset-guard requires the ruleset-change label, because 'this test is flaky, trust me' is a deliberate human decision. Removing an entry (a fixed/stable test) is tightening — no label. Generate quarantine candidates with `foundry-flaky classify`; prune stable ones. See designs/flake-triager.md.", + "quarantined": [ + { + "id": "com.autoapplicant.crawler.CrawlerRateLimitTest.backsOffUnderLoad", + "reason": "timing-dependent sleep assertion; races on a loaded runner. Tracking: #123", + "since": "2026-09-21" + }, + { + "id": "cart > recalculates total after concurrent add", + "reason": "order-dependent against a shared in-memory store. Tracking: #200", + "since": "2026-09-21" + } + ] +} diff --git a/presets/prompt-eval/PromptEvalHarnessTest.java b/presets/prompt-eval/PromptEvalHarnessTest.java new file mode 100644 index 0000000..ceb015b --- /dev/null +++ b/presets/prompt-eval/PromptEvalHarnessTest.java @@ -0,0 +1,69 @@ +// Foundry prompt / agent-output regression harness — Java (JUnit 5). +// +// Generalises AutoApplicant's PromptEvalHarnessTest. Deterministic + OFFLINE: score +// hand-written good/weak samples with YOUR scorer and assert it RANKS them (good >= FLOOR, +// weak <= CEILING, gap >= SEPARATION). Runs under `mise run test` — it IS a test, no new verb. +// foundry ships NO scorer: implement QualityScorer (and, if you assert prompt blocks, a +// prompt composer) in your own code. Fixtures live in src/test/resources/prompt-eval/, +// listed in manifest.json. +// +// RATCHET: the fixture COUNT may only grow (never delete a regression case) — ruleset-guard +// watches manifest.json with the `coverage` kind. Do NOT ratchet the SCORES upward (it +// overfits the scorer). Thresholds are ruleset, guard-watched via the ruleset-file watch. +package com.example.prompteval; // <- your package + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +class PromptEvalHarnessTest { + + private static final int FLOOR = 80; // a good sample must clear this + private static final int CEILING = 60; // a weak sample must stay under this + private static final int SEPARATION = 30; // the gap that makes the score trustworthy + + private final ObjectMapper mapper = new ObjectMapper(); + + /** YOUR deterministic scorer — this is the domain-specific half foundry does not ship. */ + private int scoreOutput(String output, String input) { + // return new DocumentQualityEvaluator(...).evaluate(output, input).total(); + throw new UnsupportedOperationException("wire in your QualityScorer"); + } + + private record Fixture(String name, String input, String good, String weak) {} + + private List load() throws IOException { + List out = new ArrayList<>(); + try (InputStream mf = getClass().getResourceAsStream("/prompt-eval/manifest.json")) { + assertThat(mf).as("prompt-eval/manifest.json on the test classpath").isNotNull(); + for (JsonNode name : mapper.readTree(mf).path("fixtures")) { + try (InputStream in = getClass().getResourceAsStream("/prompt-eval/" + name.asText())) { + JsonNode n = mapper.readTree(in); + out.add(new Fixture(n.path("name").asText(), n.path("input").asText(), + n.path("good").asText(), n.path("weak").asText())); + } + } + } + return out; + } + + @TestFactory + Stream scorerSeparatesGoodFromWeak() throws IOException { + return load().stream().map(f -> DynamicTest.dynamicTest(f.name(), () -> { + int good = scoreOutput(f.good(), f.input()); + int weak = scoreOutput(f.weak(), f.input()); + assertThat(good).as("good sample for %s", f.name()).isGreaterThanOrEqualTo(FLOOR); + assertThat(weak).as("weak sample for %s", f.name()).isLessThanOrEqualTo(CEILING); + assertThat(good - weak).as("separation for %s", f.name()).isGreaterThanOrEqualTo(SEPARATION); + })); + } +} diff --git a/presets/prompt-eval/fixtures/example.json b/presets/prompt-eval/fixtures/example.json new file mode 100644 index 0000000..a154c1f --- /dev/null +++ b/presets/prompt-eval/fixtures/example.json @@ -0,0 +1,7 @@ +{ + "name": "English platform engineer, mid-career", + "input": "Job posting + candidate profile the prompt is given (kept minimal here). Your scorer receives this as context to judge the output against.", + "expectedBlocks": ["## Honesty & ATS Rules", "## Banned Phrases"], + "good": "At Netcompany I owned the Terraform estate across 30 AWS accounts; median deploy time went from 40 minutes to 9 because we stopped treating every environment as a special case. That was mostly observability work — you cannot shorten a deploy you cannot see. What draws me to this role is owning the estate, not a queue of tickets against it.", + "weak": "I am writing to apply for the platform engineer position. I am a results-driven team player who is passionate about cloud infrastructure and confident I would be a great fit. In today's fast-paced environment I hit the ground running and wear many hats. I look forward to hearing from you." +} diff --git a/presets/prompt-eval/manifest.json b/presets/prompt-eval/manifest.json new file mode 100644 index 0000000..b498c7e --- /dev/null +++ b/presets/prompt-eval/manifest.json @@ -0,0 +1,6 @@ +{ + "//": "The fixture set the harness loads. This is the ratcheted surface: the list may ONLY GROW. Removing an entry drops a regression case — ruleset-guard's `coverage` kind flags it as a loosening (needs the ruleset-change label). Adding one is tightening (more coverage, no label). Fixture files live next to this in ./fixtures/ (TS) or /prompt-eval/ on the test classpath (Java).", + "fixtures": [ + "example.json" + ] +} diff --git a/presets/prompt-eval/prompt-eval.test.ts b/presets/prompt-eval/prompt-eval.test.ts new file mode 100644 index 0000000..c1b80e1 --- /dev/null +++ b/presets/prompt-eval/prompt-eval.test.ts @@ -0,0 +1,61 @@ +// Foundry prompt / agent-output regression harness — TypeScript (vitest). +// +// Generalises AutoApplicant's PromptEvalHarnessTest. Deterministic and OFFLINE: it scores +// hand-written good/weak samples with YOUR scorer and asserts the scorer RANKS them — +// good >= FLOOR, weak <= CEILING, and the gap >= SEPARATION. Separation is the trustworthy +// signal (a scorer earns trust by ranking; an absolute cutoff on hand-written text is +// arbitrary). It also checks the composed prompt still carries the blocks a fixture needs, +// so a prompt regression is caught here, not in a bad output weeks later. +// +// This runs under `test` — it IS a test, no new verb. foundry ships NO scorer: you provide +// `scoreOutput` (your deterministic quality scorer) and optionally `composePrompt`. +// +// RATCHET: the fixture COUNT may only grow — never delete a regression case. Enforced on +// manifest.json by ruleset-guard's `coverage` kind. Do NOT ratchet the SCORES upward: +// chasing an ever-higher floor overfits the scorer, which is exactly the game-the-proxy +// failure agent-loop.md warns against. The thresholds below are ruleset (guard-watched via +// the ruleset-file watch — lowering FLOOR / raising CEILING / lowering SEPARATION needs a +// human label), not a ratchet. + +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +// You implement these in your repo — they are the domain-specific half foundry does not ship: +// scoreOutput(output, input) -> number (your deterministic quality score, 0..100) +// composePrompt(input) -> string (optional; only if you assert prompt blocks) +import { composePrompt, scoreOutput } from "./your-scorer"; + +const FLOOR = 80; // a good sample must clear this +const CEILING = 60; // a weak sample must stay under this +const SEPARATION = 30; // the gap that makes the score trustworthy as a regression metric + +interface Fixture { + name: string; + input: string; + good: string; + weak: string; + expectedBlocks?: string[]; // substrings the composed prompt must contain +} + +const here = (p: string) => new URL(p, import.meta.url); +const manifest: { fixtures: string[] } = JSON.parse(readFileSync(here("./manifest.json"), "utf8")); +const fixtures: Fixture[] = manifest.fixtures.map( + (f) => JSON.parse(readFileSync(here(`./fixtures/${f}`), "utf8")), +); + +describe("prompt eval", () => { + it.each(fixtures)("$name — scorer separates good from weak", (f) => { + const good = scoreOutput(f.good, f.input); + const weak = scoreOutput(f.weak, f.input); + expect(good, `good sample for ${f.name}`).toBeGreaterThanOrEqual(FLOOR); + expect(weak, `weak sample for ${f.name}`).toBeLessThanOrEqual(CEILING); + expect(good - weak, `separation for ${f.name}`).toBeGreaterThanOrEqual(SEPARATION); + }); + + const withBlocks = fixtures.filter((f) => f.expectedBlocks?.length); + it.each(withBlocks)("$name — composed prompt carries required blocks", (f) => { + const prompt = composePrompt(f.input); + for (const block of f.expectedBlocks ?? []) { + expect(prompt, `prompt for ${f.name} missing block: ${block}`).toContain(block); + } + }); +}); diff --git a/scripts/foundry-flaky b/scripts/foundry-flaky new file mode 100644 index 0000000..2ad2dda --- /dev/null +++ b/scripts/foundry-flaky @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""foundry-flaky - the deterministic Triager (DESIGN §4.2), built. + +Two jobs, no model: + + gate [baseline] + Read a test report (JUnit XML or vitest JSON) and a flaky-baseline.json, then + decide the gate outcome: exit 1 ONLY if a test that is NOT quarantined failed. + A quarantined test that fails is printed (never hidden) but does not gate. + This is post-processing over the runner's own report - we do not fight the + runner with plugins or @Disabled; the runner runs everything, we judge the log. + + classify --runs N --rerun "" [baseline] + Rerun each FAILED test N times to tell a flake from a real failure. A test that + both passes and fails across the reruns is a flake CANDIDATE - printed for a + human to add to the baseline (adding is a ratchet loosening; ruleset-guard makes + it a labelled decision). A test that fails every rerun is real; a quarantined + test that now passes every rerun is a prune CANDIDATE (re-arm it). + +Baseline (flaky-baseline.json), committed, may only shrink: + { "quarantined": [ { "id": "", "reason": "...", "since": "" } ] } + +Test id format (match the baseline to these): + JUnit XML -> "." (Gradle/PHPUnit/JUnit) + vitest -> the assertion "fullName" (ancestor titles + test title, space-joined) +""" +import json +import subprocess +import sys +import xml.etree.ElementTree as ET + + +def parse_report(path): + """Return (failed_ids, total) from a JUnit XML or vitest JSON report.""" + with open(path, encoding="utf-8") as fh: + head = fh.read(4096) + fh.seek(0) + body = fh.read() + if head.lstrip().startswith("<"): + return _parse_junit(body) + return _parse_vitest(body) + + +def _parse_junit(text): + failed, total = [], 0 + root = ET.fromstring(text) + # may be the root or nested under . + for case in root.iter("testcase"): + total += 1 + cls, name = case.get("classname", ""), case.get("name", "") + tid = f"{cls}.{name}" if cls else name + # JUnit marks a failed case with a direct or child. + if case.find("failure") is not None or case.find("error") is not None: + failed.append(tid) + return failed, total + + +def _parse_vitest(text): + data = json.loads(text) + failed, total = [], 0 + for suite in data.get("testResults", []): + for a in suite.get("assertionResults", []): + total += 1 + tid = a.get("fullName") or " ".join( + a.get("ancestorTitles", []) + [a.get("title", "")] + ).strip() + if a.get("status") == "failed": + failed.append(tid) + return failed, total + + +def load_quarantine(path): + if not path: + return {} + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except FileNotFoundError: + return {} + q = {} + for e in data.get("quarantined", []): + q[e["id"] if isinstance(e, dict) else e] = ( + e.get("reason", "") if isinstance(e, dict) else "" + ) + return q + + +def cmd_gate(report, baseline): + failed, total = parse_report(report) + quar = load_quarantine(baseline) + real = [t for t in failed if t not in quar] + flaky_hits = [t for t in failed if t in quar] + + print(f"flaky: {total} tests, {len(failed)} failed " + f"({len(real)} real, {len(flaky_hits)} quarantined)") + for t in flaky_hits: + print(f" quarantined (not gating): {t} [{quar[t] or 'no reason recorded'}]") + for t in real: + print(f" FAIL (gating): {t}") + if real: + print("flaky: gate FAILS - a non-quarantined test failed.") + return 1 + if flaky_hits: + print("flaky: gate passes - only quarantined flakes failed. Fix or prune them; " + "quarantine is debt, not a resting place.") + else: + print("flaky: gate passes - clean.") + return 0 + + +def cmd_classify(report, baseline, runs, rerun_tmpl): + failed, _ = parse_report(report) + quar = load_quarantine(baseline) + if not failed: + print("flaky classify: nothing failed - nothing to classify.") + return 0 + print(f"flaky classify: rerunning {len(failed)} failed test(s) x{runs}...") + new_flakes, real_fails, prune = [], [], [] + for tid in failed: + passes = fails = 0 + for _ in range(runs): + cmd = rerun_tmpl.replace("{test}", tid) + rc = subprocess.run(cmd, shell=True).returncode + if rc == 0: + passes += 1 + else: + fails += 1 + flips = passes > 0 and fails > 0 + tag = "FLAKE" if flips else ("real" if fails == runs else "passed-all") + print(f" {tid}: {passes} pass / {fails} fail -> {tag}") + if flips and tid not in quar: + new_flakes.append(tid) + elif fails == runs and tid not in quar: + real_fails.append(tid) + if tid in quar and passes == runs: + prune.append(tid) + if new_flakes: + print("\nQuarantine CANDIDATES (add to flaky-baseline.json - needs the " + "ruleset-change label, since growing the baseline is a loosening):") + for t in new_flakes: + print(f' {{ "id": "{t}", "reason": "TODO", "since": "TODO" }}') + if prune: + print("\nPRUNE candidates (quarantined but now stable - remove to re-arm; " + "removal is tightening, no label needed):") + for t in prune: + print(f" {t}") + if real_fails: + print("\nReal failures (fix these - not flaky):") + for t in real_fails: + print(f" {t}") + return 0 + + +def main(argv): + if not argv or argv[0] not in ("gate", "classify"): + print(__doc__) + return 2 + mode, rest = argv[0], argv[1:] + if mode == "gate": + if not rest: + print("usage: foundry-flaky gate [baseline]", file=sys.stderr) + return 2 + return cmd_gate(rest[0], rest[1] if len(rest) > 1 else None) + # classify + runs, rerun = 5, None + pos = [] + i = 0 + while i < len(rest): + if rest[i] == "--runs": + runs = int(rest[i + 1]); i += 2 + elif rest[i] == "--rerun": + rerun = rest[i + 1]; i += 2 + else: + pos.append(rest[i]); i += 1 + if not pos or not rerun: + print('usage: foundry-flaky classify --runs N --rerun "" [baseline]', + file=sys.stderr) + return 2 + return cmd_classify(pos[0], pos[1] if len(pos) > 1 else None, runs, rerun) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/foundry-init.sh b/scripts/foundry-init.sh index 41bcf38..2cdc2d3 100644 --- a/scripts/foundry-init.sh +++ b/scripts/foundry-init.sh @@ -52,6 +52,20 @@ echo "- shared presets" fetch "presets/gitleaks.toml" ".gitleaks.toml" fetch "presets/renovate.json" "renovate.json" fetch "scripts/ruleset_guard.py" "scripts/ruleset_guard.py" + +# Loop telemetry: the verb wrapper (on PATH via the mise template's {{config_root}}/scripts) +# and the offline summariser. See designs/loop-telemetry.md. +fetch "scripts/foundry-verb-wrap" "scripts/foundry-verb-wrap" +fetch "scripts/foundry-loop-report" "scripts/foundry-loop-report" +chmod +x scripts/foundry-verb-wrap scripts/foundry-loop-report 2>/dev/null || true +# The telemetry log is local-only observability, never committed. +if [ ! -f .gitignore ]; then + printf '# Foundry loop telemetry (local observability)\n.foundry/\n' > .gitignore + echo " wrote: .gitignore (+.foundry/)" +elif ! grep -qxF '.foundry/' .gitignore 2>/dev/null; then + printf '\n# Foundry loop telemetry (local observability)\n.foundry/\n' >> .gitignore + echo " updated: .gitignore (+.foundry/)" +fi if [ "$STACK" = "java" ]; then fetch "presets/pmd/ruleset.xml" "$WD/pmd/ruleset.xml" fetch "presets/pmd/no-var.xml" "$WD/config/pmd/no-var.xml" diff --git a/scripts/foundry-loop-report b/scripts/foundry-loop-report new file mode 100644 index 0000000..5ba5bd0 --- /dev/null +++ b/scripts/foundry-loop-report @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""foundry-loop-report - make the agent loop legible from verb telemetry. + +Reads the JSONL emitted by foundry-verb-wrap (see designs/loop-telemetry.md) and +prints where the loop spends time, whether it is converging or thrashing, and how +each verb has been faring. Deterministic and offline; no model, no network. + +Usage: + foundry-loop-report [PATH] + +PATH defaults to $FOUNDRY_TELEMETRY, then .foundry/telemetry.jsonl. + +Honest scope (v1): this summarises *verb* telemetry (timing, exit, scope, +placement) and derives a thrash signal from pass->fail regressions. Smell-trend +convergence needs smell counts, which the habit-hooks Stop hook will emit later +(a follow-on); until then this reports "thrash" from verb regressions only. +""" +import json +import os +import sys +from collections import defaultdict + + +def load(path): + rows = [] + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + # A torn last line (crash mid-write) shouldn't sink the report. + continue + except FileNotFoundError: + return None + return rows + + +def fmt_ms(ms): + return f"{ms / 1000:.1f}s" if ms >= 1000 else f"{ms}ms" + + +def main(): + path = ( + sys.argv[1] if len(sys.argv) > 1 + else os.environ.get("FOUNDRY_TELEMETRY", ".foundry/telemetry.jsonl") + ) + rows = load(path) + if rows is None: + print(f"No telemetry at {path}. Run the gate once with foundry-verb-wrap wired in.") + return 0 + if not rows: + print(f"{path} is empty - nothing to report yet.") + return 0 + + rows.sort(key=lambda r: r.get("ms", 0)) + + per = defaultdict(lambda: {"runs": 0, "fails": 0, "regress": 0, "total": 0, "last_ok": None}) + placements, scopes = defaultdict(int), defaultdict(int) + for r in rows: + v = r.get("verb", "?") + st = per[v] + st["runs"] += 1 + st["total"] += r.get("duration_ms", 0) + ok = r.get("exit", 0) == 0 + if not ok: + st["fails"] += 1 + # A pass followed later by a fail is a regression - the thrash signal. + if st["last_ok"] is True and not ok: + st["regress"] += 1 + st["last_ok"] = ok + placements[r.get("placement", "?")] += 1 + scopes[r.get("scope", "?")] += 1 + + total_ms = sum(s["total"] for s in per.values()) + first, last = rows[0].get("ts", "?"), rows[-1].get("ts", "?") + lint_runs = per.get("lint", {}).get("runs", 0) + + print(f"Foundry loop report - {path}") + print(f"Window: {first} -> {last} ({len(rows)} verb runs" + + (f", ~{lint_runs} gate attempts" if lint_runs else "") + ")") + print() + print("Per verb (by total time):") + print(f" {'verb':<12}{'runs':>5}{'fails':>6}{'regress':>8}{'total':>10}{'avg':>9}") + for v, s in sorted(per.items(), key=lambda kv: kv[1]["total"], reverse=True): + avg = s["total"] // s["runs"] if s["runs"] else 0 + print(f" {v:<12}{s['runs']:>5}{s['fails']:>6}{s['regress']:>8}" + f"{fmt_ms(s['total']):>10}{fmt_ms(avg):>9}") + print() + + # Time sink. + if total_ms: + sink, ss = max(per.items(), key=lambda kv: kv[1]["total"]) + pct = 100 * ss["total"] / total_ms + print(f"Time sink: {sink} ({fmt_ms(ss['total'])}, {pct:.0f}% of tracked verb time)") + + # Thrash / convergence. + thrash = {v: s["regress"] for v, s in per.items() if s["regress"]} + if thrash: + detail = ", ".join(f"{v} x{n}" for v, n in sorted(thrash.items(), key=lambda kv: -kv[1])) + print(f"Thrash: regressed after passing - {detail}. " + f"Fixing one verb may be breaking another; stop and look before another round.") + else: + print("Thrash: none - no verb regressed after passing (loop looks convergent).") + + # Latest snapshot. + latest = {} + for r in rows: + latest[r.get("verb", "?")] = r.get("exit", 0) == 0 + red = [v for v, ok in latest.items() if not ok] + print("Latest: " + ("all tracked verbs green" if not red else "still red -> " + ", ".join(red))) + + print(f"Placements: " + ", ".join(f"{k} {v}" for k, v in sorted(placements.items()))) + print(f"Scopes: " + ", ".join(f"{k} {v}" for k, v in sorted(scopes.items()))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/foundry-verb-wrap b/scripts/foundry-verb-wrap new file mode 100644 index 0000000..2f44145 --- /dev/null +++ b/scripts/foundry-verb-wrap @@ -0,0 +1,56 @@ +#!/bin/sh +# foundry-verb-wrap — time a verb, record the outcome, propagate its exit code. +# +# foundry-verb-wrap -- [args...] +# +# Runs , then appends ONE JSON line to the telemetry log describing the +# run (verb, exit, duration, scope, placement). The command's exit code is always +# propagated unchanged — telemetry is best-effort and MUST NOT change a verb's +# outcome. This is the single instrumentation point every placement routes through +# (see designs/loop-telemetry.md); wire it into a mise template's verbs. +# +# Env: +# FOUNDRY_TELEMETRY log path (default: .foundry/telemetry.jsonl, gitignored) +# FOUNDRY_PLACEMENT ci|ship|hook|manual (default: "ci" if $CI set, else "manual") +# FOUNDRY_SINCE if set (even empty), scope is recorded as "changed", else "whole" +set -u + +verb="${1:-unknown}" +[ $# -gt 0 ] && shift +[ "${1:-}" = "--" ] && shift + +telemetry="${FOUNDRY_TELEMETRY:-.foundry/telemetry.jsonl}" + +placement="${FOUNDRY_PLACEMENT:-}" +if [ -z "$placement" ]; then + if [ -n "${CI:-}" ]; then placement="ci"; else placement="manual"; fi +fi + +# FOUNDRY_SINCE set (even to empty) => the verb ran changed-scoped; unset => whole-tree. +if [ -n "${FOUNDRY_SINCE+x}" ]; then scope="changed"; else scope="whole"; fi + +# Epoch milliseconds; fall back to second-resolution where date lacks %N (e.g. BSD). +now_ms() { + d=$(date +%s%3N 2>/dev/null) + case "$d" in + ''|*[!0-9]*) echo "$(date +%s)000" ;; + *) echo "$d" ;; + esac +} + +start=$(now_ms) +"$@" +rc=$? +dur=$(( $(now_ms) - start )) +[ "$dur" -lt 0 ] && dur=0 +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "") + +# Best-effort append. Any failure here is swallowed so the verb's exit code stands. +( + dir=$(dirname "$telemetry") + [ -d "$dir" ] || mkdir -p "$dir" 2>/dev/null || exit 0 + printf '{"ts":"%s","ms":%s,"verb":"%s","scope":"%s","placement":"%s","exit":%d,"duration_ms":%d}\n' \ + "$ts" "$start" "$verb" "$scope" "$placement" "$rc" "$dur" >> "$telemetry" 2>/dev/null || true +) 2>/dev/null || true + +exit "$rc" diff --git a/scripts/ruleset_guard.py b/scripts/ruleset_guard.py index f66fb79..b77d6c3 100644 --- a/scripts/ruleset_guard.py +++ b/scripts/ruleset_guard.py @@ -11,7 +11,17 @@ the inline gate and foundry's reusable tier0.yml (which runs in the caller's checkout) can call it. -Usage: ruleset_guard.py +Kinds: + eslint eslint-suppressions.json (per file+rule count) + snooze habit-hooks snooze.json, and dependency-cruiser known-violations JSON + (position-independent multiset of scalar leaves — an added violation + increments its scalars, so it's caught; reorder/remove are safe) + lines ArchUnit FreezingArchRule store files (one frozen violation per line; + run once per changed store file under archunit_store/) + coverage prompt-eval manifest.json — INVERSE: a REMOVED fixture is less coverage, + so removal (not addition) is the loosening + +Usage: ruleset_guard.py """ import json import subprocess @@ -46,7 +56,34 @@ def walk(v): return c +def line_counts(text): + """ArchUnit freeze store: one violation per line -> multiset of non-empty lines.""" + c = Counter() + for ln in (text or "").splitlines(): + ln = ln.strip() + if ln: + c[ln] += 1 + return c + + +def coverage_counts(text): + """prompt-eval manifest.json: multiset of the `fixtures` list (the coverage surface).""" + d = json.loads(text) if text.strip() else {} + c = Counter() + for item in (d.get("fixtures") or []): + c[item] += 1 + return c + + def loosened(kind, old_text, new_text): + if kind == "lines": + o, n = line_counts(old_text), line_counts(new_text) + return [k for k in n if n[k] > o.get(k, 0)] + if kind == "coverage": + # INVERSE of a debt baseline: for a test-coverage surface, a REMOVED entry is + # *less* coverage = a loosening. Adding entries is fine (tightening). + o, n = coverage_counts(old_text), coverage_counts(new_text) + return [k for k in o if n.get(k, 0) < o[k]] old = json.loads(old_text) if old_text.strip() else None new = json.loads(new_text) if new_text.strip() else None counts = eslint_counts if kind == "eslint" else value_counts