From fa2284612681b92b79fd83d1ef0da1f948bd19c8 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 11:07:11 -0500 Subject: [PATCH 01/14] Add the autoseat proposer, a pure function that seats each audit role from the operator's pool and the recorded rankings, so a lineup can be explained and reproduced from its input. Pins win, measured seats take the measured best at or above the floor, the deriver and critic take pool order, and a critic or verifier equal to its writer or reviewer refuses the plan. A seat nothing qualifies for is left unfilled with the flag that would fill it, never filled by lowering the floor or by a model outside the pool. Co-Authored-By: Claude Fable 5.2 --- internal/autoseat/autoseat.go | 296 +++++++++++++++++++++++++++++ internal/autoseat/autoseat_test.go | 150 +++++++++++++++ 2 files changed, 446 insertions(+) create mode 100644 internal/autoseat/autoseat.go create mode 100644 internal/autoseat/autoseat_test.go diff --git a/internal/autoseat/autoseat.go b/internal/autoseat/autoseat.go new file mode 100644 index 00000000..57f1f48f --- /dev/null +++ b/internal/autoseat/autoseat.go @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: Elastic-2.0 + +// Package autoseat proposes which model of an operator's pool takes each +// audit seat, from the rankings corral has recorded. +// +// WHY A PURE FUNCTION. Staffing is a decision, and a decision that reads a +// clock, a network or a model cannot be reproduced from the record that +// justifies it. Propose takes the evidence as an argument and returns the +// same lineup for the same input, so a proposal can be tested table-style and +// explained to the operator row by row. +// +// WHY NO DEFAULTS. corral names no model of its own. Every seat is filled by +// something the operator typed: a pin, or a member of the pool they named. A +// model outside the pool that is not pinned never takes a seat, however well +// it ranks, and a seat nothing qualifies for is left unfilled with the flag +// that would fill it, never filled with a substitute. The evidence floor is +// the same: a row below it is not evidence, and the floor is never lowered to +// fill a seat. +package autoseat + +import ( + "fmt" + "strings" + + "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/modelrank" +) + +// How a seat came to be filled. The three are the whole vocabulary a plan and +// the record use, so a reader never has to guess which rule applied. +const ( + ByOperator = "operator" + ByEvidence = "evidence" + ByOrder = "order" +) + +// PinFlag maps a role to the flag that pins it. It is the one table the +// "pin --writer-model" text is built from, so a refusal names a flag that +// exists. +var PinFlag = map[string]string{ + modelrank.SeatGoalDeriver: "derive-model", modelrank.SeatMutantGenerator: "mutant-model", + modelrank.SeatTestWriter: "writer-model", modelrank.SeatTestCritic: "critic-model", + modelrank.SeatReviewer: "reviewer-model", modelrank.SeatVerifier: "verifier-model", +} + +// off is the pin that disables a seat. It satisfies the decorrelation rule +// because there is no critic left to be correlated with the writer. +const off = "off" + +// Member is one pool entry, in the order the operator named it. +type Member struct { + Given string // exactly what the operator typed; what a seat flag is set to + Recorded string // the name the record ranks it under (ResolvedSeat.Spec) + Provider string + Rankable bool // false for a bare agent seat with no model +} + +// Input is everything a proposal depends on. +type Input struct { + Roles []string // in the order the verb meets them + Pool []Member // in the order the operator named it + Pins map[string]string // role -> the operator's flag value ("off" included) + Report modelrank.Report // already filtered to the pool and the language + Floor int +} + +// Seat is the proposal for one role. +type Seat struct { + Role string + Given string // "" when unfilled + Recorded string + By string + Reason string + Unfilled string // the fix, when the role could not be filled +} + +// Plan is the whole lineup, one Seat per role in Roles order. +type Plan struct { + Seats []Seat + Refused bool // true when any seat is Unfilled or the lineup breaks a hard rule + Why string // the lineup-level hard-rule failure, when there is one +} + +// counterpart is the seat a role must differ from: a critic that is its own +// writer, or a verifier that is its own reviewer, is a model grading its own +// work, which is what the decorrelation rule exists to prevent. +var counterpart = map[string]string{ + modelrank.SeatTestCritic: modelrank.SeatTestWriter, + modelrank.SeatVerifier: modelrank.SeatReviewer, +} + +// measured roles are filled only from sufficient evidence. The critic is not +// here: it is measured only where a person adjudicated it, and otherwise +// falls through to pool order. +var measured = map[string]bool{ + modelrank.SeatMutantGenerator: true, modelrank.SeatTestWriter: true, + modelrank.SeatReviewer: true, modelrank.SeatVerifier: true, +} + +// Propose applies the rules in order: pins win; measured seats take the +// measured best; unmeasured seats take the pool order; the hard rules are +// checked on the whole lineup, pins included. +func Propose(in Input) Plan { + plan := Plan{Seats: make([]Seat, len(in.Roles))} + if in.Floor < 1 { + // The floor is how "enough evidence" is defined; without one every + // row would silently count, which is a default by another name. + plan.Refused = true + plan.Why = "the evidence floor must be at least 1; none was given" + for i, r := range in.Roles { + plan.Seats[i] = Seat{Role: r, Unfilled: "no evidence floor was given"} + } + return plan + } + + done := map[string]Seat{} + // Seats with a counterpart depend on it, so they are filled second, but + // the plan keeps the caller's order. + for _, dependent := range []bool{false, true} { + for i, r := range in.Roles { + if _, dep := counterpart[r]; dep != dependent { + continue + } + plan.Seats[i] = in.seat(r, done) + done[r] = plan.Seats[i] + } + } + + for _, s := range plan.Seats { + if s.Unfilled != "" { + plan.Refused = true + } + } + for role, other := range counterpart { + a, b := done[role], done[other] + if a.Given == "" || b.Given == "" || strings.EqualFold(a.Given, off) { + continue + } + if agentbackend.SameSeat(a.Given, b.Given) { + plan.Refused = true + plan.Why = fmt.Sprintf("%s and %s are both %q; a model must not grade its own work", + role, other, a.Given) + } + } + return plan +} + +func (in Input) seat(role string, done map[string]Seat) Seat { + if v, ok := in.Pins[role]; ok { + return Seat{Role: role, Given: v, Recorded: in.recordedFor(v), By: ByOperator, + Reason: fmt.Sprintf("pinned by --%s", PinFlag[role])} + } + avoid := in.avoid(role, done) + if row, ok := in.best(role, avoid); ok { + mem, _ := in.member(row.Model) + return Seat{Role: role, Given: mem.Given, Recorded: mem.Recorded, By: ByEvidence, Reason: row.Evidence} + } + switch { + case measured[role]: + return Seat{Role: role, Unfilled: fmt.Sprintf( + "no model in --models has %d+ %s as %s; pin --%s", + in.Floor, in.unit(role), role, PinFlag[role])} + case role == modelrank.SeatGoalDeriver, role == modelrank.SeatTestCritic: + return in.byOrder(role, avoid) + } + return Seat{Role: role, Unfilled: fmt.Sprintf("no rule seats %s; pin --%s", role, PinFlag[role])} +} + +// byOrder seats the first pool member that is not the counterpart, preferring +// a different provider: two models from one vendor satisfy the letter of +// decorrelation and not its point. +func (in Input) byOrder(role, avoid string) Seat { + const reason = "no measurement for this seat: taken from the order you named" + var same *Member + for i, mem := range in.Pool { + if avoid != "" && agentbackend.SameSeat(mem.Recorded, avoid) { + continue + } + if avoid != "" && same == nil && in.providerOf(avoid) != "" && mem.Provider == in.providerOf(avoid) { + same = &in.Pool[i] + continue + } + return Seat{Role: role, Given: mem.Given, Recorded: mem.Recorded, By: ByOrder, Reason: reason} + } + if same != nil { + return Seat{Role: role, Given: same.Given, Recorded: same.Recorded, By: ByOrder, + Reason: reason + " (same provider as " + counterpart[role] + "; no other member was available)"} + } + fix := fmt.Sprintf("name a second model in --models, or pin --%s", PinFlag[role]) + if role == modelrank.SeatTestCritic { + fix = "name a second model in --models, or pass --critic-model off" + } + return Seat{Role: role, Unfilled: fmt.Sprintf("no pool member other than the %s can take %s; %s", + counterpart[role], role, fix)} +} + +// best walks the group's rows in the order modelrank returned them and takes +// the first sufficient, rankable pool member that is not the avoided model. +// Members whose metric ties the best are resolved by pool position. +func (in Input) best(role, avoid string) (modelrank.Row, bool) { + var cands []modelrank.Row + for _, g := range in.Report.Groups { + if g.Seat != role { + continue + } + for _, r := range g.Rows { + mem, ok := in.member(r.Model) + if !ok || !mem.Rankable || !r.Sufficient || r.Metric == nil { + continue + } + if avoid != "" && agentbackend.SameSeat(mem.Recorded, avoid) { + continue + } + cands = append(cands, r) + } + break + } + if len(cands) == 0 { + return modelrank.Row{}, false + } + win, winPos := cands[0], in.position(cands[0].Model) + for _, r := range cands[1:] { + if *r.Metric == *cands[0].Metric && in.position(r.Model) < winPos { + win, winPos = r, in.position(r.Model) + } + } + return win, true +} + +// avoid is the recorded name the role must differ from, taken from the +// counterpart's seat (a pin included), or "". +func (in Input) avoid(role string, done map[string]Seat) string { + other, ok := counterpart[role] + if !ok { + return "" + } + s := done[other] + if s.Given == "" || strings.EqualFold(s.Given, off) { + return "" + } + if s.Recorded != "" { + return s.Recorded + } + return s.Given +} + +func (in Input) member(recorded string) (Member, bool) { + for _, m := range in.Pool { + if m.Recorded == recorded { + return m, true + } + } + return Member{}, false +} + +func (in Input) position(recorded string) int { + for i, m := range in.Pool { + if m.Recorded == recorded { + return i + } + } + return len(in.Pool) +} + +// recordedFor resolves a pin to the name the record knows it by when it is a +// pool member; a pin outside the pool is recorded as typed. +func (in Input) recordedFor(pin string) string { + for _, m := range in.Pool { + if agentbackend.SameSeat(m.Given, pin) || agentbackend.SameSeat(m.Recorded, pin) { + return m.Recorded + } + } + return pin +} + +func (in Input) providerOf(recorded string) string { + if m, ok := in.member(recorded); ok { + return m.Provider + } + return "" +} + +// unit names the seat's own evidence unit for the refusal text, read from a +// row of the report so it is the unit modelrank counted in. +func (in Input) unit(role string) string { + for _, g := range in.Report.Groups { + if g.Seat == role { + for _, r := range g.Rows { + if r.NUnit != "" { + return r.NUnit + } + } + } + } + return "observations" +} diff --git a/internal/autoseat/autoseat_test.go b/internal/autoseat/autoseat_test.go new file mode 100644 index 00000000..f0663b0d --- /dev/null +++ b/internal/autoseat/autoseat_test.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package autoseat + +import ( + "strings" + "testing" + + "github.com/pdbethke/corralai/internal/modelrank" +) + +func f(v float64) *float64 { return &v } + +func row(seat, model string, metric float64, sufficient bool) modelrank.Row { + return modelrank.Row{Seat: seat, Model: model, Metric: f(metric), Sufficient: sufficient, + Evidence: "evidence for " + model, N: 9, NUnit: "survivors attempted"} +} + +func report(rows ...modelrank.Row) modelrank.Report { + var r modelrank.Report + bySeat := map[string]int{} + for _, rw := range rows { + i, ok := bySeat[rw.Seat] + if !ok { + r.Groups = append(r.Groups, modelrank.Group{Seat: rw.Seat}) + i = len(r.Groups) - 1 + bySeat[rw.Seat] = i + } + r.Groups[i].Rows = append(r.Groups[i].Rows, rw) + } + return r +} + +func m(name, provider string) Member { + return Member{Given: name, Recorded: name, Provider: provider, Rankable: true} +} + +var certifyRoles = []string{modelrank.SeatGoalDeriver, modelrank.SeatMutantGenerator, modelrank.SeatTestWriter, modelrank.SeatTestCritic} + +func seatOf(t *testing.T, p Plan, role string) Seat { + t.Helper() + for _, s := range p.Seats { + if s.Role == role { + return s + } + } + t.Fatalf("no seat for %s", role) + return Seat{} +} + +func TestPropose(t *testing.T) { + W, G, C, D := modelrank.SeatTestWriter, modelrank.SeatMutantGenerator, modelrank.SeatTestCritic, modelrank.SeatGoalDeriver + three := []Member{m("a", "anthropic"), m("b", "anthropic"), m("c", "openai")} + tests := []struct { + name string + in Input + role string + given string + by string + reason string + refused bool + }{ + {"pin wins over a better row", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "x"}, + Report: report(row(W, "a", 0.99, true)), Floor: 5}, W, "x", ByOperator, "", false}, + {"measured best", Input{Roles: certifyRoles, Pool: three, + Report: report(row(W, "b", 0.99, true), row(W, "a", 0.80, true)), Floor: 5}, W, "b", ByEvidence, "evidence for b", false}, + {"tie goes to pool order", Input{Roles: certifyRoles, Pool: three, + Report: report(row(W, "c", 0.9, true), row(W, "b", 0.9, true)), Floor: 5}, W, "b", ByEvidence, "evidence for b", false}, + {"below the floor", Input{Roles: certifyRoles, Pool: three, + Report: report(row(W, "a", 0.99, false)), Floor: 5}, W, "", "", "", true}, + {"row outside the pool is never chosen", Input{Roles: certifyRoles, Pool: three, + Report: report(row(W, "zzz", 0.99, true), row(W, "c", 0.5, true)), Floor: 5}, W, "c", ByEvidence, "", false}, + {"unrankable member is not chosen by evidence", Input{Roles: certifyRoles, + Pool: []Member{{Given: "codex", Recorded: "codex", Provider: "openai"}, m("b", "anthropic")}, + Report: report(row(G, "codex", 0.99, true), row(G, "b", 0.4, true)), Floor: 5}, G, "b", ByEvidence, "", false}, + {"unrankable member can take the deriver", Input{Roles: certifyRoles, + Pool: []Member{{Given: "codex", Recorded: "codex", Provider: "openai"}, m("b", "anthropic")}, Floor: 5}, + D, "codex", ByOrder, "no measurement for this seat: taken from the order you named", false}, + {"critic prefers a different provider", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "a"}, Floor: 5}, + C, "c", ByOrder, "taken from the order", false}, + {"critic with a sufficient row is by evidence", Input{Roles: certifyRoles, Pool: three, + Report: report(row(C, "b", 0.7, true)), Floor: 5}, C, "b", ByEvidence, "evidence for b", false}, + {"critic evidence equal to writer is skipped", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "a"}, + Report: report(row(C, "a", 0.9, true), row(C, "b", 0.7, true)), Floor: 5}, C, "b", ByEvidence, "evidence for b", false}, + {"critic evidence only the writer falls back to order", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "a"}, + Report: report(row(C, "a", 0.9, true)), Floor: 5}, C, "c", ByOrder, "", false}, + {"one model cannot seat a critic", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Pins: map[string]string{W: "a"}, Floor: 5}, + C, "", "", "", true}, + {"critic off is a pin", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Pins: map[string]string{W: "a", C: "off"}, Floor: 5}, + C, "off", ByOperator, "", false}, + {"pinned writer b leaves critic a", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic"), m("b", "anthropic")}, + Pins: map[string]string{W: "b"}, Floor: 5}, C, "a", ByOrder, "", false}, + {"pinned critic equal to writer breaks the hard rule", Input{Roles: certifyRoles, Pool: three, + Pins: map[string]string{W: "a", C: "a"}, Floor: 5}, C, "a", ByOperator, "", true}, + {"verifier is never the reviewer", Input{Roles: []string{modelrank.SeatReviewer, modelrank.SeatVerifier}, Pool: three, + Report: report(row(modelrank.SeatReviewer, "a", 0.9, true), row(modelrank.SeatVerifier, "a", 0.95, true), row(modelrank.SeatVerifier, "c", 0.6, true)), + Floor: 5}, modelrank.SeatVerifier, "c", ByEvidence, "evidence for c", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p := Propose(tc.in) + s := seatOf(t, p, tc.role) + if s.Given != tc.given || (tc.by != "" && s.By != tc.by) { + t.Fatalf("seat = %+v, want given %q by %q", s, tc.given, tc.by) + } + if tc.reason != "" && !strings.Contains(s.Reason, tc.reason) { + t.Fatalf("reason %q lacks %q", s.Reason, tc.reason) + } + // The other roles of the lineup may be legitimately unfilled in + // these hand-built inputs, so the case asserts about ITS seat (or + // the lineup-level rule), and that any such failure refuses the plan. + failed := s.Unfilled != "" || p.Why != "" + if failed != tc.refused || (failed && !p.Refused) { + t.Fatalf("failed = %v, want %v (plan %+v)", failed, tc.refused, p) + } + if len(p.Seats) != len(tc.in.Roles) { + t.Fatalf("%d seats for %d roles", len(p.Seats), len(tc.in.Roles)) + } + }) + } +} + +func TestProposeUnfilledNamesTheFix(t *testing.T) { + W, C := modelrank.SeatTestWriter, modelrank.SeatTestCritic + p := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, + Pins: map[string]string{W: "a"}, Report: report(row(W, "a", 0.99, false)), Floor: 5}) + w := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, + Report: report(row(W, "a", 0.99, false)), Floor: 5}) + if s := seatOf(t, w, W); !strings.Contains(s.Unfilled, "pin --writer-model") || !strings.Contains(s.Unfilled, "5") || !w.Refused { + t.Fatalf("writer unfilled text = %q refused=%v", s.Unfilled, w.Refused) + } + if s := seatOf(t, p, C); !strings.Contains(s.Unfilled, "--critic-model off") { + t.Fatalf("critic unfilled text = %q", s.Unfilled) + } +} + +func TestProposeHardRuleSetsWhy(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "x"), m("b", "x")}, + Pins: map[string]string{modelrank.SeatTestWriter: "a", modelrank.SeatTestCritic: "A"}, Floor: 5}) + if !p.Refused || p.Why == "" { + t.Fatalf("plan = %+v, want refused with Why", p) + } +} + +func TestProposeRefusesAMissingFloor(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "x")}}) + if !p.Refused || !strings.Contains(p.Why, "floor") { + t.Fatalf("plan = %+v, want a refusal naming the floor", p) + } +} From f0ac51fb984ff9fd600b7765c0a5e4049ea899ef Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 11:21:26 -0500 Subject: [PATCH 02/14] Carry how each seat was chosen, and what the ranking read, on the record, because a lineup proposed by --models is otherwise indistinguishable from one the operator named by hand. SeatedBy (role -> operator, evidence or order) and SeatEvidence travel the road SeatProviders took in e30847d2: RunSpec -> Verdict on both construction paths (they ride signedRoster, so tickAggregate and timeoutVerdict cannot differ, and SeatedOnly drops a seat the roster leaves off), ledger.Row, reviewrecord.Review, and the corral_audits and corral_reviews warehouse columns with their read path. Absent fields are omitempty and NULL, so an entry written without --models is byte-identical to today's, and absence reads as "every seat was named by flag". Tier is declared, not measured: it is corral's statement of how it made a choice, not something execution observed. The in-toto statements are not touched; no gate required it, and who chose a seat is on the ledger row. VerdictGeneration is 48 because the Verdict shape changed on a generation that is on main; a cached "47" document reads both fields as absent. Nothing sets the fields yet outside tests; the cmd/corral doors come in later tasks. Co-Authored-By: Claude Fable 5.2 --- docs/corral/record-fields.md | 4 ++ internal/advpool/driver.go | 15 ++++++ internal/advpool/run.go | 12 ++++- internal/advpool/seated_by_test.go | 55 ++++++++++++++++++++ internal/auditpush/bundle.go | 14 +++-- internal/auditpush/read.go | 8 +-- internal/auditpush/review_grains.go | 21 +++++--- internal/auditpush/seated_by_test.go | 74 +++++++++++++++++++++++++++ internal/ledger/row.go | 11 +++- internal/ledger/tiers.go | 6 +++ internal/reposcan/generation.go | 14 ++++- internal/reviewrecord/reviewrecord.go | 6 +++ 12 files changed, 223 insertions(+), 17 deletions(-) create mode 100644 internal/advpool/seated_by_test.go create mode 100644 internal/auditpush/seated_by_test.go diff --git a/docs/corral/record-fields.md b/docs/corral/record-fields.md index 20a97c2f..b24985ed 100644 --- a/docs/corral/record-fields.md +++ b/docs/corral/record-fields.md @@ -259,7 +259,9 @@ The table is checked against the record's Go types by a test, so a field cannot | `ReviewerModelResolved` | measured | | | `ReviewerTool` | measured | | | `Scope` | measured | | +| `SeatEvidence` | declared | how corral chose the seat: the operator's flag, the ranking, or the order the pool was named; a statement of the choice, not a measurement | | `SeatProviders` | measured | the provider corral resolved and sent the seat's calls to; not a check of what served the model behind that endpoint, and for an agent seat a vendor label | +| `SeatedBy` | declared | how corral chose the seat: the operator's flag, the ranking, or the order the pool was named; a statement of the choice, not a measurement | | `Sound` | meta | a model's opinion: advisory, signed but not verified as true | | `StartedAt` | measured | | | `StatementSHA256` | key | | @@ -354,8 +356,10 @@ The table is checked against the record's Go types by a test, so a field cannot | `RunCache` | meta | the run's own account of itself: signed, but not verified as true | | `RunURL` | key | | | `ScanID` | key | | +| `SeatEvidence` | declared | how corral chose the seat: the operator's flag, the ranking, or the order the pool was named; a statement of the choice, not a measurement | | `SeatProviders` | measured | the provider corral resolved and sent the seat's calls to; not a check of what served the model behind that endpoint, and for an agent seat a vendor label | | `SeatTools` | measured | | +| `SeatedBy` | declared | how corral chose the seat: the operator's flag, the ranking, or the order the pool was named; a statement of the choice, not a measurement | | `SelectedTests` | measured | | | `SelectionExcluded` | measured | | | `SelectionExcludedRed` | measured | | diff --git a/internal/advpool/driver.go b/internal/advpool/driver.go index 72b67f23..dabcbc72 100644 --- a/internal/advpool/driver.go +++ b/internal/advpool/driver.go @@ -998,6 +998,15 @@ type Verdict struct { // signedSeats, so both construction paths carry it, and JSON-tagged, so // the --local signed digest covers it. SeatProviders map[string]string `json:"seat_providers,omitempty"` + // SeatedBy is how each seat's model was chosen (RunSpec.SeatedBy): + // operator, evidence or order, keyed by role, for the seats the signed + // roster names. A statement of the choice, not a measurement. Absent + // means every seat was named by flag, so a verdict cached before the + // field existed reads the same. Set by signedSeats on both paths. + SeatedBy map[string]string `json:"seated_by,omitempty"` + // SeatEvidence is what the ranking behind the "evidence" seats read + // (RunSpec.SeatEvidence). "" when no seat was seated by evidence. + SeatEvidence string `json:"seat_evidence,omitempty"` // MutantSources is where each mutant of the exam came from (graded, or // refused by the compile gate, or unmeasured — so waste can be counted // per source too), keyed by mutant ID: Kind model (with the generator seat's model) for what this run's @@ -3919,6 +3928,8 @@ type signedRoster struct { selection []shadowpool.Selection tools map[string]string providers map[string]string + seatedBy map[string]string + evidence string } // stamp writes the signed seats onto v. @@ -3927,6 +3938,8 @@ func (s signedRoster) stamp(v *Verdict) { v.ShadowSelection = s.selection v.SeatTools = s.tools v.SeatProviders = s.providers + v.SeatedBy = s.seatedBy + v.SeatEvidence = s.evidence } // signedSeats is the roster a verdict signs AND the pool draws, tools and @@ -3948,6 +3961,8 @@ func (r *runState) signedSeats(assign RoleAssignment) signedRoster { roster: roster, selection: sels, tools: SeatedOnly(r.rs.SeatTools, roster), providers: SeatedOnly(r.rs.SeatProviders, roster), + seatedBy: SeatedOnly(r.rs.SeatedBy, roster), + evidence: r.rs.SeatEvidence, } } diff --git a/internal/advpool/run.go b/internal/advpool/run.go index f6f2b922..7046f026 100644 --- a/internal/advpool/run.go +++ b/internal/advpool/run.go @@ -165,7 +165,17 @@ type RunSpec struct { // absent, never filled in: the brain's daemon path resolves none and // leaves this nil. Carried to Verdict.SeatProviders by signedSeats on // BOTH construction paths. - SeatProviders map[string]string + SeatProviders map[string]string + // SeatedBy is how each seat's model was chosen, keyed by role: + // "operator" (named by flag), "evidence" (the recorded ranking picked + // it) or "order" (the first of the pool the operator named). Nil when + // every seat was named by flag, which is every run without --models. + // Carried to Verdict.SeatedBy by signedSeats on BOTH construction paths. + SeatedBy map[string]string + // SeatEvidence names what the ranking read to seat the "evidence" + // seats, for example "the machine ledger (31 scans)". "" when nothing + // was read. Carried to Verdict.SeatEvidence the same way. + SeatEvidence string ShadowSelection []shadowpool.Selection // WriterMode is HOW the writer seat attacks the survivors: diff --git a/internal/advpool/seated_by_test.go b/internal/advpool/seated_by_test.go new file mode 100644 index 00000000..8fcb08f6 --- /dev/null +++ b/internal/advpool/seated_by_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package advpool + +import ( + "encoding/json" + "testing" +) + +// How each seat was chosen (operator flag, ranking, pool order) and the +// evidence the ranking read travel RunSpec -> Verdict on BOTH construction +// paths, narrowed to the seats the roster names: a critic that is "off" was +// chosen by nothing, so it carries no basis. +func TestVerdictCarriesSeatedByOnBothConstructionPaths(t *testing.T) { + const evidence = "the machine ledger (31 scans)" + rs := RunSpec{Repo: "r", Commit: "c", Lang: "go", + SeatedBy: map[string]string{ + RoleTestWriter: "evidence", RoleTestCritic: "order", RoleMutantGenerator: "operator", + }, + SeatEvidence: evidence, + } + assign := RoleAssignment{RoleTestWriter: "w", RoleTestCritic: "off", RoleMutantGenerator: "g"} + d := &Driver{Assign: assign} + converged := Verdict{} + (&runState{rs: rs}).signedSeats(assign).stamp(&converged) + timeout := d.timeoutVerdict(&runState{rs: rs}) + for name, v := range map[string]Verdict{"converged": converged, "timeout": timeout} { + if len(v.SeatedBy) != 2 || v.SeatedBy[RoleTestWriter] != "evidence" || v.SeatedBy[RoleMutantGenerator] != "operator" { + t.Errorf("%s: SeatedBy = %v, want writer=evidence and generator=operator only (the off critic dropped)", name, v.SeatedBy) + } + if v.SeatEvidence != evidence { + t.Errorf("%s: SeatEvidence = %q, want %q", name, v.SeatEvidence, evidence) + } + b, _ := json.Marshal(v) + var doc struct { + SeatedBy map[string]string `json:"seated_by"` + SeatEvidence string `json:"seat_evidence"` + } + if err := json.Unmarshal(b, &doc); err != nil { + t.Fatal(err) + } + if doc.SeatedBy[RoleTestWriter] != "evidence" || doc.SeatEvidence != evidence { + t.Errorf("%s: serialized %s", name, b) + } + } + // Without --models nothing is set, and the verdict serializes as before. + b, _ := json.Marshal(d.timeoutVerdict(&runState{rs: RunSpec{Repo: "r"}})) + var raw map[string]json.RawMessage + _ = json.Unmarshal(b, &raw) + for _, k := range []string{"seated_by", "seat_evidence"} { + if _, ok := raw[k]; ok { + t.Errorf("a verdict with no seating basis serialized %s", k) + } + } +} diff --git a/internal/auditpush/bundle.go b/internal/auditpush/bundle.go index a6f371fb..2a3b5a3c 100644 --- a/internal/auditpush/bundle.go +++ b/internal/auditpush/bundle.go @@ -270,6 +270,8 @@ CREATE TABLE IF NOT EXISTS corral_audits ( seat_tools VARCHAR, covering_units INTEGER, seat_providers VARCHAR, + seated_by VARCHAR, + seat_evidence VARCHAR, files INTEGER, run_cache VARCHAR );` @@ -478,6 +480,12 @@ var corralAuditsMigrationCols = []struct{ name, ddl string }{ // (Row.SeatProviders). NULL on every row before it: not recorded, never // to be filled in from a model's name. {"seat_providers", "seat_providers VARCHAR"}, + // How each roster seat was chosen (Row.SeatedBy, JSON keyed by role) and + // what the ranking read (Row.SeatEvidence). NULL on every row before + // them: every seat was named by flag, which is also what NULL means on a + // row written without --models. + {"seated_by", "seated_by VARCHAR"}, + {"seat_evidence", "seat_evidence VARCHAR"}, // How many walked files the row stands for (Row.Files). NULL on every // row before it and on a one-file row: read back as 0, which FileCount // turns into one. @@ -1342,8 +1350,8 @@ func insertFileRow(tx *sql.Tx, uid string, now time.Time, r Row) error { symbols, symbols_probed, decisions, decisions_probed, mutants_unplaced, mutants_unmeasured, mutants_duplicate, writer_attempts, challenger_mutants, challenger_survived_writer, challenger_survived_shadow, challenger_union, challenger_shared, - priors_applied, prior_digest, computed_at, mutants_from, authored_landing, baseline_flaky, seat_tools, covering_units, seat_providers, files, run_cache - ) VALUES (`+placeholders(107)+`)`, // #nosec G202 -- placeholders(n) emits only "?, ?, …" for a constant count; every value is a bound parameter and no external input reaches the SQL text + priors_applied, prior_digest, computed_at, mutants_from, authored_landing, baseline_flaky, seat_tools, covering_units, seat_providers, seated_by, seat_evidence, files, run_cache + ) VALUES (`+placeholders(109)+`)`, // #nosec G202 -- placeholders(n) emits only "?, ?, …" for a constant count; every value is a bound parameter and no external input reaches the SQL text uid, now, r.Repo, r.Commit, r.Path, r.Lang, killRate, r.Survivors, r.ProvenMissed, r.TimedOut, r.TestWriterFailed, r.PoolTestUnsound, @@ -1376,7 +1384,7 @@ func insertFileRow(tx *sql.Tx, uid string, now time.Time, r Row) error { r.ChallengerMutants, r.ChallengerSurvivedWriter, r.ChallengerSurvivedShadow, r.ChallengerUnion, r.ChallengerShared, r.PriorsApplied, nullIfEmpty(r.PriorDigest), nullTime(r.ComputedAt), nullIfEmpty(r.MutantsFrom), nullIfEmpty(r.AuthoredLanding), nullIfEmpty(r.BaselineFlaky), nullIfEmpty(r.SeatTools), r.CoveringUnits, - nullIfEmpty(r.SeatProviders), nullIfZeroInt(r.Files), nullIfEmpty(r.RunCache), + nullIfEmpty(r.SeatProviders), nullIfEmpty(r.SeatedBy), nullIfEmpty(r.SeatEvidence), nullIfZeroInt(r.Files), nullIfEmpty(r.RunCache), ) if err != nil { return fmt.Errorf("auditpush: insert %s: %w", r.Path, err) diff --git a/internal/auditpush/read.go b/internal/auditpush/read.go index df56290d..e72f807e 100644 --- a/internal/auditpush/read.go +++ b/internal/auditpush/read.go @@ -368,7 +368,7 @@ func readFileRows(db *sql.DB, where string, args ...any) ([]Row, error) { symbols, symbols_probed, decisions, decisions_probed, mutants_unplaced, mutants_unmeasured, mutants_duplicate, writer_attempts, challenger_mutants, challenger_survived_writer, challenger_survived_shadow, challenger_union, challenger_shared, - priors_applied, prior_digest, computed_at, mutants_from, authored_landing, baseline_flaky, seat_tools, covering_units, seat_providers, files, run_cache + priors_applied, prior_digest, computed_at, mutants_from, authored_landing, baseline_flaky, seat_tools, covering_units, seat_providers, seated_by, seat_evidence, files, run_cache FROM corral_audits WHERE `+where, args...) // #nosec G202 -- where is a constant clause from readGrains; every value is a bound parameter if err != nil { return nil, err @@ -405,7 +405,7 @@ func readFileRows(db *sql.DB, where string, args ...any) ([]Row, error) { var coveringTests, coveringUnits sql.NullInt64 var files sql.NullInt64 var runCache sql.NullString - var mutantsFrom, authoredLanding, baselineFlaky, seatTools, seatProviders sql.NullString + var mutantsFrom, authoredLanding, baselineFlaky, seatTools, seatProviders, seatedBy, seatEvidence sql.NullString var computedAt sql.NullTime var importOnly sql.NullBool var rowPassed sql.NullBool @@ -435,7 +435,7 @@ func readFileRows(db *sql.DB, where string, args ...any) ([]Row, error) { &symbols, &symbolsProbed, &decisions, &decisionsProbed, &mutantsUnplaced, &mutantsUnmeasured, &mutantsDuplicate, &writerAttempts, &chMutants, &chSurvW, &chSurvS, &chUnion, &chShared, - &priorsApplied, &priorDigest, &computedAt, &mutantsFrom, &authoredLanding, &baselineFlaky, &seatTools, &coveringUnits, &seatProviders, &files, &runCache, + &priorsApplied, &priorDigest, &computedAt, &mutantsFrom, &authoredLanding, &baselineFlaky, &seatTools, &coveringUnits, &seatProviders, &seatedBy, &seatEvidence, &files, &runCache, ); err != nil { return nil, err } @@ -507,6 +507,8 @@ func readFileRows(db *sql.DB, where string, args ...any) ([]Row, error) { r.BaselineFlaky = baselineFlaky.String r.SeatTools = seatTools.String r.SeatProviders = seatProviders.String + r.SeatedBy = seatedBy.String + r.SeatEvidence = seatEvidence.String r.Files = int(files.Int64) r.RunCache = runCache.String diff --git a/internal/auditpush/review_grains.go b/internal/auditpush/review_grains.go index 8bbed9cc..24040b79 100644 --- a/internal/auditpush/review_grains.go +++ b/internal/auditpush/review_grains.go @@ -57,7 +57,9 @@ CREATE TABLE IF NOT EXISTS corral_reviews ( generated_with VARCHAR, reviewer_tool VARCHAR, verifier_tool VARCHAR, - seat_providers VARCHAR + seat_providers VARCHAR, + seated_by VARCHAR, + seat_evidence VARCHAR );` const findingsSchema = ` @@ -128,6 +130,11 @@ var ( // (review.Review.SeatProviders), as JSON keyed reviewer/verifier. // NULL on rows before it: not recorded. {"seat_providers", "seat_providers VARCHAR"}, + // How the reviewer and verifier seats were chosen + // (review.Review.SeatedBy), as JSON keyed reviewer/verifier, and what + // the ranking read. NULL on rows before them. + {"seated_by", "seated_by VARCHAR"}, + {"seat_evidence", "seat_evidence VARCHAR"}, } corralFindingsMigrationCols = []struct{ name, ddl string }{ // The harness marker (review.Finding.Unrun): a script the harness @@ -169,14 +176,14 @@ func insertReviewEntry(db sqlExecer, e LedgerEntry, withSource bool) (ReviewCoun review_uid, ts, repo, commit_sha, scope, lang, reviewer_model, verifier_model, substrate, started_at, files_shown, truncated, reproduced, code_read, hypothesis, sound_items, coverage, opinion_sha256, statement_sha256, input_tokens, output_tokens, source_pushed, schema_version, - author, committer, co_authors, unmetered_calls, reviewer_tool, verifier_tool, generated_with, seat_providers - ) VALUES (`+placeholders(31)+`)`, // #nosec G202 -- placeholders(n) emits only "?, ?, …" for a constant count + author, committer, co_authors, unmetered_calls, reviewer_tool, verifier_tool, generated_with, seat_providers, seated_by, seat_evidence + ) VALUES (`+placeholders(33)+`)`, // #nosec G202 -- placeholders(n) emits only "?, ?, …" for a constant count e.Hash, e.Pushed, r.Repo, r.Commit, nullIfEmpty(r.Scope), nullIfEmpty(r.Lang), nullIfEmpty(r.ReviewerModel), nullIfEmpty(r.VerifierModel), nullIfEmpty(r.Substrate), nullTime(nilIfZero(r.StartedAt)), len(r.FilesShown), r.Truncated, rep, cr, hy, len(r.Sound), nullIfEmpty(r.Coverage), nullIfEmpty(sha256HexOf(r.Opinion)), nullIfEmpty(r.StatementSHA256), reviewTokens(r, r.InputTokens), reviewTokens(r, r.OutputTokens), withSource, SchemaVersion, nullIfEmpty(r.Author), nullIfEmpty(r.Committer), nullIfEmpty(r.CoAuthors), r.UnmeteredCalls, nullIfEmpty(r.ReviewerTool), nullIfEmpty(r.VerifierTool), nullIfEmpty(r.GeneratedWith), - nullIfEmpty(seatProvidersText(r.SeatProviders)), + nullIfEmpty(roleMapText(r.SeatProviders)), nullIfEmpty(roleMapText(r.SeatedBy)), nullIfEmpty(r.SeatEvidence), ); err != nil { return ReviewCounts{}, fmt.Errorf("auditpush: insert review row: %w", err) } @@ -297,9 +304,9 @@ func reviewTokens(r *review.Review, n int64) any { return n } -// seatProvidersText is a review's seat providers as the JSON text the -// corral_reviews column holds, "" (NULL) when it recorded none. -func seatProvidersText(m map[string]string) string { +// roleMapText is a review's role-keyed seat map (providers, seated_by) as the +// JSON text the corral_reviews column holds, "" (NULL) when it recorded none. +func roleMapText(m map[string]string) string { if len(m) == 0 { return "" } diff --git a/internal/auditpush/seated_by_test.go b/internal/auditpush/seated_by_test.go new file mode 100644 index 00000000..0b2fd7b8 --- /dev/null +++ b/internal/auditpush/seated_by_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package auditpush + +import ( + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/pdbethke/corralai/internal/review" +) + +// How a seat was chosen survives the warehouse on both grains: the file row's +// SeatedBy and SeatEvidence are pushed and read back beside the neighbouring +// SeatProviders (so the INSERT's placeholder count and the read's Scan order +// both line up), and a review's land in corral_reviews as JSON text and text. +// A row that recorded none writes NULL on both. +func TestSeatedByReachesTheWarehouse(t *testing.T) { + const seatedBy = `{"mutant-generator":"operator","test-writer":"evidence"}` + const evidence = "the machine ledger (31 scans)" + const providers = `{"test-writer":"openai"}` + b := sampleBundle() + b.Files[0].SeatProviders = providers + b.Files[0].SeatedBy = seatedBy + b.Files[0].SeatEvidence = evidence + target := filepath.Join(t.TempDir(), "w.duckdb") + if _, err := PushBundle(target, b); err != nil { + t.Fatalf("PushBundle: %v", err) + } + db := openWarehouse(t, target) + back, err := ReadBundle(db, b.Scan.Repo, b.Scan.ScanID) + if err != nil { + t.Fatalf("ReadBundle: %v", err) + } + f := back.Files[0] + if f.SeatedBy != seatedBy || f.SeatEvidence != evidence || f.SeatProviders != providers { + t.Errorf("read back seated_by %q seat_evidence %q seat_providers %q", f.SeatedBy, f.SeatEvidence, f.SeatProviders) + } + + at := time.Date(2026, 11, 19, 12, 0, 0, 0, time.UTC) + with := LedgerEntry{Hash: "aa11", Kind: KindReview, Pushed: at, + Review: &review.Review{Repo: "o/r", Commit: "deadbeef", Scope: "pkg", ReviewerModel: "m", VerifierModel: "v", + SeatedBy: map[string]string{"reviewer": "evidence", "verifier": "order"}, + SeatEvidence: evidence}} + without := LedgerEntry{Hash: "bb22", Kind: KindReview, Pushed: at, + Review: &review.Review{Repo: "o/r", Commit: "deadbeef", Scope: "pkg", ReviewerModel: "m"}} + rt := filepath.Join(t.TempDir(), "r.duckdb") + for _, e := range []LedgerEntry{with, without} { + if _, err := PushReviewEntry(rt, e, true); err != nil { + t.Fatalf("PushReviewEntry %s: %v", e.Hash, err) + } + } + rows, err := openWarehouse(t, rt).Query(`SELECT review_uid, seated_by, seat_evidence FROM corral_reviews ORDER BY review_uid`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + got := map[string][2]sql.NullString{} + for rows.Next() { + var uid string + var s, e sql.NullString + if err := rows.Scan(&uid, &s, &e); err != nil { + t.Fatal(err) + } + got[uid] = [2]sql.NullString{s, e} + } + if w := got["aa11"]; w[0].String != `{"reviewer":"evidence","verifier":"order"}` || w[1].String != evidence { + t.Errorf("review seated_by/seat_evidence = %+v, want the recorded values", w) + } + if w := got["bb22"]; w[0].Valid || w[1].Valid { + t.Errorf("a review that recorded none wrote %+v, want NULL on both", w) + } +} diff --git a/internal/ledger/row.go b/internal/ledger/row.go index 26c52874..4bc2cc8f 100644 --- a/internal/ledger/row.go +++ b/internal/ledger/row.go @@ -51,7 +51,16 @@ type Row struct { // existed and for a run that resolved no provider: not recorded, and a // reader must never put a provider guessed from a model's name in its // place. - SeatProviders string `json:",omitempty"` + SeatProviders string `json:",omitempty"` + // SeatedBy is how each seat's model was chosen, as JSON keyed by role + // like ModelsByRole: "operator" (named by flag), "evidence" (the recorded + // ranking picked it) or "order" (first of the pool the operator named). + // "" on an entry written without --models and before the field existed, + // both of which mean every seat was named by flag. + SeatedBy string `json:",omitempty"` + // SeatEvidence names what the ranking read to seat the "evidence" seats + // (for example "the machine ledger (31 scans)"); "" when nothing was read. + SeatEvidence string `json:",omitempty"` MinKillRate *float64 MaxProvenMissed *int Passed *bool // nil = not recorded; see ScanRow.Passed diff --git a/internal/ledger/tiers.go b/internal/ledger/tiers.go index 67232256..0f853609 100644 --- a/internal/ledger/tiers.go +++ b/internal/ledger/tiers.go @@ -60,6 +60,7 @@ const ( noteModelOpinion = "a model's opinion: advisory, signed but not verified as true" notePerson = "declared by the person who wrote it: their words, not verified as true" noteAdjudicator = "declared by the person who adjudicated" + noteSeatedBy = "how corral chose the seat: the operator's flag, the ranking, or the order the pool was named; a statement of the choice, not a measurement" noteSeatProvider = "the provider corral resolved and sent the seat's calls to; not a check of what served the model behind that endpoint, and for an agent seat a vendor label" ) @@ -152,6 +153,10 @@ func buildFieldTiers() map[[2]string]FieldTier { // declaration, exactly as ModelsByRole is corral's record of the named // model; and not META: it is not the run's narrative about itself. add("Row", TierMeasured, noteSeatProvider, "SeatProviders") + // SeatedBy and SeatEvidence record HOW a seat was chosen, which is + // corral's account of a decision, not something execution measured: + // DECLARED, as the operator's own choice of seating policy is. + add("Row", TierDeclared, noteSeatedBy, "SeatedBy SeatEvidence") add("Row", TierMeta, noteRunAccount, "ConcurrencyNote Detail Reason RunCache") add("Row", TierMeta, "carries the run's full verdict document, critic opinion included: signed, but not verified as true", "VerdictJSON") add("Row", TierMeta, noteModelOpinion, "VacuousFindings") @@ -197,6 +202,7 @@ func buildFieldTiers() map[[2]string]FieldTier { add("Review", TierMeta, noteModelOpinion, "Opinion Sound VerifierNote VerifierOpinion") // The same tier and note as Row.SeatProviders, for the same reason. add("Review", TierMeasured, noteSeatProvider, "SeatProviders") + add("Review", TierDeclared, noteSeatedBy, "SeatedBy SeatEvidence") add("Finding", TierKey, "", "ID") add("Finding", TierMeasured, "", "Demoted ExitCode Script Stdout Tier Unrun") add("Finding", TierMeta, noteModelOpinion, "Claim Declared File Line Severity") diff --git a/internal/reposcan/generation.go b/internal/reposcan/generation.go index 6138f7aa..2d5cee02 100644 --- a/internal/reposcan/generation.go +++ b/internal/reposcan/generation.go @@ -479,7 +479,17 @@ package reposcan // added to every file's selection instead (ramsey/uuid: 59 of 67 test files // reported before, 67 after). "46" was released in rc.34, so this gets its // own generation. -const VerdictGeneration = "47" +// +// "48" (2026-11-19): the Verdict gained SeatedBy and SeatEvidence: how each +// seat's model was chosen (operator flag, the recorded ranking, or the order +// the operator named the pool) and what the ranking read, for the run's +// --models proposal. A cached "47" document unmarshals both as absent, which +// means "every seat was named by flag": true of every verdict that +// generation could have produced, so it is not misread, but "47" is on main, +// so this is a shape bump and not a ride-along. The verdict cache key does +// not name how a seat was chosen, so a verdict served from cache carries the +// seating basis it was measured under. +const VerdictGeneration = "48" // VerdictShapeSHA256 fingerprints advpool.Verdict's serialized shape: every // exported field's name, type and json tag, sorted by name and hashed. @@ -501,4 +511,4 @@ const VerdictGeneration = "47" // Verdict's own FIELDS") and nothing wider. Nor can any fingerprint catch the // behaviour half: a scorer that computes a DIFFERENT number into the SAME // field is invisible here, and remains a review responsibility. -const VerdictShapeSHA256 = "21a4e1eafa4589ddc215061ac93b0295fb169f384a26fbb48cf951c2234cbff0" +const VerdictShapeSHA256 = "865ab8f5b25f36561eb7ce28d58891d6d5f39ae20a2f9cb768aa6f59bc4ed49e" diff --git a/internal/reviewrecord/reviewrecord.go b/internal/reviewrecord/reviewrecord.go index 4bc6e43d..44c95b64 100644 --- a/internal/reviewrecord/reviewrecord.go +++ b/internal/reviewrecord/reviewrecord.go @@ -84,6 +84,12 @@ type Review struct { // the field existed, and for a seat with no resolved provider: not // recorded, never inferred from a model's name. SeatProviders map[string]string `json:"seat_providers,omitempty"` + // SeatedBy is how each seat's model was chosen, keyed "reviewer" and + // "verifier" like SeatProviders: operator, evidence or order. Absent + // means named by flag. + SeatedBy map[string]string `json:"seated_by,omitempty"` + // SeatEvidence names what the ranking read to seat the "evidence" seats. + SeatEvidence string `json:"seat_evidence,omitempty"` // ReviewerModelResolved / VerifierModelResolved report whether the model // field beside them names an actual MODEL, or merely the tool that ran // one. An agentic seat given without a pin ("claude-code", "antigravity") From 0135fbf5b2e42cc78f51cc5354bbe6f79f221d74 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 11:33:21 -0500 Subject: [PATCH 03/14] Add seatFromPool, the one helper every door will call to turn --models into a proposed lineup, so the pool is parsed, resolved, credential-checked, ranked and offered the same way on every verb rather than once per door. Every member is resolved and checked before any evidence is read, so a pool that works today cannot fail on the day a different member wins. The plan prints before anything is spent; a refused lineup exits 2 without asking, a terminal is asked once, and with no terminal the lineup is printed as the explicit seat flags and nothing runs, because there is nobody to accept it. Only an accepted lineup writes the chosen members, as the operator typed them, into the door's own flag variables, so the door's normal resolution runs on them unchanged. The language has no default: a door passes the scan's language or shadowpool.LangAny, and a scan of several languages ranks each seat across all of them as one group. --models is nil until given, so an empty pool is refused rather than read as absent. golang.org/x/term becomes a direct requirement for the terminal check. The printed-metrics gate gains rows for the plan's numbers and for autoseat's floor, which the walk now reaches. No door is wired yet. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/models_pool.go | 343 ++++++++++++++++++++ cmd/corral/models_pool_test.go | 495 +++++++++++++++++++++++++++++ cmd/corral/printed_metrics_test.go | 6 + go.mod | 2 +- 4 files changed, 845 insertions(+), 1 deletion(-) create mode 100644 cmd/corral/models_pool.go create mode 100644 cmd/corral/models_pool_test.go diff --git a/cmd/corral/models_pool.go b/cmd/corral/models_pool.go new file mode 100644 index 00000000..657da88a --- /dev/null +++ b/cmd/corral/models_pool.go @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + + "golang.org/x/term" + + "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/autoseat" + "github.com/pdbethke/corralai/internal/modelrank" + "github.com/pdbethke/corralai/internal/seatrecord" + "github.com/pdbethke/corralai/internal/shadowpool" +) + +// modelsFlags is --models. --autoseat and --no-explore join it in phase 3, +// in this struct and this function, so no door can have one without the others. +// +// pool is nil until the flag is given. A plain string flag cannot tell +// `--models ""` from no flag at all, and an empty pool silently treated as +// absent would be a flag quietly ignored; nil is "not given", and anything +// given, empty included, is parsed and refused if it names nothing. +type modelsFlags struct{ pool *string } + +func registerModelsFlags(fs *flag.FlagSet, help string) *modelsFlags { + f := &modelsFlags{} + fs.Func("models", help, func(v string) error { + if f.pool != nil { + // The flag package would keep the last; dropping the first pool + // without a word is the silent substitution corral refuses. + return errors.New("given twice: name every member in one comma-separated list") + } + f.pool = &v + return nil + }) + return f +} + +// poolSeating is what a door records. +type poolSeating struct { + SeatedBy map[string]string // role -> operator|evidence|order; nil when --models was not given + SeatEvidence string // what the ranking read, as the plan's evidence line says it; "" when no seat was seated by evidence +} + +// poolRole ties a role to the flag variable the door already has. +type poolRole struct { + role string + val *string // rewritten in place with the chosen member's Given text +} + +// poolDeps are seatFromPool's seams. load reads the evidence and confirm asks +// the operator; both are injected so a test touches neither the machine +// ledger nor a terminal. +type poolDeps struct { + load rankLoader + // confirm is handed the plan exactly as it was printed. It reports + // whether the operator accepted and whether anyone could be asked at + // all: with no terminal there is nobody to accept, which is not a "no". + confirm func(plan string) (accepted bool, interactive bool, err error) + stdout io.Writer + stderr io.Writer +} + +// seatFromPool returns done=true with an exit code when the door must stop +// (refusal: 2; declined or no terminal: 0). With --models unset it returns +// a zero poolSeating and done=false, and changes nothing. +// +// The order is the spec's (5.3): the pool is parsed, then every member is +// resolved and credential-checked whether or not it will be seated, so a pool +// that works today does not fail the day a different member wins; only then +// is the evidence read, ranked and proposed. Nothing is spent before the +// operator accepts, and a lineup with an unfilled seat is never offered. +// +// lang is the scan's language, or shadowpool.LangAny for a scan of several +// (the marker certify --repo already hands the shadow draw). It has no +// default: "" is a door that forgot to say, and is refused. +func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, f *modelsFlags, roles []poolRole, deps poolDeps) (seating poolSeating, exit int, done bool) { + if f == nil || f.pool == nil { + return poolSeating{}, 0, false + } + refuse := func(format string, args ...any) (poolSeating, int, bool) { + fmt.Fprintf(deps.stderr, "%s: --models: %s; not run, nothing was spent\n", cmdName, fmt.Sprintf(format, args...)) + return poolSeating{}, 2, true + } + if strings.TrimSpace(lang) == "" { + return refuse("no language was given for this scan (pass the scan's language, or %q for a scan of several), so the evidence cannot be filtered", shadowpool.LangAny) + } + poolText := strings.TrimSpace(*f.pool) + typed, err := parseModelsPool(poolText) + if err != nil { + return refuse("%v", err) + } + pool, err := resolveModelsPool(ctx, cmdName, repoRoot, typed) + if err != nil { + return refuse("%v", err) + } + + pins := map[string]string{} + roleNames := make([]string, len(roles)) + for i, r := range roles { + roleNames[i] = r.role + if v := strings.TrimSpace(*r.val); v != "" { + pins[r.role] = v + } + } + if len(pins) == len(roles) { + // Nothing to propose: every seat is the operator's, so no evidence + // is read and nobody is asked. The pool was still checked above. + fmt.Fprintf(deps.stderr, "%s: every seat is pinned, so --models seats nothing; the pins run as named\n", cmdName) + by := map[string]string{} + for _, r := range roleNames { + by[r] = autoseat.ByOperator + } + return poolSeating{SeatedBy: by}, 0, false + } + + ev, err := deps.load(rankSource{Kind: rankSourceLedger}) + if err != nil { + return refuse("cannot read the evidence: %v — nothing was proposed", err) + } + report, evidence := rankPool(ev, lang, pool) + plan := autoseat.Propose(autoseat.Input{ + Roles: roleNames, Pool: pool, Pins: pins, Report: report, Floor: modelrank.DefaultMinRuns, + }) + + text := renderPoolPlan(poolText, plan, evidence, ev.Generations.Line()) + fmt.Fprint(deps.stdout, text) + if plan.Refused { + return refuse("the proposed lineup is refused (the plan above names each unfilled seat and what would fill it)") + } + + accepted, interactive, err := deps.confirm(text) + switch { + case err != nil: + return refuse("reading the answer: %v", err) + case !interactive: + // Nobody to ask, so nothing runs; the operator gets the lineup as + // flags to run deliberately. There is no --yes (spec 5.3). + var b strings.Builder + b.WriteString("to run this lineup:") + for _, s := range plan.Seats { + if s.By != autoseat.ByOperator { + b.WriteString(" --" + autoseat.PinFlag[s.Role] + " " + s.Given) + } + } + fmt.Fprintln(deps.stdout, b.String()) + return poolSeating{}, 0, true + case !accepted: + fmt.Fprintln(deps.stdout, "not run; nothing was spent") + return poolSeating{}, 0, true + } + + seating.SeatedBy = map[string]string{} + for i, s := range plan.Seats { + seating.SeatedBy[s.Role] = s.By + if s.By == autoseat.ByOperator { + continue + } + // The member as the operator typed it: the door's own resolution + // runs next and rewrites an alias or vendor/model exactly as it + // would have had the operator typed this flag. + *roles[i].val = s.Given + if s.By == autoseat.ByEvidence { + seating.SeatEvidence = evidence + } + } + return seating, 0, false +} + +// parseModelsPool splits --models. shadowpool.ParsePool is not reused: it +// requires two members, because a challenger pool of one is no draw, whereas +// a --models pool of one is a legitimate request that seats every seat but +// the critic (the proposer refuses that seat and says so). Its other +// refusals are the same three, here: an empty member, a member named twice, +// and "off", which disables a seat and is not a model. +func parseModelsPool(text string) ([]string, error) { + var out []string + seen := map[string]bool{} + for _, raw := range strings.Split(text, ",") { + m := strings.TrimSpace(raw) + switch { + case m == "": + return nil, fmt.Errorf("--models %q has an empty member", text) + case strings.EqualFold(m, "off"): + return nil, errors.New(`--models cannot contain "off": the pool names models; to disable a seat, pin it off with its own flag (--critic-model off)`) + case seen[strings.ToLower(m)]: + return nil, fmt.Errorf("--models names %q twice", m) + } + seen[strings.ToLower(m)] = true + out = append(out, m) + } + return out, nil +} + +// resolveModelsPool resolves each member as its own seat, with one shared +// listing of OLLAMA_URL's daemon (shadowCandidates' pattern), and checks each +// with shadowMemberRunnable, the one per-member credential check. The +// resolver rewrites the value it is handed, so it resolves a copy and the +// member keeps what the operator typed as Given. +func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []string) ([]autoseat.Member, error) { + listing := &ollamaListing{} + byRecorded := map[string]string{} + var out []autoseat.Member + for _, given := range typed { + concrete := given + set, err := resolveSeatsListed(ctx, cmdName, repoRoot, []seatFlag{{flag: "models", val: &concrete}}, nil, io.Discard, listing) + if err != nil { + return nil, fmt.Errorf("member %q: %v", given, err) + } + rs := set["models"] + if err := shadowMemberRunnable(rs); err != nil { + return nil, fmt.Errorf("member %q cannot run: %v — every member is checked before anything is proposed, so fix or remove it", given, err) + } + // Two spellings of one model (an alias and its concrete name) would + // make the ranking's row ambiguous between them. + if prev, dup := byRecorded[strings.ToLower(rs.Spec)]; dup { + return nil, fmt.Errorf("members %q and %q are both the model %q; name it once", prev, given, rs.Spec) + } + byRecorded[strings.ToLower(rs.Spec)] = given + // A bare agent seat names a tool, not a model, so the record ranks + // nothing under it: it can be seated by order or a pin, never by + // evidence. + tool := "" + if isAgentSeat(rs.Spec) { + tool, _, _ = agentbackend.AgentSeat(rs.Spec) + } + out = append(out, autoseat.Member{ + Given: given, Recorded: rs.Spec, Provider: rs.Provider, + Rankable: seatrecord.RecordedSeatNamesAModel(rs.Spec, tool), + }) + } + return out, nil +} + +// rankPool ranks the evidence for this scan and returns the report with the +// plan's evidence line (the source verbatim, then the language it covers). +// +// A scan of several languages ranks across all of them: the language is +// cleared on a copy of the observations, so each seat is one group rather +// than one per language (the proposer reads a seat's first group only). +// Evidence that records no language cannot be filtered to one, and says so +// rather than ranking nothing. +func rankPool(ev rankEvidence, lang string, pool []autoseat.Member) (modelrank.Report, string) { + declared := map[string]string{} + for _, m := range pool { + declared[m.Recorded] = m.Given + } + opt := modelrank.Options{MinRuns: modelrank.DefaultMinRuns, Declared: declared, Source: ev.Source} + obs := ev.Obs + recordsLang := false + for _, o := range obs { + if strings.TrimSpace(o.Lang) != "" { + recordsLang = true + break + } + } + var scope string + switch { + case lang == shadowpool.LangAny: + obs = make([]modelrank.Observation, len(ev.Obs)) + copy(obs, ev.Obs) + for i := range obs { + obs[i].Lang = "" + } + scope = "all languages (this scan has more than one)" + case !recordsLang: + scope = "all languages (the evidence records none)" + default: + opt.Lang = lang + scope = "language " + lang + } + return modelrank.Rank(obs, opt), ev.Source + "; " + scope +} + +// renderPoolPlan is the plan as spec 5.3 draws it: one row per role (role, +// model, how it was chosen, why), then what the ranking read and the grading +// rules it spans. Columns are aligned by tabwriter, so a long model name +// widens its column instead of breaking the table. +func renderPoolPlan(poolText string, plan autoseat.Plan, evidence, generations string) string { + var b strings.Builder + b.WriteString("seats (from --models " + poolText + "):\n") + tw := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0) + for _, s := range plan.Seats { + if s.Unfilled != "" { + fmt.Fprintf(tw, " %s\t—\tREFUSED\t%s\n", s.Role, s.Unfilled) + continue + } + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\n", s.Role, seatDisplay(s), s.By, s.Reason) + } + _ = tw.Flush() + if plan.Why != "" { + b.WriteString("refused: " + plan.Why + "\n") + } + b.WriteString("evidence: " + evidence + "\n") + if generations != "" { + b.WriteString("grading rules: " + generations + "\n") + } + return b.String() +} + +// seatDisplay is the model a seat runs, by the name the record ranks it +// under; a pinned agent seat reads "model (agent)". +func seatDisplay(s autoseat.Seat) string { + name := s.Recorded + if name == "" { + name = s.Given + } + if agent, model, ok := agentbackend.AgentSeat(name); ok && model != "" { + return model + " (" + agent + ")" + } + return name +} + +// terminalConfirm asks on in, when in is a terminal, after the caller has +// printed the plan (so the plan argument is not printed again). Anything but +// y or yes declines. When in is not a terminal it asks nothing and reports +// interactive=false: a pipe or a CI log cannot accept a lineup. +func terminalConfirm(in *os.File, out io.Writer) func(plan string) (bool, bool, error) { + return func(string) (bool, bool, error) { + if !term.IsTerminal(int(in.Fd())) { // #nosec G115 -- a file descriptor fits in an int + return false, false, nil + } + fmt.Fprint(out, "run with these seats? [y/N] ") + line, err := bufio.NewReader(in).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return false, true, err + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + return true, true, nil + } + return false, true, nil + } +} diff --git a/cmd/corral/models_pool_test.go b/cmd/corral/models_pool_test.go new file mode 100644 index 00000000..681f0f17 --- /dev/null +++ b/cmd/corral/models_pool_test.go @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + "testing" + + "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/modelrank" + "github.com/pdbethke/corralai/internal/models" + "github.com/pdbethke/corralai/internal/shadowpool" +) + +// The pool every test names unless it says otherwise: the one in the spec's +// sample plan (5.3). +const samplePool = "anthropic/claude-sonnet-5-5,anthropic/claude-haiku-5-5,codex:gpt-6" + +const sampleSource = "the machine ledger /state/ledger (31 scan(s), 2 review(s); a run is one SCAN)" + +// poolEnv isolates the resolver from the machine: no registry, no gateway, no +// daemon, and a credential check that passes, so no test reads a real key. +// It returns the repo root to resolve against (an empty directory, so no +// checkout registry exists). +func poolEnv(t *testing.T) string { + t.Helper() + t.Setenv(models.EnvInline, "") + t.Setenv(models.EnvFile, "") + t.Setenv("MODEL_BACKEND", "") + t.Setenv("OLLAMA_URL", "") + orig := shadowMemberRunnable + t.Cleanup(func() { shadowMemberRunnable = orig }) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } + return t.TempDir() +} + +// certifyRoles are the four certify seats as a door hands them over: each +// role tied to the variable its own flag parsed into. +type certifyVals struct{ derive, mutant, writer, critic string } + +func (v *certifyVals) roles() []poolRole { + return []poolRole{ + {role: modelrank.SeatGoalDeriver, val: &v.derive}, + {role: modelrank.SeatMutantGenerator, val: &v.mutant}, + {role: modelrank.SeatTestWriter, val: &v.writer}, + {role: modelrank.SeatTestCritic, val: &v.critic}, + } +} + +// genObs is one generator run per run name: 10 planted, 9 graded, 1 invalid, +// 4 survived. +func genObs(model, lang string, runs ...string) []modelrank.Observation { + var out []modelrank.Observation + for _, r := range runs { + out = append(out, modelrank.Observation{Model: model, Role: modelrank.SeatMutantGenerator, Lang: lang, Run: r, + MutantsPlanted: 10, MutantsGraded: 9, MutantsInvalid: 1, MutantsSurvived: 4}) + } + return out +} + +func writerObs(model, lang, run string, proven, attempted int) modelrank.Observation { + return modelrank.Observation{Model: model, Role: modelrank.SeatTestWriter, Lang: lang, Run: run, + Catches: proven, Opportunities: attempted} +} + +// sampleEvidence is the spec's sample: claude-sonnet-5-5 measured over six +// python runs in both measured seats, 193/195 as writer. +func sampleEvidence() rankEvidence { + runs := []string{"r1", "r2", "r3", "r4", "r5", "r6"} + obs := genObs("claude-sonnet-5-5", "python", runs...) + for _, r := range runs[:5] { + obs = append(obs, writerObs("claude-sonnet-5-5", "python", r, 32, 32)) + } + obs = append(obs, writerObs("claude-sonnet-5-5", "python", "r6", 33, 35)) + return rankEvidence{Obs: obs, Source: sampleSource, + Generations: generationRange{Min: "40", Max: "43", Scans: 6}} +} + +// fakeConfirm records whether it was asked, and with what plan. +type fakeConfirm struct { + accepted, interactive bool + err error + calls int + plan string +} + +func (c *fakeConfirm) fn(plan string) (bool, bool, error) { + c.calls++ + c.plan = plan + return c.accepted, c.interactive, c.err +} + +// fakeLoad returns ev (or err) and records every source it was asked for. +type fakeLoad struct { + ev rankEvidence + err error + asked []rankSource +} + +func (l *fakeLoad) fn(src rankSource) (rankEvidence, error) { + l.asked = append(l.asked, src) + return l.ev, l.err +} + +// planRow is the plan's row for role with its column padding collapsed, for a +// test that asserts a row's cells rather than the whole table's alignment. +func planRow(plan, role string) string { + for _, line := range strings.Split(plan, "\n") { + if f := strings.Fields(line); len(f) > 0 && f[0] == role { + return strings.Join(f, " ") + } + } + return "" +} + +func poolFlags(pool string) *modelsFlags { return &modelsFlags{pool: &pool} } + +func runPool(t *testing.T, root, lang, pool string, vals *certifyVals, load *fakeLoad, confirm *fakeConfirm) (poolSeating, int, bool, string, string) { + t.Helper() + var out, errb bytes.Buffer + f := &modelsFlags{} + if pool != "" { + f = poolFlags(pool) + } + seating, exit, done := seatFromPool(context.Background(), "corral certify --local", root, lang, f, vals.roles(), + poolDeps{load: load.fn, confirm: confirm.fn, stdout: &out, stderr: &errb}) + return seating, exit, done, out.String(), errb.String() +} + +func TestSeatFromPoolUnsetChangesNothing(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{mutant: "anthropic/claude-sonnet-5-5"} + load, confirm := &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, errs := runPool(t, root, "python", "", vals, load, confirm) + if done || exit != 0 { + t.Fatalf("done=%v exit=%d, want a door that carries on", done, exit) + } + if seating.SeatedBy != nil || seating.SeatEvidence != "" { + t.Errorf("seating = %+v, want zero", seating) + } + if *vals != (certifyVals{mutant: "anthropic/claude-sonnet-5-5"}) { + t.Errorf("flag values changed: %+v", *vals) + } + if len(load.asked) != 0 || confirm.calls != 0 || out != "" || errs != "" { + t.Errorf("without --models nothing may be read, asked or printed: load=%v confirm=%d out=%q err=%q", load.asked, confirm.calls, out, errs) + } +} + +func TestSeatFromPoolWritesChosenSeatsIntoTheFlags(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{} + load, confirm := &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, errs := runPool(t, root, "python", samplePool, vals, load, confirm) + if done || exit != 0 { + t.Fatalf("done=%v exit=%d stderr=%q, want an accepted lineup to carry on", done, exit, errs) + } + want := certifyVals{ + derive: "anthropic/claude-sonnet-5-5", mutant: "anthropic/claude-sonnet-5-5", + writer: "anthropic/claude-sonnet-5-5", critic: "codex:gpt-6", + } + if *vals != want { + t.Errorf("flags = %+v, want each set to the chosen member as the operator typed it: %+v", *vals, want) + } + wantBy := map[string]string{ + modelrank.SeatGoalDeriver: "order", modelrank.SeatMutantGenerator: "evidence", + modelrank.SeatTestWriter: "evidence", modelrank.SeatTestCritic: "order", + } + if fmt.Sprint(seating.SeatedBy) != fmt.Sprint(wantBy) { + t.Errorf("SeatedBy = %v, want %v", seating.SeatedBy, wantBy) + } + wantEvidence := sampleSource + "; language python" + if seating.SeatEvidence != wantEvidence { + t.Errorf("SeatEvidence = %q, want %q", seating.SeatEvidence, wantEvidence) + } + wantPlan := strings.Join([]string{ + "seats (from --models " + samplePool + "):", + " goal-deriver claude-sonnet-5-5 order no measurement for this seat: taken from the order you named", + " mutant-generator claude-sonnet-5-5 evidence 60 planted, 54 graded, 6 invalid, 24 survived over 6 runs", + " test-writer claude-sonnet-5-5 evidence 193/195 survivors proven over 6 runs", + " test-critic gpt-6 (codex) order no measurement for this seat: taken from the order you named", + "evidence: " + sampleSource + "; language python", + "grading rules: generations 40 to 43 across 6 scan(s) with evidence", + "", + }, "\n") + if out != wantPlan { + t.Errorf("plan:\n%s\nwant:\n%s", out, wantPlan) + } + if confirm.calls != 1 || confirm.plan != wantPlan { + t.Errorf("confirm asked %d time(s) with plan %q, want once with the printed plan", confirm.calls, confirm.plan) + } + if len(load.asked) != 1 || load.asked[0] != (rankSource{Kind: rankSourceLedger}) { + t.Errorf("evidence read from %v, want the machine ledger once", load.asked) + } + + // A pin is the operator's: recorded as such, printed as such, and left + // exactly as typed. + vals = &certifyVals{critic: "off"} + confirm = &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, errs = runPool(t, root, "python", samplePool, vals, &fakeLoad{ev: sampleEvidence()}, confirm) + if done || exit != 0 { + t.Fatalf("pinned critic: done=%v exit=%d stderr=%q", done, exit, errs) + } + if seating.SeatedBy[modelrank.SeatTestCritic] != "operator" || vals.critic != "off" { + t.Errorf("pinned critic: SeatedBy=%v critic=%q, want operator and off", seating.SeatedBy, vals.critic) + } + if got := planRow(out, modelrank.SeatTestCritic); got != "test-critic off operator pinned by --critic-model" { + t.Errorf("pinned critic row missing from the plan:\n%s", out) + } +} + +func TestSeatFromPoolRefusesBelowTheFloor(t *testing.T) { + root := poolEnv(t) + ev := sampleEvidence() + // Writer evidence below the floor of five survivors attempted. + var obs []modelrank.Observation + for _, o := range ev.Obs { + if o.Role != modelrank.SeatTestWriter { + obs = append(obs, o) + } + } + ev.Obs = append(obs, writerObs("claude-sonnet-5-5", "python", "r1", 3, 4)) + vals := &certifyVals{} + confirm := &fakeConfirm{accepted: true, interactive: true} + _, exit, done, out, errs := runPool(t, root, "python", samplePool, vals, &fakeLoad{ev: ev}, confirm) + if exit != 2 || !done { + t.Fatalf("exit=%d done=%v, want a refusal (2, done)", exit, done) + } + if confirm.calls != 0 { + t.Errorf("a refused lineup must never be offered; confirm was called %d time(s)", confirm.calls) + } + if *vals != (certifyVals{}) { + t.Errorf("a refusal changed the flags: %+v", *vals) + } + if !strings.Contains(out, "REFUSED") || !strings.Contains(out, "pin --writer-model") { + t.Errorf("plan must mark the unfilled seat and name its fix:\n%s", out) + } + if got := planRow(out, modelrank.SeatTestWriter); got != "test-writer — REFUSED no model in --models has 5+ survivors attempted as test-writer; pin --writer-model" { + t.Errorf("refused row has the wrong shape: %q\n%s", got, out) + } + if !strings.Contains(errs, "nothing was spent") { + t.Errorf("stderr must say nothing was spent: %q", errs) + } +} + +func TestSeatFromPoolNoEvidenceYet(t *testing.T) { + root := poolEnv(t) + src := "the machine ledger /state/ledger (no entries yet)" + load := &fakeLoad{ev: rankEvidence{Source: src}} + confirm := &fakeConfirm{accepted: true, interactive: true} + vals := &certifyVals{} + _, exit, done, out, _ := runPool(t, root, "python", samplePool, vals, load, confirm) + if exit != 2 || !done || confirm.calls != 0 { + t.Fatalf("exit=%d done=%v confirm=%d, want refused before any question", exit, done, confirm.calls) + } + if !strings.Contains(out, src) { + t.Errorf("the refusal must name what was read (%q):\n%s", src, out) + } + if len(load.asked) != 1 || load.asked[0].Kind != rankSourceLedger { + t.Errorf("load asked %v, want the ledger once and no other source", load.asked) + } + if *vals != (certifyVals{}) { + t.Errorf("flags changed: %+v", *vals) + } +} + +func TestSeatFromPoolBadPool(t *testing.T) { + root := poolEnv(t) + cases := []struct{ pool, want string }{ + {"anthropic/claude-sonnet-5-5,,codex:gpt-6", "empty member"}, + {"anthropic/claude-sonnet-5-5,anthropic/claude-sonnet-5-5", `"anthropic/claude-sonnet-5-5" twice`}, + {"anthropic/claude-sonnet-5-5,off", `"off"`}, + {"anthropic/claude-sonnet-5-5,claude-haiku-5-5", `member "claude-haiku-5-5"`}, + } + for _, c := range cases { + load, confirm := &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{accepted: true, interactive: true} + vals := &certifyVals{} + _, exit, done, out, errs := runPool(t, root, "python", c.pool, vals, load, confirm) + if exit != 2 || !done { + t.Errorf("%q: exit=%d done=%v, want 2 and done", c.pool, exit, done) + } + if !strings.Contains(errs, c.want) { + t.Errorf("%q: stderr %q does not name the member (%s)", c.pool, errs, c.want) + } + if len(load.asked) != 0 || confirm.calls != 0 || out != "" || *vals != (certifyVals{}) { + t.Errorf("%q: a bad pool must stop before evidence, plan or question: load=%v confirm=%d out=%q vals=%+v", + c.pool, load.asked, confirm.calls, out, *vals) + } + } + + // A member whose credential is missing is refused up front, drawn or not. + shadowMemberRunnable = func(rs agentbackend.ResolvedSeat) error { + if rs.Spec == "claude-haiku-5-5" { + return errors.New("ANTHROPIC_API_KEY is not set") + } + return nil + } + _, exit, done, _, errs := runPool(t, root, "python", samplePool, &certifyVals{}, &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{}) + if exit != 2 || !done || !strings.Contains(errs, `member "anthropic/claude-haiku-5-5" cannot run`) || !strings.Contains(errs, "ANTHROPIC_API_KEY") { + t.Errorf("credential: exit=%d done=%v stderr=%q", exit, done, errs) + } +} + +func TestSeatFromPoolAllPinned(t *testing.T) { + root := poolEnv(t) + pinned := certifyVals{derive: "a", mutant: "b", writer: "c", critic: "off"} + vals := pinned + load, confirm := &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, _ := runPool(t, root, "python", samplePool, &vals, load, confirm) + if done || exit != 0 { + t.Fatalf("done=%v exit=%d, want the door to carry on", done, exit) + } + if confirm.calls != 0 || len(load.asked) != 0 { + t.Errorf("every seat is pinned: nothing to propose, so no evidence and no question (load=%v confirm=%d)", load.asked, confirm.calls) + } + for _, r := range vals.roles() { + if seating.SeatedBy[r.role] != "operator" { + t.Errorf("SeatedBy[%s] = %q, want operator", r.role, seating.SeatedBy[r.role]) + } + } + if vals != pinned || seating.SeatEvidence != "" || out != "" { + t.Errorf("vals=%+v evidence=%q out=%q, want the pins untouched, no evidence and no plan", vals, seating.SeatEvidence, out) + } +} + +func TestSeatFromPoolNoTerminal(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{critic: "off"} + confirm := &fakeConfirm{interactive: false} + _, exit, done, out, _ := runPool(t, root, "python", samplePool, vals, &fakeLoad{ev: sampleEvidence()}, confirm) + if exit != 0 || !done { + t.Fatalf("exit=%d done=%v, want 0 and done: no terminal means nothing runs", exit, done) + } + want := "to run this lineup: --derive-model anthropic/claude-sonnet-5-5 --mutant-model anthropic/claude-sonnet-5-5 --writer-model anthropic/claude-sonnet-5-5\n" + if !strings.HasSuffix(out, want) { + t.Errorf("stdout must end with the explicit flags for the proposed (not pinned) seats:\n%s\nwant suffix %q", out, want) + } + if !strings.HasPrefix(out, "seats (from --models ") { + t.Errorf("the plan must come first:\n%s", out) + } + if *vals != (certifyVals{critic: "off"}) { + t.Errorf("flags changed: %+v", *vals) + } +} + +func TestSeatFromPoolDeclined(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{} + _, exit, done, out, _ := runPool(t, root, "python", samplePool, vals, &fakeLoad{ev: sampleEvidence()}, + &fakeConfirm{accepted: false, interactive: true}) + if exit != 0 || !done { + t.Fatalf("exit=%d done=%v, want 0 and done", exit, done) + } + if !strings.HasSuffix(out, "not run; nothing was spent\n") { + t.Errorf("stdout must end with the decline:\n%s", out) + } + if *vals != (certifyVals{}) { + t.Errorf("a declined lineup changed the flags: %+v", *vals) + } +} + +func TestSeatFromPoolAlias(t *testing.T) { + root := poolEnv(t) + t.Setenv(models.EnvInline, `{"edge": {"provider": "ollama", "model": "localmodel-a:1", "endpoint": "http://127.0.0.1:11434"}}`) + var obs []modelrank.Observation + obs = append(obs, genObs("localmodel-a:1", "python", "r1", "r2", "r3", "r4", "r5")...) + for _, r := range []string{"r1", "r2", "r3", "r4", "r5"} { + obs = append(obs, writerObs("localmodel-a:1", "python", r, 2, 2)) + } + vals := &certifyVals{derive: "anthropic/claude-haiku-5-5", critic: "off"} + confirm := &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, errs := runPool(t, root, "python", "edge,anthropic/claude-haiku-5-5", vals, + &fakeLoad{ev: rankEvidence{Obs: obs, Source: sampleSource}}, confirm) + if done || exit != 0 { + t.Fatalf("done=%v exit=%d stderr=%q out=%s", done, exit, errs, out) + } + if vals.writer != "edge" || vals.mutant != "edge" { + t.Errorf("writer=%q mutant=%q, want the alias as typed (the door's own resolution rewrites it)", vals.writer, vals.mutant) + } + if seating.SeatedBy[modelrank.SeatTestWriter] != "evidence" { + t.Errorf("SeatedBy = %v, want the writer seated by evidence under its concrete model", seating.SeatedBy) + } + if got := planRow(out, modelrank.SeatTestWriter); got != "test-writer localmodel-a:1 evidence 10/10 survivors proven over 5 runs" { + t.Errorf("the plan must show the concrete model the evidence ranks:\n%s", out) + } +} + +func TestSeatFromPoolEvidenceUnreadable(t *testing.T) { + root := poolEnv(t) + confirm := &fakeConfirm{accepted: true, interactive: true} + vals := &certifyVals{} + _, exit, done, out, errs := runPool(t, root, "python", samplePool, vals, + &fakeLoad{err: errors.New("the machine ledger /state/ledger: chain does not verify")}, confirm) + if exit != 2 || !done { + t.Fatalf("exit=%d done=%v, want a refusal", exit, done) + } + if !strings.Contains(errs, "chain does not verify") { + t.Errorf("stderr must carry the loader's error: %q", errs) + } + if out != "" || confirm.calls != 0 || *vals != (certifyVals{}) { + t.Errorf("nothing may be proposed: out=%q confirm=%d vals=%+v", out, confirm.calls, *vals) + } +} + +// The language line: a scan of one language ranks that language when the +// evidence records languages; a scan of several (shadowpool.LangAny, the +// marker certify --repo already passes) ranks across all of them; evidence +// that records no language is across all languages and says so. +func TestSeatFromPoolLanguageLine(t *testing.T) { + root := poolEnv(t) + stripped := sampleEvidence() + for i := range stripped.Obs { + stripped.Obs[i].Lang = "" + } + cases := []struct { + lang string + ev rankEvidence + want string + }{ + {shadowpool.LangAny, sampleEvidence(), "; all languages (this scan has more than one)\n"}, + {"python", stripped, "; all languages (the evidence records none)\n"}, + } + for _, c := range cases { + confirm := &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, errs := runPool(t, root, c.lang, samplePool, &certifyVals{}, &fakeLoad{ev: c.ev}, confirm) + if done || exit != 0 { + t.Errorf("%s: done=%v exit=%d stderr=%q\n%s", c.lang, done, exit, errs, out) + continue + } + if !strings.Contains(out, "evidence: "+sampleSource+c.want) || !strings.HasSuffix(seating.SeatEvidence, strings.TrimSuffix(c.want, "\n")) { + t.Errorf("%s: evidence line wrong (SeatEvidence %q):\n%s", c.lang, seating.SeatEvidence, out) + } + } + + // Across languages means one ranking per seat over all of them: a writer + // with 3 survivors attempted in python and 3 in go has 6, above the + // floor, though neither language alone clears it. + split := sampleEvidence() + var obs []modelrank.Observation + for _, o := range split.Obs { + if o.Role != modelrank.SeatTestWriter { + obs = append(obs, o) + } + } + split.Obs = append(obs, writerObs("claude-sonnet-5-5", "python", "r1", 3, 3), writerObs("claude-sonnet-5-5", "go", "r2", 3, 3)) + seating, exit, done, out, errs := runPool(t, root, shadowpool.LangAny, samplePool, &certifyVals{}, &fakeLoad{ev: split}, + &fakeConfirm{accepted: true, interactive: true}) + if done || exit != 0 || seating.SeatedBy[modelrank.SeatTestWriter] != "evidence" { + t.Errorf("languages pooled: done=%v exit=%d SeatedBy=%v, want the writer seated from both languages (stderr %q)\n%s", done, exit, seating.SeatedBy, errs, out) + } + + // No language at all is a door's bug, not a choice to guess at. + _, exit, done, _, errs = runPool(t, root, "", samplePool, &certifyVals{}, &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{}) + if exit != 2 || !done || !strings.Contains(errs, "language") { + t.Errorf("empty language: exit=%d done=%v stderr=%q, want a refusal naming it", exit, done, errs) + } +} + +func TestRegisterModelsFlags(t *testing.T) { + fs := flag.NewFlagSet("t", flag.ContinueOnError) + fs.SetOutput(io.Discard) + f := registerModelsFlags(fs, "the pool") + if f.pool != nil { + t.Fatalf("pool set before parsing: %q", *f.pool) + } + if err := fs.Parse([]string{"--models", "a,b"}); err != nil || f.pool == nil || *f.pool != "a,b" { + t.Fatalf("parse: err=%v pool=%v", err, f.pool) + } + fs = flag.NewFlagSet("t", flag.ContinueOnError) + fs.SetOutput(io.Discard) + registerModelsFlags(fs, "the pool") + if err := fs.Parse([]string{"--models", "a", "--models", "b"}); err == nil { + t.Errorf("--models given twice was accepted; one would be silently dropped") + } +} + +func TestTerminalConfirmNotATerminal(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + defer w.Close() + var out bytes.Buffer + accepted, interactive, err := terminalConfirm(r, &out)("plan") + if accepted || interactive || err != nil || out.Len() != 0 { + t.Errorf("a pipe is not a terminal: accepted=%v interactive=%v err=%v out=%q", accepted, interactive, err, out.String()) + } +} diff --git a/cmd/corral/printed_metrics_test.go b/cmd/corral/printed_metrics_test.go index f3748f93..ed44c5a2 100644 --- a/cmd/corral/printed_metrics_test.go +++ b/cmd/corral/printed_metrics_test.go @@ -428,6 +428,11 @@ var printedMetrics = map[string]printedRow{ "cmd/corral/ledger.go:runLedgerPush": {Exempt: "ledger push result: counts of entries pushed, skipped and left out, the operation's own output; the rows are the record"}, "cmd/corral/matrix.go:runMatrix": {Fields: []string{"MutantRow.KilledBy", "MutantRow.Outcome"}}, + // The --models plan prints each evidence seat's modelrank.Row.Evidence + // text (the same counts models rank prints) and the evidence line, whose + // scan count is recorded verbatim as SeatEvidence. + "cmd/corral/models_pool.go:seatFromPool": {Fields: []string{"SeatRow.Catches", "SeatRow.Opportunities", "SeatRow.MutantsPlanted", "SeatRow.MutantsSurvived", "Row.SeatEvidence"}, Exempt: "refusal text on stderr carries causes via %v; no measured number"}, + "cmd/corral/models_pool.go:resolveModelsPool": {Exempt: "error message: a pool member's resolution or credential failure, a string"}, "cmd/corral/models_rank.go:printRankReport": {Fields: []string{"SeatRow.Catches", "SeatRow.Opportunities", "SeatRow.ParseRetries"}, Exempt: "registry model count and --min-runs threshold are the operator's inputs"}, "cmd/corral/models_rank.go:scanGenerationsByUID": {Fields: []string{"ScanRow.VerdictGeneration"}, Exempt: "formats the highest recorded generation per scan back to text; it prints nothing else"}, "cmd/corral/models_rank.go:summarizeGenerations": {Fields: []string{"ScanRow.VerdictGeneration"}, Exempt: "formats the lowest and highest recorded generation back to text; the scan count is the evidence-size count, not a record field"}, @@ -493,6 +498,7 @@ var printedMetrics = map[string]printedRow{ "cmd/corral/verify_attest.go:runVerifyLedger": {Exempt: "verify result: entries checked and entries with a problem, computed at verify time"}, "cmd/corral/verify_attest.go:verifyReviewEntry": {Exempt: exReview}, + "internal/autoseat/autoseat.go:seat": {Exempt: "the evidence floor in a refusal is modelrank.DefaultMinRuns, a constant the caller passes, not a measurement"}, "internal/hoststat/hoststat.go:unreadable": {Exempt: "error message: the reason a host figure could not be read, a string (the figure itself is on the record as nil with HostNotMeasured)"}, "internal/pricing/pricing.go:BasisPartlyUnmetered": {Fields: []string{"ModelCallRow.Calls", "ModelCallRow.UnmeteredCalls", "ModelCallRow.CostBasis"}}, "internal/hoststat/hoststat_linux.go:usage": {Exempt: "error message: the errno of a failed getrusage, a string"}, diff --git a/go.mod b/go.mod index 3f11837b..7ea84ea0 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( golang.org/x/crypto v0.59.0 golang.org/x/net v0.62.0 golang.org/x/sys v0.50.0 + golang.org/x/term v0.48.0 golang.org/x/time v0.16.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 @@ -116,7 +117,6 @@ require ( golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.25.0 // indirect golang.org/x/telemetry v0.0.0-20261009151841-97e8ea77aa2a // indirect - golang.org/x/term v0.48.0 // indirect golang.org/x/text v0.44.0 // indirect golang.org/x/tools v0.52.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect From 3a19921995e043e5cdce45b8907f82e8a4a2c944 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 12:04:54 -0500 Subject: [PATCH 04/14] Wire --models into certify --local, certify --repo, doctor and demo, and record on each written entry how every seat was chosen, so a lineup the evidence proposed is never signed as one the operator named. Each door registers the flag through registerModelsFlags and calls seatFromPool once, after flag parsing and before the shadow draw and seat resolution, so a proposed member is drawn against and resolved exactly as the same model typed into its flag would be. - certify --local ranks by the audited file's language (drawLang) and refuses at the door, naming --lang, when none can be detected. The seating travels localAuditInput -> RunSpec -> Verdict -> ledger row. - certify --repo ranks by the scan's language (call C7): the set the evidence pass instruments, from the walk the scan already makes. The seat block moved, unchanged, below that walk rather than walking twice; the --scope-tests refusal moved with it so it still follows seat resolution. Rows carry SeatedBy narrowed to the roster, as providers are. --dry-run spends nothing and asks for no credential, so there the pool is not credential-checked and the plan is shown without a question. - doctor prints the lineup certify would propose, says it ran nothing, and checks those seats. Its one paid check, the derive probe, now runs only for a deriver the operator named, never one a pool proposed. - demo forwards the pool to the real certify --local, which proposes and records; proposing in demo and passing plain flags on would record the seats as operator-named. The production loader and terminal confirm sit behind doorPoolDeps so no door test reads the machine ledger or a terminal. Gate tests require every seat-flag door to register --models (review.go is a named pending exemption that fails once Task 5 wires it) and keep autoseat imported by models_pool.go alone. seatProvidersJSON becomes roleMapJSON, since it now serialises SeatedBy too. A run without --models writes entries shaped as before, asserted by reading them back. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/certify_local.go | 31 +++- cmd/corral/certify_local_entry.go | 5 +- cmd/corral/certify_repo.go | 184 ++++++++++++------- cmd/corral/certify_repo_bundle.go | 12 +- cmd/corral/demo.go | 31 +++- cmd/corral/doctor.go | 34 +++- cmd/corral/models_doors_test.go | 151 ++++++++++++++++ cmd/corral/models_flags_test.go | 124 +++++++++++++ cmd/corral/models_pool.go | 51 +++++- cmd/corral/seat_resolver.go | 12 +- cmd/corral/seated_by_record_test.go | 220 +++++++++++++++++++++++ docs/cli/corral.md | 8 + site/src/content/docs/docs/cli/corral.md | 8 + testdata/executed-surfaces.tsv | 4 + 14 files changed, 786 insertions(+), 89 deletions(-) create mode 100644 cmd/corral/models_doors_test.go create mode 100644 cmd/corral/models_flags_test.go create mode 100644 cmd/corral/seated_by_record_test.go diff --git a/cmd/corral/certify_local.go b/cmd/corral/certify_local.go index 67dc84a6..9bf14b14 100644 --- a/cmd/corral/certify_local.go +++ b/cmd/corral/certify_local.go @@ -143,6 +143,7 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write writerPool: "a comma-separated POOL of challenger WRITER models, drawn per run exactly as --shadow-pool is. Mutually exclusive with --shadow-writer-model. OFF unless named; NEVER gates the verdict", seed: shadowSeedReplayHelp, }) + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool") writerModeFlag := fs.String("writer-mode", "", "how the test-writer attacks this file's survivors: `per-survivor` (the default) makes ONE call per survivor — each carrying the file once as a cacheable shared prefix plus that survivor's diff, each repaired on its own budget and each PROVEN ALONE against its own mutant — or `batched`, the original shape: one call carrying every survivor, one repair budget, one proof pass over all of them. Nothing measured changes between them (a survivor is proven iff an authored test kills it alone and passes on the original, either way); what changes is that one unbuildable test no longer spends the whole file's retries and takes every other survivor down with it. Each survivor's proof in per-survivor mode runs its OWN compliant baseline (a compliant pass plus a canary, per seat), so a file with N survivors pays N baselines where batched paid one: on a repo whose suite takes a minute, prefer --writer-mode batched or expect N baselines' worth of wall clock.") matrixFlag := fs.Bool("matrix", false, "opt into the tests×mutants matrix: after the primary pass, re-score EVERY dev test ALONE against the run's mutants — a per-test adequacy readout + a delete-candidate list, instead of one dev-suite-wide number. COSTLY: T tests × M mutants extra jail runs (T×M, on top of the primary pass), so leave off by default on a big suite") var localEndpointFlag stringSlice @@ -201,8 +202,10 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write // The shadow draw happens HERE, before the registry resolves any seat, so // the drawn member — as typed — is resolved exactly like a named one. pooled := strings.TrimSpace(*shadow.pool) != "" || strings.TrimSpace(*shadow.writerPool) != "" + // --models reads the evidence for the same language the draw reads. + modelsGiven := modelsPool.pool != nil drawLang := strings.TrimSpace(*langFlag) - if drawLang != "" && pooled { + if drawLang != "" && (pooled || modelsGiven) { // Refused HERE, before the draw reads the store: an unknown name // would otherwise print a measured "0 row(s)" for a language corral // does not have, and only then be refused below. @@ -221,6 +224,23 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write fmt.Fprintln(stderr, "corral certify --local: a shadow pool reads history for the audited file's language, and it cannot be detected — pass --lang (or --code)") return 2 } + if drawLang == "" && modelsGiven { + fmt.Fprintln(stderr, "corral certify --local: --models ranks evidence for the audited file's language, and it cannot be detected — pass --lang (or --code)") + return 2 + } + // --models proposes the seats HERE: after the flags are parsed and before + // the shadow draw (which reads the primary writer and generator) and the + // seat resolution, so a proposed member is drawn against and resolved + // exactly as the same model typed into its flag would be. The flags the + // operator did type are pins. + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, modelsPool, []poolRole{ + {role: advpool.RoleMutantGenerator, val: mutantModel}, + {role: advpool.RoleTestWriter, val: writerModel}, + {role: advpool.RoleTestCritic, val: criticModel}, + }, doorPoolDeps(stdout, stderr)) + if poolDone { + return poolExit + } // The bug-catching scorecard store: a --local run's per-seat metrics // (including the shadow head-to-head) persist next to the signed build // ledger and feed the same `corral scorecard` surface the daemon's runs feed. @@ -550,6 +570,7 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write shadowWriterModel: *shadow.writerModel, shadowSelection: shadowSels, seats: seats, + seating: seating, writerMode: writerMode, jail: *jailFlag, checkArgv: checkArgv, @@ -694,6 +715,12 @@ type localAuditInput struct { // verdict and the signed record say which member ran and why. shadowSelection []shadowpool.Selection + // seating is how --models chose each seat and what the ranking read + // (seatFromPool), carried onto the RunSpec so the verdict, and from it + // the ledger row, records it. Zero when --models was not given: every + // seat was named by flag, and nothing is recorded. + seating poolSeating + // seats is every seat of this invocation, resolved once (resolveSeats): // the provider, wire, endpoint and key each model is served by. Provider // feeds the decorrelation DISCLOSURE (decorrelation still refuses on the @@ -1982,6 +2009,8 @@ func newAuditRunSpec(in localAuditInput, roles auditRoles, subj runSubject) advp ShadowSelection: in.shadowSelection, SeatTools: roles.tools, SeatProviders: roles.providers, + SeatedBy: in.seating.SeatedBy, + SeatEvidence: in.seating.SeatEvidence, // HOW the writer attacks. The CLI's default is per-survivor; an // EMPTY value here means batched, which is what a caller outside the // CLI (the brain, a test) gets — see RunSpec.WriterMode. diff --git a/cmd/corral/certify_local_entry.go b/cmd/corral/certify_local_entry.go index d2decdd3..043877a7 100644 --- a/cmd/corral/certify_local_entry.go +++ b/cmd/corral/certify_local_entry.go @@ -212,7 +212,10 @@ func localEntryBundle(in localAuditInput, v advpool.Verdict, t advpool.RunTeleme ToolMutants: run.toolMutants, // The verdict's own providers, as its roster is: the seats this // run signed, never re-read from the invocation's flags. - SeatProviders: seatProvidersJSON(v.SeatProviders), + SeatProviders: roleMapJSON(v.SeatProviders), + // How each seat was chosen, from the verdict for the same + // reason: what the run signed, never re-read from --models. + SeatedBy: roleMapJSON(v.SeatedBy), SeatEvidence: v.SeatEvidence, }) if run.recorder != nil { run.recorder.stampHunks(b.Mutants) diff --git a/cmd/corral/certify_repo.go b/cmd/corral/certify_repo.go index dca4e7c7..df9aae06 100644 --- a/cmd/corral/certify_repo.go +++ b/cmd/corral/certify_repo.go @@ -83,6 +83,7 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { writerPool: "a comma-separated POOL of challenger WRITER models, drawn once per scan exactly as --shadow-pool is. Mutually exclusive with --shadow-writer-model. OFF unless named; NEVER gates the verdict", seed: shadowSeedReplayHelp, }) + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it spans several), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits; --dry-run prints the plan and asks nothing). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool") owner := fs.String("owner", "local", "owning account for the scan (tenant identifier)") commit := fs.String("commit", "", "commit SHA the report is bound to") swarmFlag := fs.Int("swarm", 0, "max concurrent audit workers (0 = auto-size to this host's cores); on --substrate workspace it also sizes the private trees that score one file's mutants at once (budget/4, min 1), so --swarm 4 is one tree") @@ -198,68 +199,6 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { } } - // The shadow draw happens HERE, before the registry resolves any seat, so - // the drawn member — as typed — is resolved exactly like a named one. A - // scan spans languages and draws ONCE for all of them, so it reads the - // scorecard's record across every language. - // - // --dry-run audits nothing, so it draws nothing: it VALIDATES the pools - // (a malformed one is refused here exactly as a real run refuses it) but - // prints no draw, opens no scorecard and — like every other seat on the - // free inventory — demands no member's credential. - shadowPrimary := map[string]string{advpool.RoleMutantGenerator: *mutantModelFlag, advpool.RoleTestWriter: *writerModelFlag} - var shadowSels []shadowpool.Selection - var drawErr error - if *dryRun { - drawErr = validateShadowPools("corral certify --repo", *repoDir, shadow, shadowPrimary, false, stderr) - } else { - bugCatchDBPath, pathErr := statedir.PathForWrite(statedir.BugCatchDB) - if pathErr != nil { - fmt.Fprintln(stderr, "corral certify --repo:", pathErr) - return 2 - } - shadowSels, drawErr = drawShadowSeats("corral certify --repo", *repoDir, shadowpool.LangAny, shadow, - shadowPrimary, bugCatchDBPath, stderr) - } - if drawErr != nil { - fmt.Fprintf(stderr, "corral certify --repo: %v\n", drawErr) - return 2 - } - - // The model registry (docs/design/model-registry.md): a project declares - // its models once in .corral/models.json and names seats by alias. - // ADDITIVE — with no registry nothing here emits or changes anything, and - // a value that is not a declared alias is still the concrete model name - // this flag has always accepted. - // - // Resolved HERE, before any seat value is read by the preflight, the cache - // key, the ledger or the signed statement, so every one of them records - // the CONCRETE model. An alias is a label for humans and is never - // authoritative. - // --local-endpoint placements are parsed BEFORE the seats resolve: a - // placement is itself a declared source for a seat with a bare local name, - // and the preflight below (resolveAuditRoles) must see it folded in. - localEndpoints, lerr := parseLocalEndpoints(localEndpointFlag) - if lerr != nil { - fmt.Fprintf(stderr, "corral certify --repo: %v\n", lerr) - return 2 - } - // context.Background(): no cancellable context exists here. The SIGINT - // handler (auditContext) is installed after the goal derivation, on - // purpose — an interrupt before any audit has nothing to record, and - // TestAnInterruptedRepoScanRecordsNothing_APISeat depends on it. The one - // network call, listing OLLAMA_URL's daemon, has its own 10s bound. - seats, regErr := resolveSeatsPlaced(context.Background(), "corral certify --repo", *repoDir, - certifySeats(deriveModel, mutantModelFlag, writerModelFlag, criticModelFlag, shadow.model, shadow.writerModel), localEndpoints, stderr) - if regErr != nil { - fmt.Fprintf(stderr, "corral certify --repo: %v\n", regErr) - return 2 - } - if *scopeTestsFlag { - fmt.Fprintln(stderr, "corral certify --repo: --scope-tests was removed. Its paired-FILE scoping inverted verdicts (requests/adapters.py 1.00 -> 0.00). Selection is now by coverage evidence and on by default; pass --whole-suite to grade against the whole suite. See docs/design/test-selection.md") - return 2 - } - // --transparency logs an attestation; there is none without --attest — // caught here, before anything is spent, rather than discovered at the // end of a full run when writeAuditStatement never ran. @@ -479,6 +418,114 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { stdout = io.Discard } + // --models proposes the scan's seats HERE: after every flag-only refusal + // and the walk, because the evidence is ranked for the scan's language + // (spec call C7) and the walk is what knows it, and before the shadow + // draw (which reads the primary writer and generator) and the seat + // resolution, so a proposed member is drawn against and resolved exactly + // as the same model typed into its flag would be. The language set is + // the one the evidence pass instruments (preflightLanguages over the + // enumerated sources): a superset of what the scan audits, so a scan + // ranked by one language never audits another. One language ranks by it; + // none or several rank across all of them. + // + // A dry run spends nothing and demands no credential for any seat, so + // there the pool is not credential-checked and the plan is shown without + // a question. Under --json the plan goes to stderr: not one byte of prose + // may precede the inventory. + scanLang := shadowpool.LangAny + if langs := preflightLanguages(enumeratedSourcePaths(cands, excl)); len(langs) == 1 { + for l := range langs { + scanLang = l + } + } + poolRoles := []poolRole{ + {role: advpool.RoleMutantGenerator, val: mutantModelFlag}, + {role: advpool.RoleTestWriter, val: writerModelFlag}, + {role: advpool.RoleTestCritic, val: criticModelFlag}, + } + if strings.TrimSpace(*goalsPath) == "" { + // The deriver is a seat only when no --goals map replaces it. + poolRoles = append([]poolRole{{role: roleGoalDeriver, val: deriveModel}}, poolRoles...) + } + planOut := stdout + if *jsonOut { + planOut = stderr + } + seatDeps := doorPoolDeps(planOut, stderr) + if *dryRun { + seatDeps.confirm, seatDeps.noCredentialCheck = acceptUnasked, true + } + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --repo", *repoDir, scanLang, modelsPool, poolRoles, seatDeps) + if poolDone { + return poolExit + } + if *dryRun && seating.proposed() { + fmt.Fprintln(planOut, "a dry run asks nothing and runs nothing: this is the lineup a real run would propose") + } + + // The shadow draw happens HERE, before the registry resolves any seat, so + // the drawn member — as typed — is resolved exactly like a named one. A + // scan spans languages and draws ONCE for all of them, so it reads the + // scorecard's record across every language. + // + // --dry-run audits nothing, so it draws nothing: it VALIDATES the pools + // (a malformed one is refused here exactly as a real run refuses it) but + // prints no draw, opens no scorecard and — like every other seat on the + // free inventory — demands no member's credential. + shadowPrimary := map[string]string{advpool.RoleMutantGenerator: *mutantModelFlag, advpool.RoleTestWriter: *writerModelFlag} + var shadowSels []shadowpool.Selection + var drawErr error + if *dryRun { + drawErr = validateShadowPools("corral certify --repo", *repoDir, shadow, shadowPrimary, false, stderr) + } else { + bugCatchDBPath, pathErr := statedir.PathForWrite(statedir.BugCatchDB) + if pathErr != nil { + fmt.Fprintln(stderr, "corral certify --repo:", pathErr) + return 2 + } + shadowSels, drawErr = drawShadowSeats("corral certify --repo", *repoDir, shadowpool.LangAny, shadow, + shadowPrimary, bugCatchDBPath, stderr) + } + if drawErr != nil { + fmt.Fprintf(stderr, "corral certify --repo: %v\n", drawErr) + return 2 + } + + // The model registry (docs/design/model-registry.md): a project declares + // its models once in .corral/models.json and names seats by alias. + // ADDITIVE — with no registry nothing here emits or changes anything, and + // a value that is not a declared alias is still the concrete model name + // this flag has always accepted. + // + // Resolved HERE, before any seat value is read by the preflight, the cache + // key, the ledger or the signed statement, so every one of them records + // the CONCRETE model. An alias is a label for humans and is never + // authoritative. + // --local-endpoint placements are parsed BEFORE the seats resolve: a + // placement is itself a declared source for a seat with a bare local name, + // and the preflight below (resolveAuditRoles) must see it folded in. + localEndpoints, lerr := parseLocalEndpoints(localEndpointFlag) + if lerr != nil { + fmt.Fprintf(stderr, "corral certify --repo: %v\n", lerr) + return 2 + } + // context.Background(): no cancellable context exists here. The SIGINT + // handler (auditContext) is installed after the goal derivation, on + // purpose — an interrupt before any audit has nothing to record, and + // TestAnInterruptedRepoScanRecordsNothing_APISeat depends on it. The one + // network call, listing OLLAMA_URL's daemon, has its own 10s bound. + seats, regErr := resolveSeatsPlaced(context.Background(), "corral certify --repo", *repoDir, + certifySeats(deriveModel, mutantModelFlag, writerModelFlag, criticModelFlag, shadow.model, shadow.writerModel), localEndpoints, stderr) + if regErr != nil { + fmt.Fprintf(stderr, "corral certify --repo: %v\n", regErr) + return 2 + } + if *scopeTestsFlag { + fmt.Fprintln(stderr, "corral certify --repo: --scope-tests was removed. Its paired-FILE scoping inverted verdicts (requests/adapters.py 1.00 -> 0.00). Selection is now by coverage evidence and on by default; pass --whole-suite to grade against the whole suite. See docs/design/test-selection.md") + return 2 + } + fmt.Fprintf(stdout, "corral certify --repo %s\n", *repoDir) if scope != "" { fmt.Fprintf(stdout, " scope: %s (--path)\n", scope) @@ -917,6 +964,7 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { writer: *writerModelFlag, mutant: *mutantModelFlag, critic: *criticModelFlag, shadow: *shadow.model, shadowWriter: *shadow.writerModel, shadowSelection: shadowSels, + seating: seating, } // Jail preflight: a host that cannot sandbox grades nothing, and saying // so now beats reporting every file as ungradable — after paying for a @@ -1925,8 +1973,12 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { bundleMeta{ ModelsByRole: string(rosterJSON), SeatTools: seatToolsJSON, - SeatProviders: seatProvidersJSON(seatProviders), - MinKillRate: minKillRate, MaxProvenMissed: maxProvenMissed, + SeatProviders: roleMapJSON(seatProviders), + // How the roster's seats were chosen, narrowed to the roster as + // its providers are (the deriver is not on it). + SeatedBy: roleMapJSON(advpool.SeatedOnly(seating.SeatedBy, models())), + SeatEvidence: seating.SeatEvidence, + MinKillRate: minKillRate, MaxProvenMissed: maxProvenMissed, Passed: boolPtr(exitCode == 0), PushedBy: auditpush.PushedByCertify, TreeDirty: auditedTreeDirty, RepoIdentity: repoIdentity(*repoDir), @@ -5220,6 +5272,9 @@ type auditModels struct { // challenger was named or off), carried into every file's RunSpec so each // verdict says which member ran and why. shadowSelection []shadowpool.Selection + // seating is how --models chose the scan's seats (zero without it), + // carried into every file's RunSpec beside the models it chose. + seating poolSeating } type localExecutor struct { @@ -5723,6 +5778,7 @@ func (l *localExecutor) auditInputFor(j reposcan.Job) localAuditInput { shadowModel: l.models.shadow, shadowWriterModel: l.models.shadowWriter, shadowSelection: l.models.shadowSelection, + seating: l.models.seating, writerMode: l.writerMode, // A nil entry is an ordinary generated run — see localExecutor.presetMutants. diff --git a/cmd/corral/certify_repo_bundle.go b/cmd/corral/certify_repo_bundle.go index c2d8b842..83af36fe 100644 --- a/cmd/corral/certify_repo_bundle.go +++ b/cmd/corral/certify_repo_bundle.go @@ -57,8 +57,13 @@ type bundleMeta struct { // SeatTools is the roster's agent tools as JSON, beside ModelsByRole. SeatTools string // SeatProviders is the roster's providers as JSON, beside ModelsByRole - // (seatProvidersJSON); "" when no seat's provider was resolved. - SeatProviders string + // (roleMapJSON); "" when no seat's provider was resolved. + SeatProviders string + // SeatedBy is how each roster seat was chosen, as JSON (roleMapJSON), + // and SeatEvidence what the ranking read; both "" when --models was not + // given, so such an entry is shaped as before. + SeatedBy string + SeatEvidence string MinKillRate *float64 MaxProvenMissed *int Passed *bool // nil when the outcome is not recoverable (scans push) @@ -202,7 +207,8 @@ func buildAuditRows(files []scanstore.File, scanID int64, meta bundleMeta) []aud ModelsByRole: meta.ModelsByRole, SeatTools: meta.SeatTools, SeatProviders: meta.SeatProviders, - MinKillRate: meta.MinKillRate, MaxProvenMissed: meta.MaxProvenMissed, + SeatedBy: meta.SeatedBy, SeatEvidence: meta.SeatEvidence, + MinKillRate: meta.MinKillRate, MaxProvenMissed: meta.MaxProvenMissed, Passed: meta.Passed, StartedAt: meta.StartedAt, // WHICH measurement the rate is, and at which grain. diff --git a/cmd/corral/demo.go b/cmd/corral/demo.go index e7a620fb..ce9d7c42 100644 --- a/cmd/corral/demo.go +++ b/cmd/corral/demo.go @@ -46,11 +46,19 @@ func runDemo(args []string, stdout, stderr io.Writer) int { writer := fs.String("writer-model", "", "model for the test-writer role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (`anthropic/…`, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_)") mutant := fs.String("mutant-model", "", "model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (`anthropic/…`, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_)") critic := fs.String("critic-model", "", `model (or agent seat) for the test-critic role, which must differ from the writer's ("off" disables it)`) + // Registered here and handed to certify --local unchanged: the demo is + // the real command reached by its argv, so certify --local proposes the + // seats (ranked for the demo's language, Go) and records how they were + // chosen. Proposing here and passing plain seat flags on would record + // a lineup the evidence chose as one the operator named. + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused") if err := fs.Parse(args); err != nil { return flagParseExit(err) } - if strings.TrimSpace(*writer) == "" || strings.TrimSpace(*mutant) == "" { + // The two required seats, unless a pool is given to propose them; with a + // pool, certify --local refuses whatever it cannot fill. + if modelsPool.pool == nil && (strings.TrimSpace(*writer) == "" || strings.TrimSpace(*mutant) == "") { fmt.Fprint(stderr, demoUsage) return 2 } @@ -79,11 +87,18 @@ func runDemo(args []string, stdout, stderr io.Writer) int { "--test", filepath.Join(root, "passwd_test.go"), "--repo-dir", root, "--goal", demoGoal, - "--writer-model", *writer, - "--mutant-model", *mutant, } - if c := strings.TrimSpace(*critic); c != "" { - argv = append(argv, "--critic-model", c) + // Only the seats named: an empty flag passed on would be a pin of + // nothing, and with --models it would hide the seat from the proposal. + for _, f := range []struct{ flag, val string }{ + {"--writer-model", *writer}, {"--mutant-model", *mutant}, {"--critic-model", *critic}, + } { + if v := strings.TrimSpace(f.val); v != "" { + argv = append(argv, f.flag, v) + } + } + if modelsPool.pool != nil { + argv = append(argv, "--models", *modelsPool.pool) } argv = append(argv, "--", "go", "test", "./...") @@ -133,9 +148,11 @@ need a Go toolchain (you have one — you installed corral with it) and one provider key. Nothing else: no venv, no database, no fixtures. Flags: - --writer-model model for the test-writer role (required) - --mutant-model model for the mutant-generator role (required) + --writer-model model for the test-writer role (required unless --models) + --mutant-model model for the mutant-generator role (required unless --models) --critic-model model for the test-critic role ("off" disables it) + --models a pool of models to propose the seats from; nothing runs + until you accept the plan --dir where to write the project (default: a temp dir, kept) ` diff --git a/cmd/corral/doctor.go b/cmd/corral/doctor.go index 5fa6135a..48c90001 100644 --- a/cmd/corral/doctor.go +++ b/cmd/corral/doctor.go @@ -20,6 +20,7 @@ import ( "github.com/pdbethke/corralai/internal/lang" "github.com/pdbethke/corralai/internal/reposcan" "github.com/pdbethke/corralai/internal/sandbox" + "github.com/pdbethke/corralai/internal/shadowpool" ) // `corral doctor` answers "will an audit work here?" BEFORE one is paid for. @@ -95,9 +96,37 @@ func runDoctor(args []string, stdout, stderr io.Writer) int { }) deriveModel := fs.String("derive-model", "", "the goal-derivation model a `certify --repo` run will name, if any. Naming it makes doctor SPEND ONE small real derive call through certify's own deriver, on a tiny built-in file with an obvious property, and fail the check when the seat errors or returns an empty reply; nothing is called unless this is named") repoDir := fs.String("repo", ".", "the repository the run will audit — where its .corral/models.json registry is read from, exactly as certify reads it") + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code) and checks those seats. doctor asks nothing and runs nothing (certify would run nothing until you accept the plan), and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused") if err := fs.Parse(args); err != nil { return flagParseExit(err) } + // Whether the operator named the deriver, read before --models can fill + // the seat: doctor's one paid check (the derive probe) is spent only on + // a deriver the operator named, never on one a pool proposed. + deriveNamed := strings.TrimSpace(*deriveModel) != "" + // --models: the lineup certify would propose, written into the seat + // flags so every check below is about the seats that would run. doctor + // spends nothing, so it accepts the plan without asking and says so. + poolLang := shadowpool.LangAny + if c := strings.TrimSpace(*code); c != "" { + if p, ok := lang.Detect(c); ok { + poolLang = p.Name() + } + } + seatDeps := doorPoolDeps(stdout, stderr) + seatDeps.confirm = acceptUnasked + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral doctor", *repoDir, poolLang, modelsPool, []poolRole{ + {role: roleGoalDeriver, val: deriveModel}, + {role: advpool.RoleMutantGenerator, val: mutantModel}, + {role: advpool.RoleTestWriter, val: writerModel}, + {role: advpool.RoleTestCritic, val: criticModel}, + }, seatDeps) + if poolDone { + return poolExit + } + if seating.proposed() { + fmt.Fprintln(stdout, "doctor runs nothing: this is the lineup `certify` would propose") + } // Every member of every challenger pool, checked exactly as certify checks // it before its draw — but never drawn: doctor answers "would this run // work", and a pool works only if EVERY member could be the one drawn. @@ -156,12 +185,13 @@ func runDoctor(args []string, stdout, stderr io.Writer) int { break } } - // The one check that spends, run only when the seat was named and its + // The one check that spends, run only when the operator named the seat + // (a deriver --models proposed is checked statically, never called) and its // static checks passed (a missing CLI or key is reported as that, not as // a failed call). It runs AFTER the other checks have printed and is // announced before the call, so a fatal check above can never hide a // call that was spent (review I1, ruling R25). - if rs := seats["derive-model"]; strings.TrimSpace(rs.Spec) != "" && checksPassed(derive) { + if rs := seats["derive-model"]; deriveNamed && strings.TrimSpace(rs.Spec) != "" && checksPassed(derive) { fmt.Fprintf(stdout, " spending one derive call on %s…\n", rs.Spec) printCheck(stdout, probeDeriveSeat(rs), &failed) } diff --git a/cmd/corral/models_doors_test.go b/cmd/corral/models_doors_test.go new file mode 100644 index 00000000..fa6dd505 --- /dev/null +++ b/cmd/corral/models_doors_test.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/pdbethke/corralai/internal/agentbackend" +) + +// What --models does at each door that is not a recorded run: doctor and a +// dry run show the lineup and ask nothing, demo hands the pool to the real +// certify --local, and every door's help says a pool is not a default. + +// doctor rehearses a run and spends nothing, so it proposes, prints the plan, +// checks the proposed seats, and never asks: there is nothing to accept. The +// pool seats the goal deriver too (doctor previews certify --repo), and a +// deriver the pool chose must not trigger the one paid probe doctor makes +// for a deriver the operator named. +func TestDoctorModelsPrintsThePlanAndAsksNothing(t *testing.T) { + asked := stubDoorPool(t) + noKeys(t) + clearSeatEnv(t) + t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") + wiredAgent(t, "fakec", "claude-json", claudeJSONReply) + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + var out, errb bytes.Buffer + runDoctor([]string{"--repo", t.TempDir(), "--models", recordPool}, &out, &errb) + if *asked != 0 { + t.Errorf("doctor asked the operator %d time(s); it runs nothing, so there is nothing to accept", *asked) + } + text := out.String() + plan := strings.Index(text, "seats (from --models "+recordPool+")") + note := strings.Index(text, "doctor runs nothing: this is the lineup `certify` would propose") + if plan < 0 || note < plan { + t.Fatalf("want the plan, then the doctor note under it:\n%s\nstderr:\n%s", text, errb.String()) + } + if strings.Contains(text, "spending one derive call") { + t.Errorf("doctor spent a derive call on a deriver the pool proposed:\n%s", text) + } + if !strings.Contains(text, "claude-sonnet-5") { + t.Errorf("doctor did not check the proposed seats:\n%s", text) + } +} + +// A dry run spends nothing and demands no credential for any seat today, so +// it neither credential-checks the pool's members nor asks: it prints the +// lineup a real run would propose and says that it asked nothing. +func TestRepoDryRunModelsAsksNothingAndChecksNoCredential(t *testing.T) { + root, _, goals := telemetryRepoFixture(t) + asked := stubDoorPool(t) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { + return errors.New("no credential in this test") + } + noKeys(t) + clearSeatEnv(t) + wiredAgent(t, "fakec", "claude-json", claudeJSONReply) + var out, errb bytes.Buffer + code := runCertifyRepo([]string{"--repo", root, "--models", recordPool, "--goals", goals, "--dry-run"}, &out, &errb) + if code != 0 { + t.Fatalf("exit=%d\n%s%s", code, out.String(), errb.String()) + } + if *asked != 0 { + t.Errorf("a dry run asked the operator %d time(s)", *asked) + } + text := out.String() + plan := strings.Index(text, "seats (from --models "+recordPool+")") + note := strings.Index(text, "a dry run asks nothing and runs nothing") + if plan < 0 || note < plan { + t.Fatalf("want the plan, then the dry-run note under it:\n%s\nstderr:\n%s", text, errb.String()) + } + if !strings.Contains(text, "language go") { + t.Errorf("a Go-only scan must rank by Go's evidence:\n%s", text) + } +} + +// certify --local ranks by the audited file's language, so a pool with no +// language to read is refused at the door with the fix, as a shadow pool is. +func TestLocalModelsNeedsALanguage(t *testing.T) { + stubDoorPool(t) + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "notes.unknownlang"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + var out, errb bytes.Buffer + code := runCertifyLocal([]string{"--code", filepath.Join(root, "notes.unknownlang"), "--goal", "g", "--models", recordPool, "--", "true"}, &out, &errb) + if code != 2 || !strings.Contains(errb.String(), "--models ranks evidence for the audited file's language") || !strings.Contains(errb.String(), "pass --lang") { + t.Fatalf("exit %d, want 2 saying --models needs the language and naming --lang:\n%s", code, errb.String()) + } +} + +// corral demo is the real certify --local, reached by its argv, so the pool +// travels there and certify --local proposes the seats (and records how they +// were chosen). A seat the pool fills is not passed as an empty flag. +func TestDemoForwardsModelsToCertifyLocal(t *testing.T) { + orig := demoRunLocal + t.Cleanup(func() { demoRunLocal = orig }) + var got []string + demoRunLocal = func(args []string, _ bool, _, _ io.Writer) int { + got = args + return 0 + } + var out, errb bytes.Buffer + if code := runDemo([]string{"--dir", t.TempDir(), "--models", recordPool}, &out, &errb); code != 0 { + t.Fatalf("exit %d: %s", code, errb.String()) + } + i := slices.Index(got, "--models") + if i < 0 || i+1 >= len(got) || got[i+1] != recordPool { + t.Fatalf("demo did not forward --models %s: %v", recordPool, got) + } + for _, f := range []string{"--writer-model", "--mutant-model", "--critic-model"} { + if slices.Contains(got, f) { + t.Errorf("demo passed %s with no value to pin: %v", f, got) + } + } + // Still refused when there is neither a pool nor the required seats. + if code := runDemo([]string{"--dir", t.TempDir()}, &bytes.Buffer{}, &bytes.Buffer{}); code != 2 { + t.Errorf("demo with no seats and no pool: exit %d, want 2", code) + } +} + +// Every door's --models help says the pool is not a default and that nothing +// runs until the operator accepts, in that door's own terms. +func TestModelsHelpIsTrueAtEachDoor(t *testing.T) { + doors := []struct { + name string + run func([]string, io.Writer, io.Writer) int + must []string + }{ + {"certify --local", runCertifyLocal, []string{"not a default", "accept", "language"}}, + {"certify --repo", runCertifyRepo, []string{"not a default", "accept", "goal-deriver", "--dry-run"}}, + {"doctor", runDoctor, []string{"not a default", "accept", "runs nothing"}}, + {"demo", runDemo, []string{"not a default", "accept", "Go"}}, + } + for _, d := range doors { + help := flagUsage(t, d.run, "models") + for _, w := range d.must { + if !strings.Contains(help, w) { + t.Errorf("%s -models help lacks %q:\n%s", d.name, w, help) + } + } + } +} diff --git a/cmd/corral/models_flags_test.go b/cmd/corral/models_flags_test.go new file mode 100644 index 00000000..565feadb --- /dev/null +++ b/cmd/corral/models_flags_test.go @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// seatFlagLiteral matches the registration of a door's primary seat flag. A +// door that seats a model can be handed a pool, so every such door must +// register --models through registerModelsFlags: derived from the source, not +// a list of commands, so a sixth door is found the day it is written. +var seatFlagLiteral = regexp.MustCompile(`\.String(Var)?\([^)]*"(writer|reviewer)-model"`) + +// modelsFlagLiteral matches a hand registration of --models, in any of the +// shapes the flag package offers. +var modelsFlagLiteral = regexp.MustCompile(`\.(String|StringVar|Func|Var)\([^)]*"models"`) + +// pendingModelsDoors are doors that register a seat flag and are not wired to +// --models yet, each with the reason. An entry for a door that now calls the +// helper fails the test below, so the list cannot outlive the gap it names. +var pendingModelsDoors = map[string]string{ + "review.go": "corral review is wired in its own change (autoseat phase 2, task 5): its seats are reviewer and verifier, not certify's", +} + +func TestModelsFlagPatternsMatchARegistration(t *testing.T) { + // Negative controls: each pattern must catch the shape it exists to + // catch, or the door test below passes vacuously. + for re, src := range map[*regexp.Regexp]string{ + seatFlagLiteral: `w := fs.String("writer-model", "", "h")`, + modelsFlagLiteral: `p := fs.String("models", "", "h")`, + } { + if !re.MatchString(src) { + t.Fatalf("%v no longer matches %q", re, src) + } + } + if !seatFlagLiteral.MatchString(`fs.StringVar(&f.model, "reviewer-model", "", "h")`) { + t.Fatal("seatFlagLiteral no longer matches review's StringVar registration") + } +} + +func TestEveryDoorWithSeatFlagsRegistersModelsFlags(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil || len(files) == 0 { + t.Fatalf("glob: %v (%d files)", err, len(files)) + } + doors := 0 + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + b, err := os.ReadFile(f) // #nosec G304 -- this package's own sources + if err != nil { + t.Fatal(err) + } + src := string(b) + if f != "models_pool.go" { + if loc := modelsFlagLiteral.FindStringIndex(src); loc != nil { + t.Errorf("%s registers --models by hand (%q); use registerModelsFlags", f, src[loc[0]:loc[1]]) + } + } + if !seatFlagLiteral.MatchString(src) { + continue + } + doors++ + wired := strings.Contains(src, "registerModelsFlags(") + why, pending := pendingModelsDoors[f] + switch { + case pending && wired: + t.Errorf("%s now calls registerModelsFlags; remove its pendingModelsDoors entry (%s)", f, why) + case !pending && !wired: + t.Errorf("%s registers a seat flag and not --models; call registerModelsFlags", f) + } + } + // certify --local, certify --repo, doctor, demo and review. + if doors < 5 { + t.Fatalf("found %d doors with a seat flag, want at least 5: the pattern has stopped matching", doors) + } +} + +// Choosing a seat from recorded evidence happens in one place. A door that +// imported the proposer or the ranking itself could seat a model by a rule +// of its own, and the "a pool is not a default" guarantees live in +// seatFromPool. +func TestOnlyAutoseatDecidesASeatFromEvidence(t *testing.T) { + allowed := map[string]map[string]bool{ + `"github.com/pdbethke/corralai/internal/autoseat"`: {"models_pool.go": true}, + // models_rank.go is the disclosure verb; review.go reads the + // ranking's seat names and predates the proposer. + `"github.com/pdbethke/corralai/internal/modelrank"`: {"models_pool.go": true, "models_rank.go": true, "review.go": true}, + } + files, err := filepath.Glob("*.go") + if err != nil || len(files) == 0 { + t.Fatalf("glob: %v (%d files)", err, len(files)) + } + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + b, err := os.ReadFile(f) // #nosec G304 -- this package's own sources + if err != nil { + t.Fatal(err) + } + for imp, ok := range allowed { + if strings.Contains(string(b), imp) && !ok[f] { + t.Errorf("%s imports %s; only %v may", f, imp, allowedFiles(ok)) + } + } + } +} + +func allowedFiles(m map[string]bool) []string { + var out []string + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/cmd/corral/models_pool.go b/cmd/corral/models_pool.go index 657da88a..bc44e8e3 100644 --- a/cmd/corral/models_pool.go +++ b/cmd/corral/models_pool.go @@ -51,6 +51,18 @@ type poolSeating struct { SeatEvidence string // what the ranking read, as the plan's evidence line says it; "" when no seat was seated by evidence } +// proposed reports whether --models chose any seat (by evidence or order), +// which is when a door that asks nothing says so under the plan. False +// without --models and when every seat was pinned, where no plan is printed. +func (s poolSeating) proposed() bool { + for _, by := range s.SeatedBy { + if by != autoseat.ByOperator { + return true + } + } + return false +} + // poolRole ties a role to the flag variable the door already has. type poolRole struct { role string @@ -68,8 +80,32 @@ type poolDeps struct { confirm func(plan string) (accepted bool, interactive bool, err error) stdout io.Writer stderr io.Writer + // noCredentialCheck skips each member's credential check, and is set + // by certify --repo --dry-run alone: a dry run demands no credential + // for any seat (validateShadowPools is called there with + // checkCredentials false), so a pool member is held to the same. The + // zero value checks, so a door that forgets this field is strict. + noCredentialCheck bool } +// doorPoolDeps is the production wiring every door hands seatFromPool: the +// evidence the ranking reads from the machine ledger (defaultRankLoader, as +// `corral models rank` reads it) and a question on the terminal. A variable, +// as listOllama and shadowMemberRunnable are, because the doors' entry +// points are called from dozens of tests by a fixed signature; a test +// replaces it with a fake loader and confirm, so no door test reads the real +// ledger or a terminal. doctor and certify --repo --dry-run keep the loader +// and replace the confirm with one that asks nothing. +var doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{load: defaultRankLoader, confirm: terminalConfirm(os.Stdin, stdout), stdout: stdout, stderr: stderr} +} + +// acceptUnasked is the confirm of a door that runs nothing (doctor, a dry +// run): it accepts the plan without a question, because there is nothing to +// spend and so nothing for the operator to agree to. The door then states, +// under the plan, that it asked nothing. +func acceptUnasked(string) (accepted bool, interactive bool, err error) { return true, true, nil } + // seatFromPool returns done=true with an exit code when the door must stop // (refusal: 2; declined or no terminal: 0). With --models unset it returns // a zero poolSeating and done=false, and changes nothing. @@ -99,7 +135,7 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, f *models if err != nil { return refuse("%v", err) } - pool, err := resolveModelsPool(ctx, cmdName, repoRoot, typed) + pool, err := resolveModelsPool(ctx, cmdName, repoRoot, typed, !deps.noCredentialCheck) if err != nil { return refuse("%v", err) } @@ -203,10 +239,11 @@ func parseModelsPool(text string) ([]string, error) { // resolveModelsPool resolves each member as its own seat, with one shared // listing of OLLAMA_URL's daemon (shadowCandidates' pattern), and checks each -// with shadowMemberRunnable, the one per-member credential check. The +// with shadowMemberRunnable, the one per-member credential check, unless +// checkCredentials is false (poolDeps.noCredentialCheck). The // resolver rewrites the value it is handed, so it resolves a copy and the // member keeps what the operator typed as Given. -func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []string) ([]autoseat.Member, error) { +func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []string, checkCredentials bool) ([]autoseat.Member, error) { listing := &ollamaListing{} byRecorded := map[string]string{} var out []autoseat.Member @@ -217,8 +254,12 @@ func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []st return nil, fmt.Errorf("member %q: %v", given, err) } rs := set["models"] - if err := shadowMemberRunnable(rs); err != nil { - return nil, fmt.Errorf("member %q cannot run: %v — every member is checked before anything is proposed, so fix or remove it", given, err) + // Without checkCredentials (a dry run) the member is still resolved, + // so one no source serves is refused, and is never asked for a key. + if checkCredentials { + if err := shadowMemberRunnable(rs); err != nil { + return nil, fmt.Errorf("member %q cannot run: %v — every member is checked before anything is proposed, so fix or remove it", given, err) + } } // Two spellings of one model (an alias and its concrete name) would // make the ranking's row ambiguous between them. diff --git a/cmd/corral/seat_resolver.go b/cmd/corral/seat_resolver.go index eb7c108a..2b826717 100644 --- a/cmd/corral/seat_resolver.go +++ b/cmd/corral/seat_resolver.go @@ -58,14 +58,14 @@ func (s seatSet) providersFor(roster map[string]string) map[string]string { return advpool.SeatedOnly(s.providers(), roster) } -// seatProvidersJSON is the ledger row's SeatProviders text: the role -> -// provider map as JSON, "" when there is none, so an entry whose seats -// resolved no provider records nothing rather than "{}". -func seatProvidersJSON(providers map[string]string) string { - if len(providers) == 0 { +// roleMapJSON is a ledger row's role-keyed seat map (SeatProviders, +// SeatedBy) as JSON text, "" when the map is empty, so an entry with nothing +// to say about its seats records nothing rather than "{}". +func roleMapJSON(byRole map[string]string) string { + if len(byRole) == 0 { return "" } - b, _ := json.Marshal(providers) + b, _ := json.Marshal(byRole) return string(b) } diff --git a/cmd/corral/seated_by_record_test.go b/cmd/corral/seated_by_record_test.go new file mode 100644 index 00000000..5d0cf0d9 --- /dev/null +++ b/cmd/corral/seated_by_record_test.go @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "io" + "path/filepath" + "strings" + "testing" + + "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/auditpush" + "github.com/pdbethke/corralai/internal/modelrank" +) + +// How each seat was chosen reaches the WRITTEN entry at every door that +// writes one. seatFromPool computes it; the bug shape this file guards is +// that value dropped at one of the struct copies between the door and the +// ledger row (AGENTS.md, "The bug shape that keeps recurring"), so each test +// reads the entry back from disk rather than inspecting an intermediate. + +// recordPool is the pool the record tests name: a hosted model the evidence +// below ranks in both measured seats, and an agent seat the record cannot +// rank, so the critic is seated by pool order. +const recordPool = "anthropic/claude-sonnet-5,fakec" + +const recordSource = "test evidence (6 scan(s))" + +// wantSeatedBy is SeatedBy for recordPool over recordEvidence, as the row's +// JSON text (encoding/json sorts the keys). +const wantSeatedBy = `{"mutant-generator":"evidence","test-critic":"order","test-writer":"evidence"}` + +// recordEvidence measures claude-sonnet-5 over six Go runs as generator and +// six as writer (five survivors attempted, all proven, in each), so both +// measured seats clear the floor of five. +func recordEvidence() rankEvidence { + runs := []string{"r1", "r2", "r3", "r4", "r5", "r6"} + obs := genObs("claude-sonnet-5", "go", runs...) + for _, r := range runs { + obs = append(obs, writerObs("claude-sonnet-5", "go", r, 5, 5)) + } + return rankEvidence{Obs: obs, Source: recordSource, Generations: generationRange{Min: "48", Max: "48", Scans: 6}} +} + +// stubDoorPool replaces the doors' production seams (the machine ledger and +// the terminal) with recordEvidence and an operator who accepts, and makes +// every pool member pass its credential check. It returns how many times the +// operator was asked. +func stubDoorPool(t *testing.T) *int { + t.Helper() + asked := new(int) + orig := doorPoolDeps + t.Cleanup(func() { doorPoolDeps = orig }) + doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{ + load: func(rankSource) (rankEvidence, error) { return recordEvidence(), nil }, + confirm: func(string) (bool, bool, error) { + *asked++ + return true, true, nil + }, + stdout: stdout, stderr: stderr, + } + } + origRun := shadowMemberRunnable + t.Cleanup(func() { shadowMemberRunnable = origRun }) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } + return asked +} + +// verdictSeating decodes seated_by and seat_evidence from a row's verdict +// document, and reports whether either key is present at all. +func verdictSeating(t *testing.T, verdictJSON string) (seatedBy map[string]string, evidence string, present bool) { + t.Helper() + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(verdictJSON), &raw); err != nil { + t.Fatalf("verdict document does not parse: %v", err) + } + _, a := raw["seated_by"] + _, b := raw["seat_evidence"] + var doc struct { + SeatedBy map[string]string `json:"seated_by"` + SeatEvidence string `json:"seat_evidence"` + } + _ = json.Unmarshal([]byte(verdictJSON), &doc) + return doc.SeatedBy, doc.SeatEvidence, a || b +} + +func TestLocalEntryRecordsHowEachSeatWasChosen(t *testing.T) { + root, setPath, _ := localEntryFixture(t) + asked := stubDoorPool(t) + ml := withMachineLedger(t) + args := []string{ + "--repo-dir", root, "--code", "pkg/a.go", "--goal", "A returns 1", + "--models", recordPool, "--mutants", setPath, "--quiet", + "--", "go", "test", "./...", + } + var out, errb bytes.Buffer + code := runCertifyLocal(args, &out, &errb) + localRunFinished(t, code, out.String(), errb.String()) + if *asked != 1 { + t.Fatalf("the operator was asked %d times, want once", *asked) + } + if !strings.Contains(out.String(), "seats (from --models "+recordPool+")") { + t.Errorf("no plan on stdout:\n%s", out.String()) + } + entries := scanEntriesIn(t, ml) + if len(entries) != 1 || len(entries[0].Bundle.Files) != 1 { + t.Fatalf("want one entry with one file row, got %d entries\n%s%s", len(entries), out.String(), errb.String()) + } + row := entries[0].Bundle.Files[0] + if row.SeatedBy != wantSeatedBy { + t.Errorf("file row SeatedBy = %q, want %q", row.SeatedBy, wantSeatedBy) + } + wantEvidence := recordSource + "; language go" + if row.SeatEvidence != wantEvidence { + t.Errorf("file row SeatEvidence = %q, want %q", row.SeatEvidence, wantEvidence) + } + by, ev, _ := verdictSeating(t, row.VerdictJSON) + if by[modelrank.SeatTestCritic] != "order" || by[modelrank.SeatTestWriter] != "evidence" || by[modelrank.SeatMutantGenerator] != "evidence" || ev != wantEvidence { + t.Errorf("verdict document seated_by = %v, seat_evidence = %q", by, ev) + } + // The proposed models are the ones that ran, not only the ones recorded. + var roster map[string]string + _ = json.Unmarshal([]byte(row.ModelsByRole), &roster) + if roster[modelrank.SeatTestWriter] != "claude-sonnet-5" || roster[modelrank.SeatTestCritic] != "fakec" { + t.Errorf("roster = %v, want the proposed writer claude-sonnet-5 and critic fakec", roster) + } +} + +func TestRepoEntryRecordsHowEachSeatWasChosen(t *testing.T) { + root, setPath, goals := telemetryRepoFixture(t) + asked := stubDoorPool(t) + machine := withMachineLedger(t) + noKeys(t) + t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") + wiredAgent(t, "fakec", "claude-json", claudeJSONReply) + args := []string{ + "--repo", root, "--models", recordPool, + "--goals", goals, "--substrate", substrateWorkspace, "--all", "--mutants", setPath, + } + var out, errb bytes.Buffer + if code := runCertifyRepo(args, &out, &errb); code != 0 { + t.Fatalf("exit=%d\n%s%s", code, out.String(), errb.String()) + } + if *asked != 1 { + t.Fatalf("the operator was asked %d times, want once for the whole scan", *asked) + } + wantEvidence := recordSource + "; language go" + for name, dir := range map[string]string{"repo": filepath.Join(root, ".corral", "ledger"), "machine": machine} { + entries, err := auditpush.ReadLedgerDir(dir) + if err != nil || len(entries) != 1 { + t.Fatalf("%s ledger: %d entries, %v", name, len(entries), err) + } + audited := 0 + for _, f := range entries[0].Bundle.Files { + if f.Disposition != "audited" { + continue + } + audited++ + if f.SeatedBy != wantSeatedBy || f.SeatEvidence != wantEvidence { + t.Errorf("%s %s row: SeatedBy %q, SeatEvidence %q; want %q, %q", name, f.Path, f.SeatedBy, f.SeatEvidence, wantSeatedBy, wantEvidence) + } + by, ev, _ := verdictSeating(t, f.VerdictJSON) + if by[modelrank.SeatTestCritic] != "order" || by[modelrank.SeatTestWriter] != "evidence" || ev != wantEvidence { + t.Errorf("%s %s verdict: seated_by %v, seat_evidence %q", name, f.Path, by, ev) + } + } + if audited != 2 { + t.Fatalf("%s ledger: %d audited rows, want the fixture's two", name, audited) + } + } +} + +// Without --models nothing is recorded: the row and the verdict document are +// shaped exactly as before, and absence reads as "every seat was named by +// flag". +func TestARunWithoutModelsRecordsNoSeating(t *testing.T) { + t.Run("certify --local", func(t *testing.T) { + root, setPath, _ := localEntryFixture(t) + ml := withMachineLedger(t) + code, out, errs := runLocalEntryFixture(t, root, setPath) + localRunFinished(t, code, out, errs) + entries := scanEntriesIn(t, ml) + if len(entries) != 1 || len(entries[0].Bundle.Files) != 1 { + t.Fatalf("want one entry with one file row, got %d", len(entries)) + } + row := entries[0].Bundle.Files[0] + if row.SeatedBy != "" || row.SeatEvidence != "" { + t.Errorf("row SeatedBy %q, SeatEvidence %q; want neither", row.SeatedBy, row.SeatEvidence) + } + if _, _, present := verdictSeating(t, row.VerdictJSON); present { + t.Errorf("the verdict document carries a seating key:\n%s", row.VerdictJSON) + } + }) + t.Run("certify --repo", func(t *testing.T) { + root, setPath, goals := telemetryRepoFixture(t) + machine := withMachineLedger(t) + code, out, errs := runTelemetryFixture(t, root, setPath, goals) + if code != 0 { + t.Fatalf("exit=%d\n%s%s", code, out, errs) + } + entries, err := auditpush.ReadLedgerDir(machine) + if err != nil || len(entries) != 1 { + t.Fatalf("machine ledger: %d entries, %v", len(entries), err) + } + for _, f := range entries[0].Bundle.Files { + if f.SeatedBy != "" || f.SeatEvidence != "" { + t.Errorf("%s row: SeatedBy %q, SeatEvidence %q; want neither", f.Path, f.SeatedBy, f.SeatEvidence) + } + if f.VerdictJSON == "" { + continue + } + if _, _, present := verdictSeating(t, f.VerdictJSON); present { + t.Errorf("%s: the verdict document carries a seating key", f.Path) + } + } + }) +} diff --git a/docs/cli/corral.md b/docs/cli/corral.md index 7dd4396f..d8185ddf 100644 --- a/docs/cli/corral.md +++ b/docs/cli/corral.md @@ -302,6 +302,8 @@ Usage of certify --local: max mutant-generator seats fanned out across the file's functions (0 = 8). Bounds PARALLELISM only — every function is probed regardless; --n-mutants is the PER-SHARD budget -max-tokens int cap on model TOKENS for the whole run, input + output, every seat (0 = no cap; an agent seat that reports no token usage is refused, since the cap could not see it). Checked before each call and charged after it, so one in-flight call can overshoot by its own size. Once reached: a generator seat that has not run makes its file ungradable (executor-error naming the cap), a writer or critic seat is skipped and the file flagged as it is for a provider failure — the dev kill rate already measured stands. The cost line says the cap was reached and after how many calls. Corral has bounded mutants, shards and wall clock and never money; this is the money bound + -models POOL + a comma-separated POOL of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string @@ -413,6 +415,8 @@ Usage of demo: model (or agent seat) for the test-critic role, which must differ from the writer's ("off" disables it) -dir string where to write the demo project (default: a new temp dir, printed and kept) + -models POOL + a comma-separated POOL of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_) -writer-model anthropic/… @@ -431,6 +435,8 @@ Usage of doctor: the goal-derivation model a certify --repo run will name, if any. Naming it makes doctor SPEND ONE small real derive call through certify's own deriver, on a tiny built-in file with an obvious property, and fail the check when the seat errors or returns an empty reply; nothing is called unless this is named -jail string sandbox backend (default: auto-detect). "container" needs CORRALAI_EXEC_IMAGE set to a toolchain image, e.g. CORRALAI_EXEC_IMAGE=python:3.12-bookworm + -models POOL + a comma-separated POOL of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code) and checks those seats. doctor asks nothing and runs nothing (certify would run nothing until you accept the plan), and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model string the mutant-generator model whose credential to check -repo string @@ -1299,6 +1305,8 @@ Usage of certify --repo: cap on model TOKENS for the whole SCAN, every file, input + output, every seat (0 = no cap; an agent seat that reports no token usage is refused, since the cap could not see it). Checked before each call and charged after it, so one in-flight call can overshoot by its own size. Once reached: a generator seat that has not run makes its file ungradable (executor-error naming the cap), a writer or critic seat is skipped and the file flagged as it is for a provider failure — the dev kill rate already measured stands. The cost line says the cap was reached and after how many calls. Corral has bounded mutants, shards and wall clock and never money; this is the money bound -min-kill-rate string fail the scan (exit 1) if ANY audited file's kill rate is below this value (0.0-1.0 inclusive; a minimum, so a file exactly at the threshold passes). Opt-in: unset by default, so exit codes are unchanged unless this is given. Applies PER FILE, not to the aggregate — a well-tested file must not mask a weak one + -models POOL + a comma-separated POOL of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it spans several), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits; --dry-run prints the plan and asks nothing). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string diff --git a/site/src/content/docs/docs/cli/corral.md b/site/src/content/docs/docs/cli/corral.md index 7dd4396f..d8185ddf 100644 --- a/site/src/content/docs/docs/cli/corral.md +++ b/site/src/content/docs/docs/cli/corral.md @@ -302,6 +302,8 @@ Usage of certify --local: max mutant-generator seats fanned out across the file's functions (0 = 8). Bounds PARALLELISM only — every function is probed regardless; --n-mutants is the PER-SHARD budget -max-tokens int cap on model TOKENS for the whole run, input + output, every seat (0 = no cap; an agent seat that reports no token usage is refused, since the cap could not see it). Checked before each call and charged after it, so one in-flight call can overshoot by its own size. Once reached: a generator seat that has not run makes its file ungradable (executor-error naming the cap), a writer or critic seat is skipped and the file flagged as it is for a provider failure — the dev kill rate already measured stands. The cost line says the cap was reached and after how many calls. Corral has bounded mutants, shards and wall clock and never money; this is the money bound + -models POOL + a comma-separated POOL of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string @@ -413,6 +415,8 @@ Usage of demo: model (or agent seat) for the test-critic role, which must differ from the writer's ("off" disables it) -dir string where to write the demo project (default: a new temp dir, printed and kept) + -models POOL + a comma-separated POOL of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_) -writer-model anthropic/… @@ -431,6 +435,8 @@ Usage of doctor: the goal-derivation model a certify --repo run will name, if any. Naming it makes doctor SPEND ONE small real derive call through certify's own deriver, on a tiny built-in file with an obvious property, and fail the check when the seat errors or returns an empty reply; nothing is called unless this is named -jail string sandbox backend (default: auto-detect). "container" needs CORRALAI_EXEC_IMAGE set to a toolchain image, e.g. CORRALAI_EXEC_IMAGE=python:3.12-bookworm + -models POOL + a comma-separated POOL of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code) and checks those seats. doctor asks nothing and runs nothing (certify would run nothing until you accept the plan), and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model string the mutant-generator model whose credential to check -repo string @@ -1299,6 +1305,8 @@ Usage of certify --repo: cap on model TOKENS for the whole SCAN, every file, input + output, every seat (0 = no cap; an agent seat that reports no token usage is refused, since the cap could not see it). Checked before each call and charged after it, so one in-flight call can overshoot by its own size. Once reached: a generator seat that has not run makes its file ungradable (executor-error naming the cap), a writer or critic seat is skipped and the file flagged as it is for a provider failure — the dev kill rate already measured stands. The cost line says the cap was reached and after how many calls. Corral has bounded mutants, shards and wall clock and never money; this is the money bound -min-kill-rate string fail the scan (exit 1) if ANY audited file's kill rate is below this value (0.0-1.0 inclusive; a minimum, so a file exactly at the threshold passes). Opt-in: unset by default, so exit codes are unchanged unless this is given. Applies PER FILE, not to the aggregate — a well-tested file must not mask a weak one + -models POOL + a comma-separated POOL of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it spans several), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits; --dry-run prints the plan and asks nothing). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string diff --git a/testdata/executed-surfaces.tsv b/testdata/executed-surfaces.tsv index f9745d31..05b83f01 100644 --- a/testdata/executed-surfaces.tsv +++ b/testdata/executed-surfaces.tsv @@ -60,6 +60,7 @@ corral certify --local --matrix executed test:cmd/corral/certify_local_test.go corral certify --local --ledger executed test:cmd/corral/certify_local_entry_test.go (TestLocalLedgerFlagWritesBoth: written beside the machine ledger, deduplicated against it, refused inside --repo-dir) corral certify --local --max-shards executed run:2026-09-02 record 89 corral certify --local --max-tokens executed test:cmd/corral/local_endpoint_test.go +corral certify --local --models executed test:cmd/corral/seated_by_record_test.go (TestLocalEntryRecordsHowEachSeatWasChosen: a fake ledger and an accepting operator; the real machine ledger and terminal are never read) corral certify --local --prior executed test:cmd/corral/certify_repo_evidence_candidacy_test.go corral certify --local --mutant-model executed test:cmd/corral/certify_repo_writer_mode_test.go corral certify --local --mutants executed test:cmd/corral/certify_repo_evidence_candidacy_test.go @@ -104,6 +105,7 @@ corral certify --repo --json executed test:cmd/corral/models_rank_test.go corral certify --repo --local-endpoint executed test:cmd/corral/local_endpoint_test.go (parseLocalEndpoints, shared by both paths) corral certify --repo --max-proven-missed executed test:cmd/corral/certify_repo_test.go (gate logic; flag parsing still unexercised) corral certify --repo --min-kill-rate executed test:cmd/corral/certify_repo_test.go +corral certify --repo --models executed test:cmd/corral/seated_by_record_test.go (TestRepoEntryRecordsHowEachSeatWasChosen), cmd/corral/models_doors_test.go (TestRepoDryRunModelsAsksNothingAndChecksNoCredential) corral certify --repo --mutant-model executed test:cmd/corral/certify_repo_writer_mode_test.go corral certify --repo --mutants executed test:cmd/corral/certify_repo_evidence_candidacy_test.go corral certify --repo --mutants-from executed test:cmd/corral/mutants_from_test.go @@ -149,6 +151,7 @@ corral certify verify --pubkey executed test:cmd/corral/verify_test.go corral certify verify --rekor-url unexecuted corral demo --critic-model executed test:cmd/corral/certify_repo_writer_mode_test.go corral demo --dir unexecuted +corral demo --models executed test:cmd/corral/models_doors_test.go (TestDemoForwardsModelsToCertifyLocal: parsed and forwarded to certify --local, which proposes the seats) corral demo --mutant-model executed test:cmd/corral/certify_repo_writer_mode_test.go corral demo --writer-model executed test:cmd/corral/certify_repo_writer_mode_test.go corral doctor --code executed test:cmd/corral/certify_adversarial_test.go @@ -157,6 +160,7 @@ corral doctor --derive-model executed test:cmd/corral/doctor_test.go corral doctor --repo executed test:cmd/corral/doctor_test.go corral doctor --shadow-model executed test:cmd/corral/doctor_test.go corral doctor --jail executed test:cmd/corral/jail_test.go +corral doctor --models executed test:cmd/corral/models_doors_test.go (TestDoctorModelsPrintsThePlanAndAsksNothing) corral doctor --mutant-model executed test:cmd/corral/certify_repo_writer_mode_test.go corral doctor --shadow-pool executed test:cmd/corral/shadow_draw_test.go (TestDoctorValidatesShadowPools) corral doctor --shadow-seed executed test:cmd/corral/shadow_draw_test.go (TestDoctorValidatesShadowPools: parsed and refused without a pool, never drawn) From 64c544afcdf77fc4a42dd3491f2405c2cebc4751 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 12:18:42 -0500 Subject: [PATCH 05/14] Record how a --repo scan's goal deriver was chosen by naming the deriver on the row's roster when it runs, and refuse certify --local's flag errors before --models asks anything, so the record matches the plan the operator accepted and no one accepts a lineup for a run that was always going to be refused. The --repo roster (shared by the rows and the --attest statement) now names goal-deriver with its concrete model when no --goals map replaced it, so SeatedBy["goal-deriver"] survives the existing narrowing to the roster and is attributed to a model; with --goals no deriver ran and none is named. advpool.SeatedOnly is unchanged. Every reader of the roster gains the key additively: the statement's models map, SeatTools for an agent deriver, and models rank, which already reports the deriver as "not scored". The verdict cache key does not read the roster, so cache hits are unchanged; the deriver's provider stays unrecorded because the derive seat has no role in certifySeats. certify --local's --local-endpoint parse and its --code, --goal, subject, bind-dir, seed-limit and --ledger-inside-checkout refusals now precede seatFromPool, the shadow draw and seat resolution, the order --repo keeps. TestEveryDoorDrawsFromItsPoolFlags now triggers its after-the-draw refusal with a missing --prior instead of a missing --code. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/certify_local.go | 100 +++++++++++++++------------- cmd/corral/certify_repo.go | 13 +++- cmd/corral/models_doors_test.go | 14 ++++ cmd/corral/seated_by_record_test.go | 62 +++++++++++++++++ cmd/corral/shadow_draw_test.go | 7 +- 5 files changed, 146 insertions(+), 50 deletions(-) diff --git a/cmd/corral/certify_local.go b/cmd/corral/certify_local.go index 9bf14b14..65d34a11 100644 --- a/cmd/corral/certify_local.go +++ b/cmd/corral/certify_local.go @@ -228,35 +228,12 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write fmt.Fprintln(stderr, "corral certify --local: --models ranks evidence for the audited file's language, and it cannot be detected — pass --lang (or --code)") return 2 } - // --models proposes the seats HERE: after the flags are parsed and before - // the shadow draw (which reads the primary writer and generator) and the - // seat resolution, so a proposed member is drawn against and resolved - // exactly as the same model typed into its flag would be. The flags the - // operator did type are pins. - seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, modelsPool, []poolRole{ - {role: advpool.RoleMutantGenerator, val: mutantModel}, - {role: advpool.RoleTestWriter, val: writerModel}, - {role: advpool.RoleTestCritic, val: criticModel}, - }, doorPoolDeps(stdout, stderr)) - if poolDone { - return poolExit - } - // The bug-catching scorecard store: a --local run's per-seat metrics - // (including the shadow head-to-head) persist next to the signed build - // ledger and feed the same `corral scorecard` surface the daemon's runs feed. - bugCatchDBPath, pathErr := statedir.PathForWrite(statedir.BugCatchDB) - if pathErr != nil { - fmt.Fprintln(stderr, "corral certify --local:", pathErr) - return 2 - } - shadowSels, drawErr := drawShadowSeats("corral certify --local", *repoDirFlag, drawLang, shadow, - map[string]string{advpool.RoleMutantGenerator: *mutantModel, advpool.RoleTestWriter: *writerModel}, - bugCatchDBPath, stderr) - if drawErr != nil { - fmt.Fprintf(stderr, "corral certify --local: %v\n", drawErr) - return 2 - } - + // Every refusal a flag alone decides comes BEFORE --models asks anything: + // an operator must never accept a lineup for a run that was always going + // to be refused. They also precede the shadow draw and seat resolution, + // which read stores and a daemon; checking the command line first is the + // cheaper order, and the order certify --repo keeps. + // // --local-endpoint placements are parsed BEFORE the seats resolve: a // placement is itself a declared source for a seat with a bare local name. localEndpoints, lerr := parseLocalEndpoints(localEndpointFlag) @@ -264,24 +241,6 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write fmt.Fprintf(stderr, "corral certify --local: %v\n", lerr) return 2 } - // Every seat resolves ONCE, here (docs/design/model-registry.md): a - // declared alias becomes its concrete model before anything else reads the - // value, and each seat's provider, wire and endpoint are decided in one - // stated order (resolveSeats) so no later door re-guesses them from the - // model's name. A registry entry places its local seat on its own daemon; - // an explicit --local-endpoint overrides it. With no registry, this - // changes only what a bare name that no source serves does: it refuses. - // - // context.Background(): no cancellable context exists yet — the SIGINT - // handler (auditContext) is installed after the preflight, so an - // interrupt before this point has nothing to unwind. The one network call - // (listing OLLAMA_URL's daemon) has its own 10s bound. - seats, regErr := resolveSeatsPlaced(context.Background(), "corral certify --local", *repoDirFlag, - certifySeats(nil, mutantModel, writerModel, criticModel, shadow.model, shadow.writerModel), localEndpoints, stderr) - if regErr != nil { - fmt.Fprintf(stderr, "corral certify --local: %v\n", regErr) - return 2 - } if strings.TrimSpace(*codePath) == "" { fmt.Fprintln(stderr, "corral certify --local: --code is required") return 2 @@ -333,6 +292,53 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write fmt.Fprintf(stderr, "corral certify --local: --ledger %s is inside --repo-dir %s — --local never writes into the checkout; name a directory outside it\n", l, *repoDirFlag) return 2 } + // --models proposes the seats HERE: after every flag refusal and before + // the shadow draw (which reads the primary writer and generator) and the + // seat resolution, so a proposed member is drawn against and resolved + // exactly as the same model typed into its flag would be. The flags the + // operator did type are pins. + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, modelsPool, []poolRole{ + {role: advpool.RoleMutantGenerator, val: mutantModel}, + {role: advpool.RoleTestWriter, val: writerModel}, + {role: advpool.RoleTestCritic, val: criticModel}, + }, doorPoolDeps(stdout, stderr)) + if poolDone { + return poolExit + } + // The bug-catching scorecard store: a --local run's per-seat metrics + // (including the shadow head-to-head) persist next to the signed build + // ledger and feed the same `corral scorecard` surface the daemon's runs feed. + bugCatchDBPath, pathErr := statedir.PathForWrite(statedir.BugCatchDB) + if pathErr != nil { + fmt.Fprintln(stderr, "corral certify --local:", pathErr) + return 2 + } + shadowSels, drawErr := drawShadowSeats("corral certify --local", *repoDirFlag, drawLang, shadow, + map[string]string{advpool.RoleMutantGenerator: *mutantModel, advpool.RoleTestWriter: *writerModel}, + bugCatchDBPath, stderr) + if drawErr != nil { + fmt.Fprintf(stderr, "corral certify --local: %v\n", drawErr) + return 2 + } + + // Every seat resolves ONCE, here (docs/design/model-registry.md): a + // declared alias becomes its concrete model before anything else reads the + // value, and each seat's provider, wire and endpoint are decided in one + // stated order (resolveSeats) so no later door re-guesses them from the + // model's name. A registry entry places its local seat on its own daemon; + // an explicit --local-endpoint overrides it. With no registry, this + // changes only what a bare name that no source serves does: it refuses. + // + // context.Background(): no cancellable context exists yet — the SIGINT + // handler (auditContext) is installed after the preflight, so an + // interrupt before this point has nothing to unwind. The one network call + // (listing OLLAMA_URL's daemon) has its own 10s bound. + seats, regErr := resolveSeatsPlaced(context.Background(), "corral certify --local", *repoDirFlag, + certifySeats(nil, mutantModel, writerModel, criticModel, shadow.model, shadow.writerModel), localEndpoints, stderr) + if regErr != nil { + fmt.Fprintf(stderr, "corral certify --local: %v\n", regErr) + return 2 + } // --record: collect the run into a replayable tape. The sink is the driver's // EventSink (pool reasoning beats) and is also fed the task lifecycle + diff --git a/cmd/corral/certify_repo.go b/cmd/corral/certify_repo.go index df9aae06..b7e15f63 100644 --- a/cmd/corral/certify_repo.go +++ b/cmd/corral/certify_repo.go @@ -1804,12 +1804,23 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { // One roster, shared by the push and the statement, so the two can never // disagree about which model held which seat. + // + // The goal deriver is on it when it is a seat of this scan (no --goals + // map replaced it), so the row names the model that derived the goals + // the grading seats were held to, and how --models chose it (SeatedBy, + // narrowed to this roster) is attributed to a named model rather than + // dropped. With --goals no deriver ran and none is named. A reader that + // ranks the roster sees the deriver as goal-deriver, which modelrank + // already reports as "not scored", so no seat's measure changes. models := func() map[string]string { m := map[string]string{ "mutant-generator": strings.TrimSpace(*mutantModelFlag), "test-writer": strings.TrimSpace(*writerModelFlag), "test-critic": strings.TrimSpace(*criticModelFlag), } + if strings.TrimSpace(*goalsPath) == "" { + m[roleGoalDeriver] = strings.TrimSpace(*deriveModel) + } for role, v := range m { if v == "" || v == "off" { delete(m, role) @@ -1975,7 +1986,7 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { SeatTools: seatToolsJSON, SeatProviders: roleMapJSON(seatProviders), // How the roster's seats were chosen, narrowed to the roster as - // its providers are (the deriver is not on it). + // its providers are (the deriver is on it only when it ran). SeatedBy: roleMapJSON(advpool.SeatedOnly(seating.SeatedBy, models())), SeatEvidence: seating.SeatEvidence, MinKillRate: minKillRate, MaxProvenMissed: maxProvenMissed, diff --git a/cmd/corral/models_doors_test.go b/cmd/corral/models_doors_test.go index fa6dd505..03386445 100644 --- a/cmd/corral/models_doors_test.go +++ b/cmd/corral/models_doors_test.go @@ -97,6 +97,20 @@ func TestLocalModelsNeedsALanguage(t *testing.T) { } } +// A run the command line already refuses is refused before --models asks +// anything: an operator never accepts a lineup for a run that cannot start. +func TestLocalModelsRefusesUsageBeforeAsking(t *testing.T) { + asked := stubDoorPool(t) + var out, errb bytes.Buffer + code := runCertifyLocal([]string{"--lang", "go", "--code", "a.go", "--models", recordPool, "--", "true"}, &out, &errb) + if code != 2 || !strings.Contains(errb.String(), "--goal is required") { + t.Fatalf("exit %d, want 2 for the missing --goal:\n%s", code, errb.String()) + } + if *asked != 0 || strings.Contains(out.String(), "seats (from --models") { + t.Errorf("asked %d time(s) and printed a plan for a run with no --goal:\n%s", *asked, out.String()) + } +} + // corral demo is the real certify --local, reached by its argv, so the pool // travels there and certify --local proposes the seats (and records how they // were chosen). A seat the pool fills is not passed as an empty flag. diff --git a/cmd/corral/seated_by_record_test.go b/cmd/corral/seated_by_record_test.go index 5d0cf0d9..14662191 100644 --- a/cmd/corral/seated_by_record_test.go +++ b/cmd/corral/seated_by_record_test.go @@ -13,6 +13,7 @@ import ( "github.com/pdbethke/corralai/internal/agentbackend" "github.com/pdbethke/corralai/internal/auditpush" "github.com/pdbethke/corralai/internal/modelrank" + "github.com/pdbethke/corralai/internal/reposcan" ) // How each seat was chosen reaches the WRITTEN entry at every door that @@ -173,6 +174,67 @@ func TestRepoEntryRecordsHowEachSeatWasChosen(t *testing.T) { } } +// A --repo scan with no --goals map runs a goal deriver, so the roster names +// it and the row records how it was chosen: by pool order when --models +// proposed it, by the operator when --derive-model named it. The deriver is +// stubbed, so the "anthropic" seat makes no call. +func TestRepoEntryRecordsHowTheDeriverWasChosen(t *testing.T) { + cases := []struct { + name string + extra []string + want string + }{ + {"proposed by --models", nil, "order"}, + {"named by --derive-model", []string{"--derive-model", "anthropic/claude-sonnet-5"}, "operator"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + root, setPath, _ := telemetryRepoFixture(t) + stubDoorPool(t) + machine := withMachineLedger(t) + noKeys(t) + t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") + wiredAgent(t, "fakec", "claude-json", claudeJSONReply) + prev := certifyRepoDeriver + t.Cleanup(func() { certifyRepoDeriver = prev }) + certifyRepoDeriver = func(rs agentbackend.ResolvedSeat) (reposcan.Deriver, error) { + return newMeteredDeriver(&fakeBackend{reply: "the function returns its documented constant"}, rs.Spec), nil + } + args := append([]string{ + "--repo", root, "--models", recordPool, "--no-goal-cache", "--cache-db", filepath.Join(t.TempDir(), "cache.duckdb"), + "--substrate", substrateWorkspace, "--all", "--mutants", setPath, + }, c.extra...) + var out, errb bytes.Buffer + if code := runCertifyRepo(args, &out, &errb); code != 0 { + t.Fatalf("exit=%d\n%s%s", code, out.String(), errb.String()) + } + entries, err := auditpush.ReadLedgerDir(machine) + if err != nil || len(entries) != 1 { + t.Fatalf("machine ledger: %d entries, %v", len(entries), err) + } + audited := 0 + for _, f := range entries[0].Bundle.Files { + if f.Disposition != "audited" { + continue + } + audited++ + var roster, by map[string]string + _ = json.Unmarshal([]byte(f.ModelsByRole), &roster) + _ = json.Unmarshal([]byte(f.SeatedBy), &by) + if roster[modelrank.SeatGoalDeriver] != "claude-sonnet-5" { + t.Errorf("%s: roster %v does not name the deriver claude-sonnet-5", f.Path, roster) + } + if by[modelrank.SeatGoalDeriver] != c.want { + t.Errorf("%s: SeatedBy %v, want goal-deriver=%s", f.Path, by, c.want) + } + } + if audited != 2 { + t.Fatalf("%d audited rows, want the fixture's two\n%s%s", audited, out.String(), errb.String()) + } + }) + } +} + // Without --models nothing is recorded: the row and the verdict document are // shaped exactly as before, and absence reads as "every seat was named by // flag". diff --git a/cmd/corral/shadow_draw_test.go b/cmd/corral/shadow_draw_test.go index cf154880..c39cb5a0 100644 --- a/cmd/corral/shadow_draw_test.go +++ b/cmd/corral/shadow_draw_test.go @@ -167,8 +167,11 @@ func TestEveryDoorDrawsFromItsPoolFlags(t *testing.T) { run func(args []string, stdout, stderr io.Writer) int args []string }{ - // No --code: refused right after the draw. - {"certify --local", runCertifyLocal, append([]string{"--lang", "go"}, pools...)}, + // A --prior that does not exist: refused after the draw and the + // seat resolution, before any jail or spend. (Missing --code was the + // trigger until the flag refusals moved ahead of the draw, so that + // --models never asks about a run the command line already refuses.) + {"certify --local", runCertifyLocal, append([]string{"--lang", "go", "--code", "a.go", "--goal", "g", "--prior", filepath.Join(t.TempDir(), "no-such-prior")}, pools...)}, // --scope-tests is refused right after the registry, which runs after the draw. {"certify --repo", runCertifyRepo, append([]string{"--repo", t.TempDir(), "--scope-tests"}, pools...)}, } From b1c719b31e5ab0512672f1480f8acb8d0a61a635 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 12:21:19 -0500 Subject: [PATCH 06/14] Make every door say why --models ranks across all languages, because the evidence line claimed 'this scan has more than one' for doctor without --code, and would have for a review, where it is false. seatFromPool now takes an explicit reason (anyLangReason, no default) that is required whenever the language is shadowpool.LangAny and is printed on the plan's evidence line and recorded in seat_evidence. certify --repo passes the several-languages wording, doctor passes 'no single language is known for this check', and review will pass 'a review is not ranked by language'. A door that gives no reason is refused before any plan is shown. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/certify_local.go | 2 +- cmd/corral/certify_repo.go | 2 +- cmd/corral/doctor.go | 2 +- cmd/corral/models_pool.go | 36 +++++++++++++++++++++++++++------- cmd/corral/models_pool_test.go | 31 ++++++++++++++++++++++++++++- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/cmd/corral/certify_local.go b/cmd/corral/certify_local.go index 65d34a11..277bd375 100644 --- a/cmd/corral/certify_local.go +++ b/cmd/corral/certify_local.go @@ -297,7 +297,7 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write // seat resolution, so a proposed member is drawn against and resolved // exactly as the same model typed into its flag would be. The flags the // operator did type are pins. - seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, modelsPool, []poolRole{ + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, "", modelsPool, []poolRole{ {role: advpool.RoleMutantGenerator, val: mutantModel}, {role: advpool.RoleTestWriter, val: writerModel}, {role: advpool.RoleTestCritic, val: criticModel}, diff --git a/cmd/corral/certify_repo.go b/cmd/corral/certify_repo.go index b7e15f63..464a3847 100644 --- a/cmd/corral/certify_repo.go +++ b/cmd/corral/certify_repo.go @@ -456,7 +456,7 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { if *dryRun { seatDeps.confirm, seatDeps.noCredentialCheck = acceptUnasked, true } - seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --repo", *repoDir, scanLang, modelsPool, poolRoles, seatDeps) + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --repo", *repoDir, scanLang, anyLangSeveral, modelsPool, poolRoles, seatDeps) if poolDone { return poolExit } diff --git a/cmd/corral/doctor.go b/cmd/corral/doctor.go index 48c90001..a5f2886b 100644 --- a/cmd/corral/doctor.go +++ b/cmd/corral/doctor.go @@ -115,7 +115,7 @@ func runDoctor(args []string, stdout, stderr io.Writer) int { } seatDeps := doorPoolDeps(stdout, stderr) seatDeps.confirm = acceptUnasked - seating, poolExit, poolDone := seatFromPool(context.Background(), "corral doctor", *repoDir, poolLang, modelsPool, []poolRole{ + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral doctor", *repoDir, poolLang, anyLangUnknown, modelsPool, []poolRole{ {role: roleGoalDeriver, val: deriveModel}, {role: advpool.RoleMutantGenerator, val: mutantModel}, {role: advpool.RoleTestWriter, val: writerModel}, diff --git a/cmd/corral/models_pool.go b/cmd/corral/models_pool.go index bc44e8e3..1654ca80 100644 --- a/cmd/corral/models_pool.go +++ b/cmd/corral/models_pool.go @@ -106,6 +106,24 @@ var doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { // under the plan, that it asked nothing. func acceptUnasked(string) (accepted bool, interactive bool, err error) { return true, true, nil } +// anyLangReason says why a door has no single language to rank by. It is +// spelled out per door and has no default, because the evidence line is the +// operator's record of what the ranking read, and "this scan has more than +// one" printed for a review or a doctor run without --code would be a claim +// about the scan that nobody made. +type anyLangReason string + +const ( + // anyLangSeveral: a scan that audits files in more than one language. + anyLangSeveral anyLangReason = "this scan has more than one" + // anyLangReview: a review's seats read prose about code, and the record + // does not rank a reviewer by the language of what it reviewed. + anyLangReview anyLangReason = "a review is not ranked by language" + // anyLangUnknown: doctor without --code, or with a file whose language + // cannot be detected. + anyLangUnknown anyLangReason = "no single language is known for this check" +) + // seatFromPool returns done=true with an exit code when the door must stop // (refusal: 2; declined or no terminal: 0). With --models unset it returns // a zero poolSeating and done=false, and changes nothing. @@ -116,10 +134,11 @@ func acceptUnasked(string) (accepted bool, interactive bool, err error) { return // is the evidence read, ranked and proposed. Nothing is spent before the // operator accepts, and a lineup with an unfilled seat is never offered. // -// lang is the scan's language, or shadowpool.LangAny for a scan of several -// (the marker certify --repo already hands the shadow draw). It has no -// default: "" is a door that forgot to say, and is refused. -func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, f *modelsFlags, roles []poolRole, deps poolDeps) (seating poolSeating, exit int, done bool) { +// lang is the scan's language, or shadowpool.LangAny when there is no single +// one (the marker certify --repo already hands the shadow draw), in which +// case anyWhy says why and is required. Neither has a default: "" is a door +// that forgot to say, and is refused. +func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy anyLangReason, f *modelsFlags, roles []poolRole, deps poolDeps) (seating poolSeating, exit int, done bool) { if f == nil || f.pool == nil { return poolSeating{}, 0, false } @@ -130,6 +149,9 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, f *models if strings.TrimSpace(lang) == "" { return refuse("no language was given for this scan (pass the scan's language, or %q for a scan of several), so the evidence cannot be filtered", shadowpool.LangAny) } + if lang == shadowpool.LangAny && strings.TrimSpace(string(anyWhy)) == "" { + return refuse("the door gave no reason for ranking across all languages, so the evidence line cannot say why") + } poolText := strings.TrimSpace(*f.pool) typed, err := parseModelsPool(poolText) if err != nil { @@ -163,7 +185,7 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, f *models if err != nil { return refuse("cannot read the evidence: %v — nothing was proposed", err) } - report, evidence := rankPool(ev, lang, pool) + report, evidence := rankPool(ev, lang, anyWhy, pool) plan := autoseat.Propose(autoseat.Input{ Roles: roleNames, Pool: pool, Pins: pins, Report: report, Floor: modelrank.DefaultMinRuns, }) @@ -290,7 +312,7 @@ func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []st // than one per language (the proposer reads a seat's first group only). // Evidence that records no language cannot be filtered to one, and says so // rather than ranking nothing. -func rankPool(ev rankEvidence, lang string, pool []autoseat.Member) (modelrank.Report, string) { +func rankPool(ev rankEvidence, lang string, anyWhy anyLangReason, pool []autoseat.Member) (modelrank.Report, string) { declared := map[string]string{} for _, m := range pool { declared[m.Recorded] = m.Given @@ -312,7 +334,7 @@ func rankPool(ev rankEvidence, lang string, pool []autoseat.Member) (modelrank.R for i := range obs { obs[i].Lang = "" } - scope = "all languages (this scan has more than one)" + scope = "all languages (" + string(anyWhy) + ")" case !recordsLang: scope = "all languages (the evidence records none)" default: diff --git a/cmd/corral/models_pool_test.go b/cmd/corral/models_pool_test.go index 681f0f17..59433795 100644 --- a/cmd/corral/models_pool_test.go +++ b/cmd/corral/models_pool_test.go @@ -129,7 +129,7 @@ func runPool(t *testing.T, root, lang, pool string, vals *certifyVals, load *fak if pool != "" { f = poolFlags(pool) } - seating, exit, done := seatFromPool(context.Background(), "corral certify --local", root, lang, f, vals.roles(), + seating, exit, done := seatFromPool(context.Background(), "corral certify --local", root, lang, anyLangSeveral, f, vals.roles(), poolDeps{load: load.fn, confirm: confirm.fn, stdout: &out, stderr: &errb}) return seating, exit, done, out.String(), errb.String() } @@ -462,6 +462,35 @@ func TestSeatFromPoolLanguageLine(t *testing.T) { } } +// A scan with no single language says WHY on the evidence line, in the +// door's own words, and a door that gives no reason is refused: the line is +// the operator's record of what the ranking read, and one door's reason +// printed by another is a false statement. +func TestSeatFromPoolSaysWhyThereIsNoSingleLanguage(t *testing.T) { + root := poolEnv(t) + for _, c := range []struct { + why anyLangReason + want string + }{ + {anyLangSeveral, "all languages (this scan has more than one)"}, + {anyLangReview, "all languages (a review is not ranked by language)"}, + {anyLangUnknown, "all languages (no single language is known for this check)"}, + } { + var out, errb bytes.Buffer + _, exit, done := seatFromPool(context.Background(), "corral x", root, shadowpool.LangAny, c.why, poolFlags(samplePool), (&certifyVals{}).roles(), + poolDeps{load: (&fakeLoad{ev: sampleEvidence()}).fn, confirm: (&fakeConfirm{accepted: true, interactive: true}).fn, stdout: &out, stderr: &errb}) + if exit != 0 || done || !strings.Contains(out.String(), "; "+c.want+"\n") { + t.Errorf("%q: exit=%d done=%v stderr=%q\n%s", c.why, exit, done, errb.String(), out.String()) + } + } + var out, errb bytes.Buffer + _, exit, done := seatFromPool(context.Background(), "corral x", root, shadowpool.LangAny, "", poolFlags(samplePool), (&certifyVals{}).roles(), + poolDeps{load: (&fakeLoad{ev: sampleEvidence()}).fn, confirm: (&fakeConfirm{accepted: true, interactive: true}).fn, stdout: &out, stderr: &errb}) + if exit != 2 || !done || !strings.Contains(errb.String(), "reason") || out.Len() != 0 { + t.Errorf("no reason: exit=%d done=%v stderr=%q out=%q, want a refusal that shows no plan", exit, done, errb.String(), out.String()) + } +} + func TestRegisterModelsFlags(t *testing.T) { fs := flag.NewFlagSet("t", flag.ContinueOnError) fs.SetOutput(io.Discard) From f5e2090822e483d3f00ce985b2af368a219bdfbd Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 12:29:57 -0500 Subject: [PATCH 07/14] Wire --models into corral review and let --verifier-model off name no verifier, so a pool can seat the reviewer and verifier from the review evidence and the entry records how each was chosen. --models proposes the reviewer and the verifier, ranked across all languages (a review is not ranked by language, and the plan says so). Every usage refusal that needs no seat (missing --scope, an unexpected argument, the ledger signer, a non-commit checkout) now precedes the plan, so nobody is asked to accept a run that was always going to be refused; only the missing reviewer is decided after it. review had no way to say 'no verifier' that --models could respect, so --verifier-model off now means exactly that, as --critic-model off does on certify: it is a pin to --models, and is normalised to an empty verifier before any check, so a run with off writes an entry identical in shape to one with no flag and the word is never compared to the reviewer as a model. seated_by and seat_evidence are recorded beside seat_providers, for a seat only when it is on the entry, and a run without --models writes neither. The pending-doors exemption in the gate test is removed now that every seat-flag door is wired, and the CLI reference is regenerated. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/models_flags_test.go | 13 +- cmd/corral/review.go | 56 +++++- cmd/corral/review_models_test.go | 206 +++++++++++++++++++++++ docs/cli/corral.md | 6 +- site/src/content/docs/docs/cli/corral.md | 6 +- testdata/executed-surfaces.tsv | 1 + 6 files changed, 269 insertions(+), 19 deletions(-) create mode 100644 cmd/corral/review_models_test.go diff --git a/cmd/corral/models_flags_test.go b/cmd/corral/models_flags_test.go index 565feadb..363691a2 100644 --- a/cmd/corral/models_flags_test.go +++ b/cmd/corral/models_flags_test.go @@ -21,13 +21,6 @@ var seatFlagLiteral = regexp.MustCompile(`\.String(Var)?\([^)]*"(writer|reviewer // shapes the flag package offers. var modelsFlagLiteral = regexp.MustCompile(`\.(String|StringVar|Func|Var)\([^)]*"models"`) -// pendingModelsDoors are doors that register a seat flag and are not wired to -// --models yet, each with the reason. An entry for a door that now calls the -// helper fails the test below, so the list cannot outlive the gap it names. -var pendingModelsDoors = map[string]string{ - "review.go": "corral review is wired in its own change (autoseat phase 2, task 5): its seats are reviewer and verifier, not certify's", -} - func TestModelsFlagPatternsMatchARegistration(t *testing.T) { // Negative controls: each pattern must catch the shape it exists to // catch, or the door test below passes vacuously. @@ -69,11 +62,7 @@ func TestEveryDoorWithSeatFlagsRegistersModelsFlags(t *testing.T) { } doors++ wired := strings.Contains(src, "registerModelsFlags(") - why, pending := pendingModelsDoors[f] - switch { - case pending && wired: - t.Errorf("%s now calls registerModelsFlags; remove its pendingModelsDoors entry (%s)", f, why) - case !pending && !wired: + if !wired { t.Errorf("%s registers a seat flag and not --models; call registerModelsFlags", f) } } diff --git a/cmd/corral/review.go b/cmd/corral/review.go index 9933b37d..4ed57f4e 100644 --- a/cmd/corral/review.go +++ b/cmd/corral/review.go @@ -28,6 +28,7 @@ import ( "github.com/pdbethke/corralai/internal/modelrank" "github.com/pdbethke/corralai/internal/review" "github.com/pdbethke/corralai/internal/sandbox" + "github.com/pdbethke/corralai/internal/shadowpool" ) // runReview implements `corral review`: a cold reviewer seat over a scope, @@ -106,6 +107,7 @@ type reviewFlags struct { substrate string repoDir, scope, model, verifier, ledger, attest, push, failOn string noLedger, pushSource bool + models *modelsFlags timeout time.Duration maxBytes int } @@ -123,7 +125,8 @@ func reviewFlagSet(out io.Writer) (*flag.FlagSet, *reviewFlags) { fs.StringVar(&f.repoDir, "repo", ".", "the checkout to review (a git repository at a commit)") fs.StringVar(&f.scope, "scope", "", "the directory or file under --repo to review (required)") fs.StringVar(&f.model, "reviewer-model", "", "the reviewer seat — an alias from the registry, a hosted model as vendor/model (`anthropic/…`, `google/…`, `openai/…`; a bare cloud name is refused), a local model the OLLAMA_URL daemon lists, or an AGENTIC seat: an agent NAME, pinned to a model as `:`. `claude-code`, `codex` and `antigravity` are defined; ANY agent is defined the same way as CORRALAI_AGENT_=\"\" ({dir} the worktree, {out} a reply file, {model} / {model:FLAG} the pin; the brief on stdin, the reply on stdout). An agentic seat is a coding agent started in a disposable copy of the repository: it reads the whole scope itself (no --max-bytes cap) and hands back scripts, which corral runs — nothing it did itself is on the record. Required; corral has no default models") - fs.StringVar(&f.verifier, "verifier-model", "", "a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named") + fs.StringVar(&f.verifier, "verifier-model", "", "a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named: `off` names no verifier, the same word --critic-model off uses on certify. With --models a verifier is proposed unless you pin it with this flag or pin `off`") + f.models = registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool") fs.StringVar(&f.ledger, "ledger", "", "the ledger directory the review entry is written to (default: /.corral/ledger, or $CORRAL_LEDGER). A copy of the entry also goes to this machine's own ledger (the state directory's ledger, see corral state paths; override CORRALAI_MACHINE_LEDGER); a machine ledger that resolves inside the checkout is skipped, with the reason printed") fs.StringVar(&f.attest, "attest", "", "write an in-toto statement (predicate https://corralai.dev/review/v1) to this path, and its DSSE envelope beside it when a certify key is configured: the REPRODUCTIONS — every finding's declared and recorded tier, the hash of its script and output, its exit, the verifier's refutation on the same terms — signed; the opinion bound by its hash and not carried. The ledger entry then names the statement. corral verify --attest --db recomputes the reproductions' hash from the entry") fs.StringVar(&f.push, "push", "", "also append the review's rows to a warehouse you own — a DuckDB path, or md: — as corral_reviews and corral_findings, keyed by the entry's hash so a later review adjudicate --push joins to them. Scripts and outputs travel as hashes unless --push-source") @@ -152,8 +155,12 @@ func runReviewRun(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "corral review: unexpected argument %q\n", fs.Arg(0)) return 2 } - if strings.TrimSpace(*scope) == "" || strings.TrimSpace(*model) == "" { - fmt.Fprintln(stderr, "corral review: --scope and --reviewer-model are required") + // Every refusal that needs no seat comes before --models is asked about, + // so nobody is shown a plan and asked to accept a run that was always + // going to be refused. Only the missing reviewer, which --models can fill, + // is decided after it. + if strings.TrimSpace(*scope) == "" { + fmt.Fprintln(stderr, "corral review: --scope is required, and a reviewer: --reviewer-model, or --models to have one proposed") return 2 } // The entry is signed after the seats are paid; a signer that cannot be @@ -174,6 +181,29 @@ func runReviewRun(args []string, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "corral review: --repo is not a git checkout at a commit — a review names the revision it reviewed") return 2 } + // --models proposes the reviewer and the verifier here, after every + // refusal that needs no seat and before the seats are resolved, so a + // proposed member is resolved exactly as the same model typed into its + // flag. A review is not ranked by language. `--verifier-model off` is a + // pin: the operator chose no verifier, and --models leaves that alone. + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral review", root, shadowpool.LangAny, anyLangReview, f.models, []poolRole{ + {role: modelrank.SeatReviewer, val: model}, + {role: modelrank.SeatVerifier, val: verifier}, + }, doorPoolDeps(stdout, stderr)) + if poolDone { + return poolExit + } + if strings.TrimSpace(*model) == "" { + fmt.Fprintln(stderr, "corral review: --scope is required, and a reviewer: --reviewer-model, or --models to have one proposed") + return 2 + } + // `off` names no verifier. From here on an operator who said off and one + // who said nothing are the same run, so the entry, the roster and the + // decorrelation check never see the word as a model. + if strings.EqualFold(strings.TrimSpace(*verifier), "off") { + *verifier = "" + delete(seating.SeatedBy, modelrank.SeatVerifier) + } // context.Background(): no cancellable context exists yet — the SIGINT // handler (auditContext) is installed after the substrate is chosen, so an // interrupt before this point has nothing to unwind. The one network call @@ -307,6 +337,7 @@ func runReviewRun(args []string, stdout, stderr io.Writer) int { Substrate: substrateOf(rep), StartedAt: time.Now().UTC(), FilesShown: sc.Files, BytesShown: sc.Bytes, Truncated: sc.Truncated} recordSeatProvider(&r, modelrank.SeatReviewer, seats["reviewer-model"]) + recordSeatedBy(&r, modelrank.SeatReviewer, seating) userTurn := review.Brief(repoName, commit, *scope, sc) if reviewerIsAgent { userTurn = review.AgentBrief(repoName, commit, *scope, allFiles) @@ -376,6 +407,7 @@ func runReviewRun(args []string, stdout, stderr io.Writer) int { r.VerifierTool = verifierTool r.VerifierModelResolved = agentbackend.SeatNamesAModel(*verifier) recordSeatProvider(&r, modelrank.SeatVerifier, seats["verifier-model"]) + recordSeatedBy(&r, modelrank.SeatVerifier, seating) review.Verify(ctx, rep, &r, *verifier, vopinion, refs) } } @@ -507,6 +539,24 @@ func recordSeatProvider(r *review.Review, seat string, rs agentbackend.ResolvedS r.SeatProviders[seat] = p } +// recordSeatedBy records how one seat was chosen, beside the seat's provider +// and under the same rule: only a seat that is on the entry is recorded (a +// verifier that failed is not), and a run without --models records nothing, +// so its entry keeps the shape it always had. The ranking's evidence line is +// recorded whenever any seat was recorded, since it explains the +// evidence-seated ones. +func recordSeatedBy(r *review.Review, seat string, seating poolSeating) { + by, ok := seating.SeatedBy[seat] + if !ok { + return + } + if r.SeatedBy == nil { + r.SeatedBy = map[string]string{} + } + r.SeatedBy[seat] = by + r.SeatEvidence = seating.SeatEvidence +} + // reviewSeatTools are the tools an agent seat in review is granted: it reads // the tree, so it may read, search and list it — and nothing else. A // certify seat is granted none; its prompt carries the code. diff --git a/cmd/corral/review_models_test.go b/cmd/corral/review_models_test.go new file mode 100644 index 00000000..65745690 --- /dev/null +++ b/cmd/corral/review_models_test.go @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "bytes" + "io" + "path/filepath" + "strings" + "testing" + + "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/auditpush" + "github.com/pdbethke/corralai/internal/modelrank" + "github.com/pdbethke/corralai/internal/review" +) + +// reviewPool names two hosted models; reviewEvidence measures both as +// reviewer and as verifier at the floor, "reviewer-x" being the better +// reviewer and "verifier-y" the better verifier, so an unpinned proposal is +// reviewer-x and verifier-y. +const reviewPool = "anthropic/reviewer-x,anthropic/verifier-y" + +func reviewEvidence() rankEvidence { + var obs []modelrank.Observation + add := func(model, role string, checked, held, calls, correct int) { + for _, run := range []string{"r1", "r2", "r3", "r4", "r5", "r6"} { + obs = append(obs, modelrank.Observation{Model: model, Role: role, Run: run, + ReviewClaimsChecked: checked, ReviewClaimsHeld: held, VerifierCalls: calls, VerifierCorrect: correct}) + } + } + add("reviewer-x", modelrank.SeatReviewer, 2, 2, 0, 0) + add("verifier-y", modelrank.SeatReviewer, 2, 1, 0, 0) + add("verifier-y", modelrank.SeatVerifier, 0, 0, 2, 2) + add("reviewer-x", modelrank.SeatVerifier, 0, 0, 2, 1) + return rankEvidence{Obs: obs, Source: "test evidence (6 review(s))", Generations: generationRange{Min: "48", Max: "48", Scans: 6}} +} + +// stubReviewPool replaces the doors' seams with reviewEvidence and an +// operator who accepts, and reports how often the operator was asked. +func stubReviewPool(t *testing.T) *int { + t.Helper() + asked := new(int) + orig := doorPoolDeps + t.Cleanup(func() { doorPoolDeps = orig }) + doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{ + load: func(rankSource) (rankEvidence, error) { return reviewEvidence(), nil }, + confirm: func(string) (bool, bool, error) { *asked++; return true, true, nil }, + stdout: stdout, stderr: stderr, + } + } + origRun := shadowMemberRunnable + t.Cleanup(func() { shadowMemberRunnable = origRun }) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } + return asked +} + +// reviewModelsRun runs one review over a tiny committed repo with canned +// seats and returns the single entry it wrote, so the tests read what was +// recorded and not an intermediate. +func reviewModelsRun(t *testing.T, args ...string) (rev review.Review, code int, out, errs string, seen map[string]int) { + t.Helper() + requireReviewSandbox(t) + declareLocalSeats(t, "reviewer-x", "verifier-y") + t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") + ledger := filepath.Join(t.TempDir(), "machine") + t.Setenv("CORRALAI_MACHINE_LEDGER", ledger) + root := t.TempDir() + gitRun := gitCmd(t, root) + mustWrite(t, filepath.Join(root, "pkg", "a.go"), "package pkg\n\nfunc Add(a, b int) int { return a - b }\n") + mustWrite(t, filepath.Join(root, "go.mod"), "module x\n\ngo 1.22\n") + gitRun("init", "-q") + gitRun("add", ".") + gitRun("commit", "-q", "-m", "base", "--no-gpg-sign") + + seen = map[string]int{} + orig := newReviewerBackend + t.Cleanup(func() { newReviewerBackend = orig }) + newReviewerBackend = func(rs agentbackend.ResolvedSeat, _ agentbackend.SeatOptions) (agentbackend.Backend, error) { + seen[rs.Spec]++ + if rs.Spec == "verifier-y" { + return cannedReviewer{reply: `{"opinion":"v","refutations":[]}`}, nil + } + return cannedReviewer{reply: `{"opinion":"o","findings":[{"claim":"a caller might overflow","tier":"HYPOTHESIS"}],"sound":[]}`}, nil + } + var o, e bytes.Buffer + code = runReview(append([]string{"--repo", root, "--scope", "pkg"}, args...), &o, &e) + entries, _ := auditpush.ReadLedgerDir(filepath.Join(root, ".corral", "ledger")) + if code == 0 { + if len(entries) != 1 || entries[0].Review == nil { + t.Fatalf("want one review entry, got %d\n%s%s", len(entries), o.String(), e.String()) + } + rev = *entries[0].Review + } + return rev, code, o.String(), e.String(), seen +} + +func TestReviewSeatedByModels(t *testing.T) { + asked := stubReviewPool(t) + rev, code, out, errs, _ := reviewModelsRun(t, "--models", reviewPool) + if code != 0 { + t.Fatalf("exit %d\n%s%s", code, out, errs) + } + if *asked != 1 { + t.Errorf("asked %d times, want once", *asked) + } + if rev.SeatedBy[modelrank.SeatReviewer] != "evidence" || rev.SeatedBy[modelrank.SeatVerifier] != "evidence" || len(rev.SeatedBy) != 2 { + t.Errorf("seated_by = %v, want both seats by evidence", rev.SeatedBy) + } + if !strings.Contains(rev.SeatEvidence, "all languages (a review is not ranked by language)") { + t.Errorf("seat_evidence = %q", rev.SeatEvidence) + } + if rev.ReviewerModel != "reviewer-x" || rev.VerifierModel != "verifier-y" { + t.Errorf("seats %q / %q, want reviewer-x / verifier-y", rev.ReviewerModel, rev.VerifierModel) + } +} + +// The proposer never seats the reviewer as the verifier, even when it is the +// better verifier too. +func TestReviewModelsNeverSeatsTheReviewerAsVerifier(t *testing.T) { + stubReviewPool(t) + ev := reviewEvidence() + for i := range ev.Obs { + if ev.Obs[i].Model == "reviewer-x" && ev.Obs[i].Role == modelrank.SeatVerifier { + ev.Obs[i].VerifierCorrect = 2 + } + } + doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{load: func(rankSource) (rankEvidence, error) { return ev, nil }, + confirm: func(string) (bool, bool, error) { return true, true, nil }, stdout: stdout, stderr: stderr} + } + rev, code, out, errs, _ := reviewModelsRun(t, "--models", reviewPool, "--reviewer-model", "anthropic/reviewer-x") + if code != 0 { + t.Fatalf("exit %d\n%s%s", code, out, errs) + } + if rev.VerifierModel == rev.ReviewerModel || rev.ReviewerModel != "reviewer-x" { + t.Errorf("reviewer %q verifier %q", rev.ReviewerModel, rev.VerifierModel) + } + if rev.SeatedBy[modelrank.SeatReviewer] != "operator" { + t.Errorf("seated_by = %v", rev.SeatedBy) + } +} + +// --verifier-model off is a pin: no verifier is proposed or run, the plan +// names the flag, and the entry carries nothing for a verifier. +func TestReviewModelsWithVerifierOffProposesOnlyTheReviewer(t *testing.T) { + stubReviewPool(t) + rev, code, out, errs, seen := reviewModelsRun(t, "--models", reviewPool, "--verifier-model", "off") + if code != 0 { + t.Fatalf("exit %d\n%s%s", code, out, errs) + } + if got := planRow(out, "verifier"); !strings.Contains(got, "operator") || !strings.Contains(got, "pinned by --verifier-model") { + t.Errorf("plan verifier row = %q", got) + } + if seen["verifier-y"] != 0 || seen["off"] != 0 || rev.VerifierModel != "" || rev.VerifierTool != "" { + t.Errorf("a verifier ran or was recorded: seen=%v entry verifier=%q", seen, rev.VerifierModel) + } + if _, has := rev.SeatedBy[modelrank.SeatVerifier]; has || rev.SeatedBy[modelrank.SeatReviewer] != "evidence" { + t.Errorf("seated_by = %v, want the reviewer only", rev.SeatedBy) + } + if _, has := rev.SeatProviders[modelrank.SeatVerifier]; has { + t.Errorf("seat_providers = %v", rev.SeatProviders) + } +} + +// Without --models, `off` is the same run as no verifier flag at all, and a +// run without --models records neither field. +func TestReviewVerifierOffIsNoVerifier(t *testing.T) { + withOff, code, out, errs, seen := reviewModelsRun(t, "--reviewer-model", "reviewer-x", "--verifier-model", "OFF ") + if code != 0 { + t.Fatalf("exit %d\n%s%s", code, out, errs) + } + without, code, out, errs, _ := reviewModelsRun(t, "--reviewer-model", "reviewer-x") + if code != 0 { + t.Fatalf("exit %d\n%s%s", code, out, errs) + } + if seen["off"] != 0 || len(seen) != 1 { + t.Errorf("seats built: %v, want only the reviewer", seen) + } + for _, r := range []review.Review{withOff, without} { + if r.VerifierModel != "" || r.SeatedBy != nil || r.SeatEvidence != "" || len(r.SeatProviders) != 1 { + t.Errorf("entry not shaped as a no-verifier, no --models run: %+v", r) + } + } +} + +// Usage refusals that need no seat come before --models is asked about. +func TestReviewModelsRefusesUsageBeforeAskingAnything(t *testing.T) { + asked := stubReviewPool(t) + var o, e bytes.Buffer + if code := runReview([]string{"--models", reviewPool}, &o, &e); code != 2 || *asked != 0 || strings.Contains(o.String(), "seats (from --models") { + t.Errorf("no --scope: exit %d asked %d\n%s%s", code, *asked, o.String(), e.String()) + } + if !strings.Contains(e.String(), "--scope is required") { + t.Errorf("stderr = %q", e.String()) + } + // A reviewer is required, and --models can fill it; with no pool and no + // reviewer the refusal stands. + o.Reset() + e.Reset() + root := t.TempDir() + if code := runReview([]string{"--repo", root, "--scope", "x", "--no-ledger"}, &o, &e); code == 0 { + t.Errorf("no reviewer was accepted") + } +} diff --git a/docs/cli/corral.md b/docs/cli/corral.md index d8185ddf..338750c6 100644 --- a/docs/cli/corral.md +++ b/docs/cli/corral.md @@ -972,6 +972,8 @@ flags of `corral review`: the ledger directory the review entry is written to (default: /.corral/ledger, or $CORRAL_LEDGER). A copy of the entry also goes to this machine's own ledger (the state directory's ledger, see corral state paths; override CORRALAI_MACHINE_LEDGER); a machine ledger that resolves inside the checkout is skipped, with the reason printed -max-bytes int how many bytes of the scope the reviewer is shown; files past the cap are listed by name and the review records them as unshown (default 200000) + -models POOL + a comma-separated POOL of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool -no-ledger print the review and write no entry, in either ledger -push string @@ -988,8 +990,8 @@ flags of `corral review`: where the reproduction scripts run: jail (default) — the offline sandbox certify grades in, over the tree at the commit with your dependency directories read-only; or `workspace` — a detached git worktree on the host, network on, model keys and your desktop session (DISPLAY, WAYLAND_DISPLAY, ...) removed from the environment, for where the caller is the isolation boundary (a CI runner, as with certify --repo --substrate workspace) (default "jail") -timeout duration wall-clock bound on each reproduction script (default 1m0s) - -verifier-model string - a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named + -verifier-model off + a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named: off names no verifier, the same word --critic-model off uses on certify. With --models a verifier is proposed unless you pin it with this flag or pin `off` ``` ## `corral review adjudicate` flags diff --git a/site/src/content/docs/docs/cli/corral.md b/site/src/content/docs/docs/cli/corral.md index d8185ddf..338750c6 100644 --- a/site/src/content/docs/docs/cli/corral.md +++ b/site/src/content/docs/docs/cli/corral.md @@ -972,6 +972,8 @@ flags of `corral review`: the ledger directory the review entry is written to (default: /.corral/ledger, or $CORRAL_LEDGER). A copy of the entry also goes to this machine's own ledger (the state directory's ledger, see corral state paths; override CORRALAI_MACHINE_LEDGER); a machine ledger that resolves inside the checkout is skipped, with the reason printed -max-bytes int how many bytes of the scope the reviewer is shown; files past the cap are listed by name and the review records them as unshown (default 200000) + -models POOL + a comma-separated POOL of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool -no-ledger print the review and write no entry, in either ledger -push string @@ -988,8 +990,8 @@ flags of `corral review`: where the reproduction scripts run: jail (default) — the offline sandbox certify grades in, over the tree at the commit with your dependency directories read-only; or `workspace` — a detached git worktree on the host, network on, model keys and your desktop session (DISPLAY, WAYLAND_DISPLAY, ...) removed from the environment, for where the caller is the isolation boundary (a CI runner, as with certify --repo --substrate workspace) (default "jail") -timeout duration wall-clock bound on each reproduction script (default 1m0s) - -verifier-model string - a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named + -verifier-model off + a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named: off names no verifier, the same word --critic-model off uses on certify. With --models a verifier is proposed unless you pin it with this flag or pin `off` ``` ## `corral review adjudicate` flags diff --git a/testdata/executed-surfaces.tsv b/testdata/executed-surfaces.tsv index 05b83f01..070e0af7 100644 --- a/testdata/executed-surfaces.tsv +++ b/testdata/executed-surfaces.tsv @@ -239,6 +239,7 @@ corral review adjudicate --confirm executed test:cmd/corral/review_test.go corral review adjudicate --reason executed test:cmd/corral/review_test.go corral review adjudicate --refute executed test:cmd/corral/review_test.go corral review --ledger executed test:cmd/corral/review_test.go +corral review --models executed test:cmd/corral/review_models_test.go (TestReviewSeatedByModels, TestReviewModelsWithVerifierOffProposesOnlyTheReviewer: a fake ledger and an accepting operator; the real machine ledger and terminal are never read) corral review --repo executed test:cmd/corral/review_test.go corral review --reviewer-model executed test:cmd/corral/review_test.go corral review --scope executed test:cmd/corral/review_test.go From 9cc1f5676dd5c6306588cd55c54beeb39d71aa9a Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 12:37:05 -0500 Subject: [PATCH 08/14] Say that the ranking selects only through --models, among models the operator named on that run, because "disclosure, never selection" stopped being true when --models gave the ranking a reader that seats models. The package comment, corral models rank's help and printed line, the top-level usage, the design notes and two diagrams now say so, and keep the router incident that the rule exists to prevent; the command itself still seats nothing, and its test now pins the new sentence. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/main.go | 6 ++++-- cmd/corral/models_rank.go | 12 +++++++---- cmd/corral/models_rank_test.go | 7 +++--- docs/cli/corral.md | 26 ++++++++++++++++------- docs/design/adversarial-review-loop.svg | 2 +- docs/design/corral-today.svg | 2 +- docs/design/model-ranking.md | 27 ++++++++++++++++++++---- docs/design/shadow-seat-selection.md | 6 ++++++ internal/modelrank/rank.go | 21 ++++++++++++------ site/src/content/docs/docs/cli/corral.md | 26 ++++++++++++++++------- 10 files changed, 97 insertions(+), 38 deletions(-) diff --git a/cmd/corral/main.go b/cmd/corral/main.go index a346b8df..2c4c9300 100644 --- a/cmd/corral/main.go +++ b/cmd/corral/main.go @@ -262,8 +262,10 @@ Usage: adjudication; the goal-deriver is reported as not scored rather than given an invented number. A model below --min-runs (default 5) is printed with its real numbers, marked insufficient, and never - preferred. DISCLOSURE, NOT SELECTION: it writes no config, changes - no default and staffs no seat — corral has no default models. + preferred. DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS + THE OPERATOR NAMED ON THAT RUN: this command writes no config, + changes no default and staffs no seat — corral has no default + models. flags: --db (a pushed warehouse instead of the local bug-catching ledger; unreachable REFUSES, never falls back) --seat --lang --min-runs n --json diff --git a/cmd/corral/models_rank.go b/cmd/corral/models_rank.go index 34cf2aa0..9729dd27 100644 --- a/cmd/corral/models_rank.go +++ b/cmd/corral/models_rank.go @@ -157,9 +157,13 @@ const modelsUsage = `usage: corral models rank [--source ledger|scorecard] [--db evidence says about them — a different metric per seat, because the seats do different jobs. - This is DISCLOSURE, NOT SELECTION. It prints a table. It writes no config, - changes no default, and feeds no router: corral has no default models, and a - ranking that quietly staffed a seat would put one back.` + This is DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE + OPERATOR NAMED ON THAT RUN. This command prints a table and seats nothing. It + writes no config, changes no default, and feeds no router: corral has no + default models, and a ranking that quietly staffed a seat would put one back. + The same ranking is read when you pass --models to certify, doctor, demo or + review: there it proposes seats only from the pool you named on that run, + and prints the lineup before anything runs.` func runModelsRank(args []string, repoRoot string, load rankLoader, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("models rank", flag.ContinueOnError) @@ -307,7 +311,7 @@ func printRankReport(rep modelrank.Report, reg *models.Registry, gen generationR if !rep.LangDimension { fmt.Fprintln(w, "language: this evidence records none, so every row is across all languages.") } - fmt.Fprintln(w, "DISCLOSURE, NOT SELECTION: this command sets no default and staffs no seat. corral has no default models.") + fmt.Fprintln(w, "DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE OPERATOR NAMED ON THAT RUN: this command sets no default and staffs no seat. corral has no default models.") if len(rep.Groups) == 0 { fmt.Fprintln(w, "\nno recorded observations for any seat yet.") return diff --git a/cmd/corral/models_rank_test.go b/cmd/corral/models_rank_test.go index d8a473ac..8613c905 100644 --- a/cmd/corral/models_rank_test.go +++ b/cmd/corral/models_rank_test.go @@ -59,13 +59,14 @@ func runRank(t *testing.T, args []string, obs []modelrank.Observation) (int, str // The whole command's reason for existing, in one assertion: the ranking is // evidence a human reads, and it says so on its face. -func TestRankSaysItIsDisclosureNotSelection(t *testing.T) { +func TestRankSaysItIsDisclosureAndSelectsOnlyThroughModels(t *testing.T) { rc, out, _ := runRank(t, nil, writerRuns("m", "", 6, 5, 10)) if rc != 0 { t.Fatalf("rc=%d", rc) } - if !strings.Contains(out, "DISCLOSURE, NOT SELECTION") { - t.Fatalf("the table must say it sets no default:\n%s", out) + if !strings.Contains(out, "DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE OPERATOR NAMED ON THAT RUN") || + !strings.Contains(out, "this command sets no default and staffs no seat") { + t.Fatalf("the table must say it sets no default and that only --models selects:\n%s", out) } if !strings.Contains(out, "min-runs: 5") { t.Fatalf("the default evidence floor must be stated:\n%s", out) diff --git a/docs/cli/corral.md b/docs/cli/corral.md index 338750c6..92afb003 100644 --- a/docs/cli/corral.md +++ b/docs/cli/corral.md @@ -87,8 +87,10 @@ Usage: adjudication; the goal-deriver is reported as not scored rather than given an invented number. A model below --min-runs (default 5) is printed with its real numbers, marked insufficient, and never - preferred. DISCLOSURE, NOT SELECTION: it writes no config, changes - no default and staffs no seat — corral has no default models. + preferred. DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS + THE OPERATOR NAMED ON THAT RUN: this command writes no config, + changes no default and staffs no seat — corral has no default + models. flags: --db (a pushed warehouse instead of the local bug-catching ledger; unreachable REFUSES, never falls back) --seat --lang --min-runs n --json @@ -878,9 +880,13 @@ usage: corral models rank [--source ledger|scorecard] [--db ] [--sea evidence says about them — a different metric per seat, because the seats do different jobs. - This is DISCLOSURE, NOT SELECTION. It prints a table. It writes no config, - changes no default, and feeds no router: corral has no default models, and a - ranking that quietly staffed a seat would put one back. + This is DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE + OPERATOR NAMED ON THAT RUN. This command prints a table and seats nothing. It + writes no config, changes no default, and feeds no router: corral has no + default models, and a ranking that quietly staffed a seat would put one back. + The same ranking is read when you pass --models to certify, doctor, demo or + review: there it proposes seats only from the pool you named on that run, + and prints the lineup before anything runs. ``` ## `corral models rank` flags @@ -902,9 +908,13 @@ usage: corral models rank [--source ledger|scorecard] [--db ] [--sea evidence says about them — a different metric per seat, because the seats do different jobs. - This is DISCLOSURE, NOT SELECTION. It prints a table. It writes no config, - changes no default, and feeds no router: corral has no default models, and a - ranking that quietly staffed a seat would put one back. + This is DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE + OPERATOR NAMED ON THAT RUN. This command prints a table and seats nothing. It + writes no config, changes no default, and feeds no router: corral has no + default models, and a ranking that quietly staffed a seat would put one back. + The same ranking is read when you pass --models to certify, doctor, demo or + review: there it proposes seats only from the pool you named on that run, + and prints the lineup before anything runs. flags: -db md: diff --git a/docs/design/adversarial-review-loop.svg b/docs/design/adversarial-review-loop.svg index 32b8eca8..76b62a14 100644 --- a/docs/design/adversarial-review-loop.svg +++ b/docs/design/adversarial-review-loop.svg @@ -225,7 +225,7 @@ corral models rank - disclosure, not selection + disclosure; staffs no seat diff --git a/docs/design/corral-today.svg b/docs/design/corral-today.svg index b390c642..39f7c4fe 100644 --- a/docs/design/corral-today.svg +++ b/docs/design/corral-today.svg @@ -169,7 +169,7 @@ models rank per language · evidence floor of 5 - disclosure, not selection + disclosure; staffs no seat The herd is the allowlist: nothing here staffs a seat you did not diff --git a/docs/design/model-ranking.md b/docs/design/model-ranking.md index fa55b76e..e70b5bdb 100644 --- a/docs/design/model-ranking.md +++ b/docs/design/model-ranking.md @@ -88,11 +88,15 @@ A ranking that promotes the 100% row is worse than no ranking at all, because it is confidently wrong in the operator's favour: it recommends the expensive model on three observations over the one with thirty-eight runs behind it. -## Disclosure, never selection +## Disclosure; selection only through `--models` **This command prints a table. It writes no config, changes no default, fills no seat, and feeds no router.** +(This section was titled "Disclosure, never selection" until `--models` was +added. The ranking now has one reader that seats models, described at the end +of this section; the command itself is unchanged.) + That is not a limitation left for later — it is the design. corral has no default models: every seat is named by the operator or the run refuses to start, because a verdict is only worth something if you know who produced it. @@ -105,9 +109,24 @@ well kept scoring well in a ranking nobody re-derived. The routing was "earned", which is precisely why nobody looked at it. So the boundary is drawn at the output: the ranking is a document a human -reads before typing a model name, and there is no code path from it to a seat. -The help text and the table's own header say so, so that an operator reading -the output cannot mistake it for something that has already acted. +reads before typing a model name. The help text and the table's own header say +so, so that an operator reading the output cannot mistake it for something +that has already acted. + +There is one code path from the ranking to a seat, and it is built against the +incident above. `--models a,b,c` on `certify --local`, `certify --repo`, +`doctor`, `demo` and `review` hands the ranking, filtered to the pool, to +`internal/autoseat`, a pure function and the only code that decides a seat from +evidence (`TestOnlyAutoseatDecidesASeatFromEvidence` holds that). It can seat +only a model the operator named in that run's pool, so a model that ranks well +but was not named is never chosen; it uses this page's floor and never lowers +it, refusing a seat no member clears; it prints the lineup with each seat's +reason, and nothing runs until the operator accepts it on a terminal. The +entry records how each seat was chosen (`SeatedBy`) and what the ranking read +(`SeatEvidence`). The router in the incident re-selected a model nobody had +named on the run; the guard here is that the operator names the candidates +every time. The operator-facing description is in +[docs/corral/agent-seats.md](../corral/agent-seats.md). Two smaller rules follow from the same principle: diff --git a/docs/design/shadow-seat-selection.md b/docs/design/shadow-seat-selection.md index 691cba4d..4e62efef 100644 --- a/docs/design/shadow-seat-selection.md +++ b/docs/design/shadow-seat-selection.md @@ -28,6 +28,12 @@ keeps statistics out of every seat that gates. It also addresses that incident's actual mechanism, which was greedy: routing that never re-tests an early winner. Thompson sampling keeps re-testing by construction. +(The rule in model-ranking.md now reads "disclosure; selection only through +`--models`, among models the operator named on that run". `--models` proposes +verdict seats from a pool the operator names and runs nothing until the +operator accepts the lineup; it does not change the shadow draw described +here.) + The shadow seats are the right place to start because they already never gate. `Shadow` on a seat is the driver's own marker for that (`internal/advpool/driver.go`), and shadow rows are already flagged in the diff --git a/internal/modelrank/rank.go b/internal/modelrank/rank.go index 071788fd..5ab04bb0 100644 --- a/internal/modelrank/rank.go +++ b/internal/modelrank/rank.go @@ -15,13 +15,20 @@ // not judged at all: no recorded outcome is attributable to it, so this // package reports it as unscored rather than inventing a number for it. // -// DISCLOSURE, NEVER SELECTION. Nothing here writes config, fills a seat, or -// feeds a router. corral has no default models — every seat is named by the -// operator or the run refuses — and a ranking that quietly became an input to -// staffing would reintroduce that rule through the back door. It has happened -// in this project before: a production router's performance statistics -// OVERRODE the operator's configured model list and re-selected a retired -// model, forever. The output of this package is a table for a human to read. +// DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE OPERATOR NAMED +// ON THAT RUN. Nothing here writes config, fills a seat, or feeds a router. +// corral has no default models — every seat is named by the operator or the +// run refuses — and a ranking that quietly became an input to staffing would +// reintroduce that rule through the back door. It has happened in this project +// before: a production router's performance statistics OVERRODE the operator's +// configured model list and re-selected a retired model, forever. The output +// of this package is a table for a human to read, and one other reader: +// internal/autoseat, which a certify, doctor, demo or review run consults only +// when the operator passes --models. It seats a model only if the operator +// named it in that run's pool, prints the lineup with each seat's reason, and +// the run starts only after the operator accepts it. A model the operator did +// not name on the run cannot be seated however it ranks, which is the guard +// against the router incident above. // // THIN EVIDENCE IS NOT A RANKING. A model with a perfect rate over three // observations has not earned a recommendation; the live scorecard has exactly diff --git a/site/src/content/docs/docs/cli/corral.md b/site/src/content/docs/docs/cli/corral.md index 338750c6..92afb003 100644 --- a/site/src/content/docs/docs/cli/corral.md +++ b/site/src/content/docs/docs/cli/corral.md @@ -87,8 +87,10 @@ Usage: adjudication; the goal-deriver is reported as not scored rather than given an invented number. A model below --min-runs (default 5) is printed with its real numbers, marked insufficient, and never - preferred. DISCLOSURE, NOT SELECTION: it writes no config, changes - no default and staffs no seat — corral has no default models. + preferred. DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS + THE OPERATOR NAMED ON THAT RUN: this command writes no config, + changes no default and staffs no seat — corral has no default + models. flags: --db (a pushed warehouse instead of the local bug-catching ledger; unreachable REFUSES, never falls back) --seat --lang --min-runs n --json @@ -878,9 +880,13 @@ usage: corral models rank [--source ledger|scorecard] [--db ] [--sea evidence says about them — a different metric per seat, because the seats do different jobs. - This is DISCLOSURE, NOT SELECTION. It prints a table. It writes no config, - changes no default, and feeds no router: corral has no default models, and a - ranking that quietly staffed a seat would put one back. + This is DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE + OPERATOR NAMED ON THAT RUN. This command prints a table and seats nothing. It + writes no config, changes no default, and feeds no router: corral has no + default models, and a ranking that quietly staffed a seat would put one back. + The same ranking is read when you pass --models to certify, doctor, demo or + review: there it proposes seats only from the pool you named on that run, + and prints the lineup before anything runs. ``` ## `corral models rank` flags @@ -902,9 +908,13 @@ usage: corral models rank [--source ledger|scorecard] [--db ] [--sea evidence says about them — a different metric per seat, because the seats do different jobs. - This is DISCLOSURE, NOT SELECTION. It prints a table. It writes no config, - changes no default, and feeds no router: corral has no default models, and a - ranking that quietly staffed a seat would put one back. + This is DISCLOSURE; SELECTION ONLY THROUGH --models, AMONG MODELS THE + OPERATOR NAMED ON THAT RUN. This command prints a table and seats nothing. It + writes no config, changes no default, and feeds no router: corral has no + default models, and a ranking that quietly staffed a seat would put one back. + The same ranking is read when you pass --models to certify, doctor, demo or + review: there it proposes seats only from the pool you named on that run, + and prints the lineup before anything runs. flags: -db md: From 7b4cd7d65861d74adae8bf037baebfadf8284b08 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 12:37:05 -0500 Subject: [PATCH 09/14] Document --models for the people who will type it and the agents who will change it: how to name a pool, what the plan shows, what refuses and why, what the entry records, and that --autoseat and exploration are not in this release. Every statement was checked against the code on this branch, the sample plan is a test's fixture labelled as one, and no run against a real ledger is claimed because none has happened yet. Co-Authored-By: Claude Fable 5.2 --- AGENTS.md | 31 ++++++ CHANGELOG.md | 16 +++ README.md | 21 +++- ROADMAP.md | 8 ++ docs/corral/agent-seats.md | 100 ++++++++++++++++++ .../docs/docs/concepts/multi-model-herds.mdx | 10 +- 6 files changed, 184 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ed023331..8acee8cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -302,6 +302,37 @@ This is deliberate. corral claims to be model-agnostic, and a binary that names one vendor's models when the operator named none is making an exception to that claim. We run Claude; we do not make anyone else. +**`--models a,b,c` lets the record propose seats, from models the operator +named on that run. A pool is not a default.** `certify --local`, `certify +--repo`, `doctor`, `demo` (which forwards it to `certify --local`) and `review` +take it. Every member is resolved and credential-checked before anything is +proposed (`certify --repo --dry-run` checks no member's credential, as it +checks none for any seat); a seat named by its own flag is a pin and wins; the measured seats +(mutant-generator, test-writer, reviewer, verifier) take the best-ranked pool +member with at least `modelrank.DefaultMinRuns` (5) observations in that seat's +own unit; the goal-deriver takes the first member in the order the operator +typed, and the critic, unless it has such a row, the first member that is not +the writer, preferring one from a different provider. A seat nothing +qualifies for is refused with the flag that would fill it, never filled from +outside the pool and never by lowering the floor. The plan is printed and +nothing runs until the operator answers `y` or `yes` on a terminal (anything +else declines, exit 0); with no terminal it +prints the equivalent seat flags and exits 0. There is no `--yes`. `doctor` and +`certify --repo --dry-run` print the plan and ask nothing. The evidence is the +machine ledger only, with no fallback to another source. The proposer is +`internal/autoseat` (a pure function) and it is the one door: +`TestOnlyAutoseatDecidesASeatFromEvidence` fails any other file in `cmd/corral` +that imports it, and `models_pool.go` is the only caller. How each seat was +chosen is recorded on the entry as `SeatedBy` (`operator`, `evidence` or +`order`, keyed by role) beside `SeatEvidence` (what the ranking read), both +tier DECLARED; a seat filled by an accepted proposal is recorded as `evidence` +or `order`, never as `operator`, because a person pressing `y` did not choose +the model. The machine ledger carries no critic adjudications, so on this +evidence the critic is always seated by order. +`corral models rank` itself still seats nothing. `--autoseat` (seat without +asking) and exploration (seating below-floor members in the shadow seats) are +NOT built; do not document them as features. + **The shadow challenger is OFF unless named**, and it doubles the mutant count when on. It used to default to a Claude model and be on, which quietly kept an Anthropic seat alive through an otherwise all-Gemini run. diff --git a/CHANGELOG.md b/CHANGELOG.md index cc15612d..a4a5f079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,22 @@ history of any release, `git log v0.3.4..v0.3.5`. ## [Unreleased] +### Added + +- **`--models` proposes which of the models you name takes each seat, from this machine's ledger.** `certify --local`, `certify --repo`, `doctor`, `demo` and `review` take `--models a,b,c`, a pool written the way a seat flag takes a model (`vendor/model`, a registry alias, a local model, an agent seat). A pool is not a default: only a model you named on the run can be seated. A seat you also name by its own flag is pinned and keeps your value. The measured seats (mutant-generator, test-writer, and on `review` the reviewer and verifier) go to the pool member `corral models rank` puts first among those with at least 5 observations in the seat's own unit: runs for the generator, survivors attempted for the writer, claims checked for the reviewer, verdicts for the verifier; equal rates go to the member named first. The goal-deriver goes to the first member, and the test-critic to the first member that is not the writer, preferring one from a different provider. The critic would be seated by its adjudication record if it had 5 adjudications, but the machine ledger carries none, so today it is always seated by order. A bare agent seat (`codex`) is never ranked; `codex:` is. + - *What you see.* Before anything is spent, the plan: one row per seat with the model, how it was chosen (`operator`, `evidence` or `order`) and why, then the evidence line (the machine ledger, its scan and review counts, and the language the ranking was read for) and the range of grading rules it spans. On a terminal corral asks `run with these seats? [y/N]`; only `y` or `yes` runs, and anything else prints `not run; nothing was spent` and exits 0. With no terminal nothing runs: corral prints the plan and the lineup as seat flags (`to run this lineup: --mutant-model … --writer-model …`) and exits 0. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask nothing; the dry run checks no member's credential, as it checks none for any seat. `demo` hands the pool to `certify --local`, which proposes the seats for Go. Under `certify --repo --json` the plan goes to stderr. + - *What refuses, exit 2, nothing spent.* A pool with an empty member, a member named twice, two members that resolve to one model, or `off`; `--models` given twice; a member that cannot run (every member is resolved and credential-checked first, whether or not it would be seated); a measured seat no member has 5 observations for, shown in the plan as `REFUSED` with its fix (`no model in --models has 5+ survivors attempted as test-writer; pin --writer-model`); a critic that would be the writer, or a verifier that would be the reviewer, including the critic of a one-model pool (`--critic-model off` satisfies it); a machine ledger that cannot be read or does not verify; and on `certify --local` a file whose language cannot be detected (pass `--lang`). The floor is never lowered, no model outside the pool is used, and no other evidence source is read in place of the machine ledger. A machine whose ledger has no entries yet refuses every measured seat you did not pin. When every seat is pinned, `--models` reads no evidence and asks nothing. + - *What is recorded.* Each entry written with `--models` carries `SeatedBy`, how each seat was chosen keyed by seat, and `SeatEvidence`, the evidence line, when any seat was seated by evidence. Both are declared fields. A seat you accepted from the plan is recorded as `evidence` or `order`, never as `operator`. An entry written without `--models` carries neither, as before. + - *Not in this release.* Seating a proposal without asking (`--autoseat`), and seating pool members below the floor in the shadow seats so that they gather evidence. +- **`review --verifier-model off` names no verifier**, the word `--critic-model off` already uses on `certify`. The entry is the same as one written with no verifier flag. With `--models` it is a pin: without it, `--models` proposes a verifier. + +### Changed + +- **`certify --repo` without `--goals` names the goal-deriver on the entry's roster.** `ModelsByRole` on the ledger entry, and the roster in the statement `--attest` writes, now hold `goal-deriver` with the model that derived the goals, so how `--models` chose it is attributed to a model. With `--goals` no deriver runs and none is named. `corral models rank` therefore lists such scans under the goal-deriver, which it reports as not scored, and no scored seat's ranking changes. +- **`demo` does not require `--writer-model` and `--mutant-model` when `--models` is given**; `certify --local` refuses any seat the pool cannot fill. +- **`corral models rank` says it is "disclosure; selection only through `--models`, among models the operator named on that run"**, where it said "disclosure, not selection". The command itself still seats nothing. +- **One-time cache miss.** `reposcan.VerdictGeneration` is now 48 (rc.35 shipped 47), because the verdict gained `SeatedBy` and `SeatEvidence`. Every cached verdict is measured once more; nothing a cached verdict measured was wrong. + ### Fixed - **`corral mcp` starts in a second agent session.** The findings server opened the local findings store read-write and held it for as long as it ran. DuckDB allows one read-write process per file, so the first session's server locked the store: every later session's server failed at start-up with `Could not set lock on file`, and `certify --local` could not record findings while any agent session stayed open. The server now opens the store read-only for each tool call and closes it before answering, so an idle server holds no lock. diff --git a/README.md b/README.md index 366528b9..d9df0cd6 100644 --- a/README.md +++ b/README.md @@ -454,7 +454,26 @@ wrote ledger entries (rc.26), are shown only under `--source scorecard`, which reads the old local bug-catching store instead. `--db` reads a ledger directory or a warehouse that `certify --repo --push` filled. A ledger that does not verify is refused, and no source falls back to -another. It prints a ranking; it never staffs a seat. +another. `corral models rank` prints a ranking and staffs no seat. + +**Letting the record propose the seats: `--models`.** Name a pool and corral +proposes which member takes each seat, from this machine's ledger: +`--models anthropic/claude-sonnet-5-5,anthropic/claude-haiku-5-5,codex:gpt-6` +(examples, not defaults) on `certify --local`, `certify --repo`, `doctor`, +`demo` or `review`. A pool is not a default: only models you named on that run +can be seated. A seat you also name by its own flag is pinned. The measured +seats (generator, writer, reviewer, verifier) go to the best-ranked member with +at least 5 observations in that seat's unit; the goal-deriver and the critic +go by the order you named the pool, because the ledger holds nothing that ranks +them, the critic never to the writer's model and preferably to another +provider. corral prints the lineup with the reason for +each seat and runs nothing until you answer `y` or `yes` on a terminal. With no terminal +it prints the lineup as seat flags and exits 0; there is no `--yes`. `doctor` +and `certify --repo --dry-run` print it and ask nothing. A seat no member +qualifies for is refused with the flag that would fill it, never filled from +outside the pool. The entry records how each seat was chosen (`SeatedBy`) and +what the ranking read (`SeatEvidence`). Details and every refusal: +[docs/corral/agent-seats.md](docs/corral/agent-seats.md#a-pool-of-seats---models). The levers that bound it: `corral doctor` (below) catches environment failures for free before a run spends anything; `--top` bounds a whole-repo scan to the diff --git a/ROADMAP.md b/ROADMAP.md index 2e7b000f..43b8053c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -142,6 +142,14 @@ of these has happened, with the evidence linked here: - **`--max-tokens` caps a certify run** across every seat, goal derivation included (rc.20), and holds under parallel calls (rc.22). `corral review` has no token cap yet. +- **`--models` lets the record propose the seats (not yet in a release).** The + operator names a pool; corral seats the measured seats from this machine's + ledger and the others by the order the pool was named, prints the lineup, + and runs nothing until the operator accepts it. Only named models can be + seated, a measured seat no member has 5 observations for is refused, and the entry + records how each seat was chosen (`SeatedBy`). Next, and not built: seating + without asking (`--autoseat`), and seating members below the evidence floor + in the shadow seats so that they gather evidence. **The review — a stranger reads your code.** - **`corral review --scope`.** A coding agent that has never seen the repository diff --git a/docs/corral/agent-seats.md b/docs/corral/agent-seats.md index 308e304f..cc6065ae 100644 --- a/docs/corral/agent-seats.md +++ b/docs/corral/agent-seats.md @@ -238,6 +238,106 @@ shell-wrapped definition proceeds unchecked. An Ollama tag that starts with an agent's name, such as `codex:latest`, reads as that agent pinned to the model `latest`; rename the model or undefine the agent. +## A pool of seats: `--models` + +`--models` names a pool of models and lets corral propose which of them takes +each seat, from what this machine's ledger recorded about them. +`certify --local`, `certify --repo`, `doctor`, `demo` and `review` take it. +A member is written exactly as a seat flag takes it: `vendor/model`, a +registry alias, a local model, an agent seat (`codex`) or a pinned agent seat +(`codex:gpt-6`). + +```bash +corral demo --models anthropic/claude-sonnet-5-5,anthropic/claude-haiku-5-5,codex:gpt-6 +``` + +The models in that command are examples, not defaults. A pool is not a +default either: corral seats only models you named on the run. + +How each seat is filled: + +- A seat you also name with its own flag (`--writer-model`, `--critic-model off` + and so on) is pinned, and the pin wins. +- The measured seats (mutant-generator, test-writer, and on `review` the + reviewer and verifier) go to the pool member that `corral models rank` puts + first among those with at least 5 observations in that seat's own unit: + runs for the generator (in the machine ledger a run is one scan), survivors + attempted for the writer, claims checked for the reviewer, verdicts for the + verifier. Equal rates go to the member you named first. +- The goal-deriver (on `certify --repo` without `--goals`, and on `doctor`) is + not measured, so it goes to the first member you named. The critic goes to + the first member that is not the writer, preferring one from a different + provider. A critic is measured only by human adjudication, which the machine + ledger does not carry, so on this evidence it is always seated this way. +- A bare agent seat (`codex`) names a tool, not a model, so nothing is ranked + under it: it can take a seat by pin or by pool order, never by evidence. + `codex:gpt-6` is ranked like any other model. + +Before anything is proposed, every member is resolved and its credential (or +its CLI) checked, whether or not it will be seated. `certify --repo --dry-run` +checks no member's credential, as it checks none for any seat. Then the plan is +printed on stdout (on stderr under `certify --repo --json`), one row per seat: +the seat, the model, how it was chosen (`operator`, `evidence` or `order`) and +why, then what the ranking read and, when the evidence records them, the +grading rules it spans. An illustration of a `certify --repo` plan for a +Python scan (the numbers are a test's fixture, not a measurement; `certify +--local` and `demo` have no goal-deriver row): + +``` +seats (from --models anthropic/claude-sonnet-5-5,anthropic/claude-haiku-5-5,codex:gpt-6): + goal-deriver claude-sonnet-5-5 order no measurement for this seat: taken from the order you named + mutant-generator claude-sonnet-5-5 evidence 60 planted, 54 graded, 6 invalid, 24 survived over 6 runs + test-writer claude-sonnet-5-5 evidence 193/195 survivors proven over 6 runs + test-critic gpt-6 (codex) order no measurement for this seat: taken from the order you named +evidence: the machine ledger /state/ledger (31 scan(s), 2 review(s); a run is one SCAN); language python +grading rules: generations 40 to 43 across 6 scan(s) with evidence +``` + +On a terminal corral then asks `run with these seats? [y/N]`. Only `y` or `yes` +runs; any other answer prints `not run; nothing was spent` and exits 0. With no +terminal (a pipe, a CI job) nothing runs: corral prints the plan and the same +lineup as seat flags (`to run this lineup: --mutant-model ... --writer-model ...`) +and exits 0, so you can run it deliberately. There is no `--yes`. `doctor` and +`certify --repo --dry-run` print the plan, say it is the lineup a real run +would propose, and ask nothing. + +What refuses, before anything is spent (exit 2): + +- a pool with an empty member, a member named twice, two members that resolve + to the same model, or `off` (pin a seat off with its own flag instead); +- `--models` given twice; +- a member that cannot run, for example a missing key; +- a measured seat no member has 5 or more observations for. The plan marks it + `REFUSED` and names the fix, for example + `no model in --models has 5+ survivors attempted as test-writer; pin --writer-model`. + The floor is never lowered and no model outside the pool is used. A machine + with no entries yet refuses every measured seat that is not pinned; +- a critic or verifier that would be the same model as its writer or reviewer, + including a one-model pool's critic (`--critic-model off` satisfies it); +- a machine ledger that cannot be read or does not verify. Nothing falls back + to another source; +- on `certify --local`, a file whose language cannot be detected (pass `--lang`). + +When every seat is pinned, `--models` seats nothing, reads no evidence and asks +nothing. On `review`, `--verifier-model off` pins no verifier. + +The ranking is read for the audited file's language on `certify --local` and +`demo` (Go), for the scan's language on `certify --repo` when it has exactly +one (otherwise across all languages), for `--code`'s language on +`doctor` (across all languages without it), and across all languages on +`review`. The evidence line says which. + +The entry records how each seat was chosen as `SeatedBy`, keyed by seat, and, +when a seat was seated by evidence, what the ranking read as `SeatEvidence`; +both are declared fields +([record-fields.md](record-fields.md)). A seat you accepted from the plan is +recorded as `evidence` or `order`, not `operator`. On `certify --repo` without +`--goals` the entry's roster also names the goal-deriver, so its seating is +attributed to a model. `corral models rank` itself still seats nothing. + +Not in this release: seating a proposal without asking (`--autoseat`), and +seating members below the floor in the shadow seats to gather evidence on them. + ## In analysis `corral models rank`, the bug-catching scorecard and the public warehouse's diff --git a/site/src/content/docs/docs/concepts/multi-model-herds.mdx b/site/src/content/docs/docs/concepts/multi-model-herds.mdx index f5235346..abc145e5 100644 --- a/site/src/content/docs/docs/concepts/multi-model-herds.mdx +++ b/site/src/content/docs/docs/concepts/multi-model-herds.mdx @@ -72,7 +72,15 @@ reach and how to define your own are in corral's own recorded evidence — a different metric per seat (the writer by proven gaps per survivor attempted, the generator by valid mutants the dev suite missed, the critic by precision against human adjudication). It is - disclosure, not selection: it writes no config and staffs no seat. + disclosure: it writes no config and staffs no seat. +- **`--models`** is the one way the ranking selects. Name a pool + (`--models a,b,c`) on `certify`, `doctor`, `demo` or `review`, and corral + proposes which member takes each seat from this machine's ledger, prints the + lineup with the reason for each seat, and runs nothing until you accept it on + a terminal. Only models you named on that run can be seated, a seat you name + by its own flag is pinned, and a seat no member has enough evidence for is + refused. See + [agent seats](https://github.com/pdbethke/corralai/blob/main/docs/corral/agent-seats.md). ## The optional daemon From e47249d9bbc42d6e71ab997b7a5b071316eefeea Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 13:09:57 -0500 Subject: [PATCH 10/14] Resolve --models pins with the members' resolver and check the lineup by the model that would run, because comparing typed spellings let one model grade its own work under a second name: a writer seated as anthropic/claude-sonnet-5-5 beside a critic pinned as claude-code:claude-sonnet-5-5 was not refused, and a writer pinned by alias was offered again as the critic. The proposer now takes resolved pins (autoseat.Pin: the typed text, the resolved name and the provider), compares resolved names in the hard rule and in "the critic is not the writer" and "the verifier is not the reviewer", prefers a different provider than a pinned writer as it did for a member, reports every broken pair in Roles order with the flags that fix it, enforces its own evidence floor instead of trusting Row.Sufficient, and says when the only member with enough evidence is already the counterpart. A pin no source serves is refused with the resolver's own words before any evidence is read. The recorded seat evidence no longer carries the operator's state path: the machine-ledger loader gives a second, path-free name built from the same counts, the plan keeps printing the directory, and a record keeps the evidence line only while a seat it names was seated by evidence (the verdict, the --repo row and the review entry alike). The plan adds one line when there is no evidence at all, the no-terminal path ends by saying nothing was audited, the prompt needs stdout to be a terminal as well as stdin, a member a placement alone serves names --local-endpoint in its refusal, and a scan with no detected language says so instead of "more than one". Co-Authored-By: Claude Fable 5.2 --- cmd/corral/models_pool.go | 155 +++++++++++++--- cmd/corral/models_pool_test.go | 246 +++++++++++++++++++++++++- cmd/corral/models_rank.go | 23 ++- cmd/corral/review.go | 9 +- cmd/corral/review_models_test.go | 111 ++++++++++-- cmd/corral/seated_by_record_test.go | 2 +- internal/advpool/driver.go | 11 +- internal/advpool/run.go | 6 +- internal/advpool/seated_by_test.go | 20 +++ internal/autoseat/autoseat.go | 154 ++++++++++------ internal/autoseat/autoseat_test.go | 141 +++++++++++++-- internal/ledger/row.go | 12 +- internal/reviewrecord/reviewrecord.go | 4 +- 13 files changed, 769 insertions(+), 125 deletions(-) diff --git a/cmd/corral/models_pool.go b/cmd/corral/models_pool.go index 1654ca80..07279af6 100644 --- a/cmd/corral/models_pool.go +++ b/cmd/corral/models_pool.go @@ -12,11 +12,13 @@ import ( "os" "strings" "text/tabwriter" + "time" "golang.org/x/term" "github.com/pdbethke/corralai/internal/agentbackend" "github.com/pdbethke/corralai/internal/autoseat" + "github.com/pdbethke/corralai/internal/hoststat" "github.com/pdbethke/corralai/internal/modelrank" "github.com/pdbethke/corralai/internal/seatrecord" "github.com/pdbethke/corralai/internal/shadowpool" @@ -47,8 +49,20 @@ func registerModelsFlags(fs *flag.FlagSet, help string) *modelsFlags { // poolSeating is what a door records. type poolSeating struct { - SeatedBy map[string]string // role -> operator|evidence|order; nil when --models was not given - SeatEvidence string // what the ranking read, as the plan's evidence line says it; "" when no seat was seated by evidence + SeatedBy map[string]string // role -> operator|evidence|order; nil when --models was not given + // SeatEvidence is what the ranking read: the source named without any + // host path (rankEvidence.RecordSource), then the language scope exactly + // as the plan printed it. "" when no seat was seated by evidence. + SeatEvidence string +} + +// seatEvidenceFor is the evidence line a record keeps beside seatedBy: the +// line only when a seat that record names was seated by evidence. A door +// narrows SeatedBy to the seats on its entry (a verifier that did not run, +// a roster without the deriver), and an evidence line left beside a record +// with no evidence seat would explain a choice the record does not hold. +func seatEvidenceFor(seatedBy map[string]string, evidence string) string { + return autoseat.EvidenceFor(seatedBy, evidence) } // proposed reports whether --models chose any seat (by evidence or order), @@ -80,6 +94,12 @@ type poolDeps struct { confirm func(plan string) (accepted bool, interactive bool, err error) stdout io.Writer stderr io.Writer + // placed is the door's --local-endpoint placements (role -> daemon + // URL, as parseLocalEndpoints returns them), or nil where the door + // takes none. A pin is resolved with them, exactly as the door's own + // resolution will resolve it, so a pin served only by its placement + // resolves here too. + placed map[string]string // noCredentialCheck skips each member's credential check, and is set // by certify --repo --dry-run alone: a dry run demands no credential // for any seat (validateShadowPools is called there with @@ -108,14 +128,19 @@ func acceptUnasked(string) (accepted bool, interactive bool, err error) { return // anyLangReason says why a door has no single language to rank by. It is // spelled out per door and has no default, because the evidence line is the -// operator's record of what the ranking read, and "this scan has more than -// one" printed for a review or a doctor run without --code would be a claim -// about the scan that nobody made. +// operator's record of what the ranking read, and "the walked sources are in +// more than one" printed for a review, a doctor run without --code or a scan +// that detected no language would be a claim about the scan that nobody made. type anyLangReason string const ( - // anyLangSeveral: a scan that audits files in more than one language. - anyLangSeveral anyLangReason = "this scan has more than one" + // anyLangSeveral: a scan whose walked sources are in more than one + // language. It speaks of the walk, the superset the evidence pass + // instruments, because --top or --diff-base may audit fewer files than + // the walk found and so fewer languages. + anyLangSeveral anyLangReason = "the walked sources are in more than one" + // anyLangNone: a scan whose walk detected no language at all. + anyLangNone anyLangReason = "no language was detected in this scan" // anyLangReview: a review's seats read prose about code, and the record // does not rank a reviewer by the language of what it reviewed. anyLangReview anyLangReason = "a review is not ranked by language" @@ -157,7 +182,8 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an if err != nil { return refuse("%v", err) } - pool, err := resolveModelsPool(ctx, cmdName, repoRoot, typed, !deps.noCredentialCheck) + listing := &ollamaListing{} + pool, err := resolveModelsPool(ctx, cmdName, repoRoot, typed, !deps.noCredentialCheck, listing) if err != nil { return refuse("%v", err) } @@ -181,16 +207,31 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an return poolSeating{SeatedBy: by}, 0, false } + // The pins are resolved as the members were, before any evidence is + // read: the hard rule and the critic's provider preference compare the + // models that would run, and a pin no source serves is refused here + // with the resolver's own words rather than after the operator accepts. + resolvedPins, err := resolvePins(ctx, cmdName, repoRoot, roles, pins, deps.placed, listing) + if err != nil { + return refuse("%v", err) + } + ev, err := deps.load(rankSource{Kind: rankSourceLedger}) if err != nil { return refuse("cannot read the evidence: %v — nothing was proposed", err) } - report, evidence := rankPool(ev, lang, anyWhy, pool) + if strings.TrimSpace(ev.RecordSource) == "" { + // The recorded line is built from this name, never by cutting the + // path out of the printed one; a loader that gives none has said + // nothing the record could keep. + return refuse("the evidence source gave no name to record without its path, so how a seat was chosen could not be recorded — nothing was proposed") + } + report, scope := rankPool(ev, lang, anyWhy, pool) plan := autoseat.Propose(autoseat.Input{ - Roles: roleNames, Pool: pool, Pins: pins, Report: report, Floor: modelrank.DefaultMinRuns, + Roles: roleNames, Pool: pool, Pins: resolvedPins, Report: report, Floor: modelrank.DefaultMinRuns, }) - text := renderPoolPlan(poolText, plan, evidence, ev.Generations.Line()) + text := renderPoolPlan(poolText, plan, ev.Source+"; "+scope, ev.Generations.Line(), len(ev.Obs) == 0) fmt.Fprint(deps.stdout, text) if plan.Refused { return refuse("the proposed lineup is refused (the plan above names each unfilled seat and what would fill it)") @@ -211,6 +252,7 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an } } fmt.Fprintln(deps.stdout, b.String()) + fmt.Fprintln(deps.stdout, noTerminalLine) return poolSeating{}, 0, true case !accepted: fmt.Fprintln(deps.stdout, "not run; nothing was spent") @@ -227,13 +269,56 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an // runs next and rewrites an alias or vendor/model exactly as it // would have had the operator typed this flag. *roles[i].val = s.Given - if s.By == autoseat.ByEvidence { - seating.SeatEvidence = evidence - } } + // The record keeps the source without its host path: the entry is + // signed and may be published, and a home directory is not evidence. + seating.SeatEvidence = seatEvidenceFor(seating.SeatedBy, ev.RecordSource+"; "+scope) return seating, 0, false } +// restartRunClock stops the sampler a door started before its --models +// question and starts the clock and a new sampler at the operator's answer. +// A door calls it only when a plan was accepted (poolSeating.proposed), so a +// run without --models, or with every seat pinned, is timed exactly as it +// always was. +func restartRunClock(old *hoststat.Sampler) (time.Time, *hoststat.Sampler) { + old.Stop() + return time.Now(), hoststat.Start(hostSampleInterval) +} + +// noTerminalLine is the last line a --models run with no terminal prints. It +// exits 0, as a decline does (spec 5.3), so a pipeline that only reads the +// exit code goes green; this line is what a person reading the log sees. +const noTerminalLine = "nothing was audited: --models without a terminal only prints the plan and exits 0; pass the seat flags above to run this lineup" + +// resolvePins resolves every pin but "off" with the resolver the members +// use, sharing their daemon listing, and with the door's --local-endpoint +// placements. Each pin resolves a copy, so the door's flag keeps what the +// operator typed and the door's own resolution, which runs after the +// operator accepts, rewrites it exactly as it always has. +func resolvePins(ctx context.Context, cmdName, repoRoot string, roles []poolRole, pins map[string]string, placed map[string]string, listing *ollamaListing) (map[string]autoseat.Pin, error) { + out := make(map[string]autoseat.Pin, len(pins)) + for _, r := range roles { + given, ok := pins[r.role] + if !ok { + continue + } + if strings.EqualFold(given, "off") { + out[r.role] = autoseat.Pin{Given: given} + continue + } + flagName := autoseat.PinFlag[r.role] + concrete := given + set, err := resolveSeatsListed(ctx, cmdName, repoRoot, []seatFlag{{flag: flagName, role: r.role, val: &concrete}}, placed, io.Discard, listing) + if err != nil { + return nil, err + } + rs := set[flagName] + out[r.role] = autoseat.Pin{Given: given, Recorded: rs.Spec, Provider: rs.Provider} + } + return out, nil +} + // parseModelsPool splits --models. shadowpool.ParsePool is not reused: it // requires two members, because a challenger pool of one is no draw, whereas // a --models pool of one is a legitimate request that seats every seat but @@ -265,14 +350,19 @@ func parseModelsPool(text string) ([]string, error) { // checkCredentials is false (poolDeps.noCredentialCheck). The // resolver rewrites the value it is handed, so it resolves a copy and the // member keeps what the operator typed as Given. -func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []string, checkCredentials bool) ([]autoseat.Member, error) { - listing := &ollamaListing{} +func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []string, checkCredentials bool, listing *ollamaListing) ([]autoseat.Member, error) { byRecorded := map[string]string{} var out []autoseat.Member for _, given := range typed { concrete := given set, err := resolveSeatsListed(ctx, cmdName, repoRoot, []seatFlag{{flag: "models", val: &concrete}}, nil, io.Discard, listing) if err != nil { + var ns errNoSource + if errors.As(err, &ns) { + // Members resolve with no placements: a placement belongs + // to one seat, and a member is not yet any seat's. + return nil, fmt.Errorf("member %q: %v — a model served only through --local-endpoint cannot be a --models member; name it with its own seat flag and place it with --local-endpoint", given, err) + } return nil, fmt.Errorf("member %q: %v", given, err) } rs := set["models"] @@ -305,7 +395,9 @@ func resolveModelsPool(ctx context.Context, cmdName, repoRoot string, typed []st } // rankPool ranks the evidence for this scan and returns the report with the -// plan's evidence line (the source verbatim, then the language it covers). +// language scope the evidence line ends with. The caller puts a source in +// front of it: the printed source for the plan, the path-free one for the +// record, so both say the same scope in the same words. // // A scan of several languages ranks across all of them: the language is // cleared on a copy of the observations, so each seat is one group rather @@ -341,14 +433,19 @@ func rankPool(ev rankEvidence, lang string, anyWhy anyLangReason, pool []autosea opt.Lang = lang scope = "language " + lang } - return modelrank.Rank(obs, opt), ev.Source + "; " + scope + return modelrank.Rank(obs, opt), scope } // renderPoolPlan is the plan as spec 5.3 draws it: one row per role (role, // model, how it was chosen, why), then what the ranking read and the grading // rules it spans. Columns are aligned by tabwriter, so a long model name // widens its column instead of breaking the table. -func renderPoolPlan(poolText string, plan autoseat.Plan, evidence, generations string) string { +// +// noEvidence adds one plain line under the evidence when the ranking read no +// observation at all: the source line alone ("no entries yet") does not say +// what that means for the plan, which is that every measured seat is refused +// until it is pinned. +func renderPoolPlan(poolText string, plan autoseat.Plan, evidence, generations string, noEvidence bool) string { var b strings.Builder b.WriteString("seats (from --models " + poolText + "):\n") tw := tabwriter.NewWriter(&b, 0, 0, 2, ' ', 0) @@ -367,6 +464,9 @@ func renderPoolPlan(poolText string, plan autoseat.Plan, evidence, generations s if generations != "" { b.WriteString("grading rules: " + generations + "\n") } + if noEvidence { + b.WriteString("no recorded evidence yet: every measured seat must be pinned until this machine has runs on its ledger\n") + } return b.String() } @@ -383,13 +483,20 @@ func seatDisplay(s autoseat.Seat) string { return name } -// terminalConfirm asks on in, when in is a terminal, after the caller has -// printed the plan (so the plan argument is not printed again). Anything but -// y or yes declines. When in is not a terminal it asks nothing and reports -// interactive=false: a pipe or a CI log cannot accept a lineup. +// isTerminal is term.IsTerminal, a variable so a test can say which ends of +// the conversation are terminals without allocating a pty. +var isTerminal = term.IsTerminal + +// terminalConfirm asks on in, when both in and out are terminals, after the +// caller has printed the plan to out (so the plan argument is not printed +// again). Anything but y or yes declines. Otherwise it asks nothing and +// reports interactive=false: a pipe or a CI log cannot accept a lineup, and +// neither can a terminal whose plan and prompt went to a redirected stdout, +// because a "y" typed there would accept a lineup nobody saw. func terminalConfirm(in *os.File, out io.Writer) func(plan string) (bool, bool, error) { return func(string) (bool, bool, error) { - if !term.IsTerminal(int(in.Fd())) { // #nosec G115 -- a file descriptor fits in an int + outFile, ok := out.(*os.File) + if !ok || !isTerminal(int(in.Fd())) || !isTerminal(int(outFile.Fd())) { // #nosec G115 -- a file descriptor fits in an int return false, false, nil } fmt.Fprint(out, "run with these seats? [y/N] ") diff --git a/cmd/corral/models_pool_test.go b/cmd/corral/models_pool_test.go index 59433795..cf4f4218 100644 --- a/cmd/corral/models_pool_test.go +++ b/cmd/corral/models_pool_test.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "testing" @@ -25,6 +26,9 @@ const samplePool = "anthropic/claude-sonnet-5-5,anthropic/claude-haiku-5-5,codex const sampleSource = "the machine ledger /state/ledger (31 scan(s), 2 review(s); a run is one SCAN)" +// sampleRecordSource is sampleSource as a record keeps it: no host path. +const sampleRecordSource = "the machine ledger (31 scan(s), 2 review(s))" + // poolEnv isolates the resolver from the machine: no registry, no gateway, no // daemon, and a credential check that passes, so no test reads a real key. // It returns the repo root to resolve against (an empty directory, so no @@ -79,7 +83,7 @@ func sampleEvidence() rankEvidence { obs = append(obs, writerObs("claude-sonnet-5-5", "python", r, 32, 32)) } obs = append(obs, writerObs("claude-sonnet-5-5", "python", "r6", 33, 35)) - return rankEvidence{Obs: obs, Source: sampleSource, + return rankEvidence{Obs: obs, Source: sampleSource, RecordSource: sampleRecordSource, Generations: generationRange{Min: "40", Max: "43", Scans: 6}} } @@ -175,7 +179,7 @@ func TestSeatFromPoolWritesChosenSeatsIntoTheFlags(t *testing.T) { if fmt.Sprint(seating.SeatedBy) != fmt.Sprint(wantBy) { t.Errorf("SeatedBy = %v, want %v", seating.SeatedBy, wantBy) } - wantEvidence := sampleSource + "; language python" + wantEvidence := sampleRecordSource + "; language python" if seating.SeatEvidence != wantEvidence { t.Errorf("SeatEvidence = %q, want %q", seating.SeatEvidence, wantEvidence) } @@ -252,7 +256,7 @@ func TestSeatFromPoolRefusesBelowTheFloor(t *testing.T) { func TestSeatFromPoolNoEvidenceYet(t *testing.T) { root := poolEnv(t) src := "the machine ledger /state/ledger (no entries yet)" - load := &fakeLoad{ev: rankEvidence{Source: src}} + load := &fakeLoad{ev: rankEvidence{Source: src, RecordSource: "the machine ledger (no entries yet)"}} confirm := &fakeConfirm{accepted: true, interactive: true} vals := &certifyVals{} _, exit, done, out, _ := runPool(t, root, "python", samplePool, vals, load, confirm) @@ -337,7 +341,7 @@ func TestSeatFromPoolNoTerminal(t *testing.T) { if exit != 0 || !done { t.Fatalf("exit=%d done=%v, want 0 and done: no terminal means nothing runs", exit, done) } - want := "to run this lineup: --derive-model anthropic/claude-sonnet-5-5 --mutant-model anthropic/claude-sonnet-5-5 --writer-model anthropic/claude-sonnet-5-5\n" + want := "to run this lineup: --derive-model anthropic/claude-sonnet-5-5 --mutant-model anthropic/claude-sonnet-5-5 --writer-model anthropic/claude-sonnet-5-5\n" + noTerminalLine + "\n" if !strings.HasSuffix(out, want) { t.Errorf("stdout must end with the explicit flags for the proposed (not pinned) seats:\n%s\nwant suffix %q", out, want) } @@ -376,7 +380,7 @@ func TestSeatFromPoolAlias(t *testing.T) { vals := &certifyVals{derive: "anthropic/claude-haiku-5-5", critic: "off"} confirm := &fakeConfirm{accepted: true, interactive: true} seating, exit, done, out, errs := runPool(t, root, "python", "edge,anthropic/claude-haiku-5-5", vals, - &fakeLoad{ev: rankEvidence{Obs: obs, Source: sampleSource}}, confirm) + &fakeLoad{ev: rankEvidence{Obs: obs, Source: sampleSource, RecordSource: sampleRecordSource}}, confirm) if done || exit != 0 { t.Fatalf("done=%v exit=%d stderr=%q out=%s", done, exit, errs, out) } @@ -423,7 +427,7 @@ func TestSeatFromPoolLanguageLine(t *testing.T) { ev rankEvidence want string }{ - {shadowpool.LangAny, sampleEvidence(), "; all languages (this scan has more than one)\n"}, + {shadowpool.LangAny, sampleEvidence(), "; all languages (the walked sources are in more than one)\n"}, {"python", stripped, "; all languages (the evidence records none)\n"}, } for _, c := range cases { @@ -472,7 +476,8 @@ func TestSeatFromPoolSaysWhyThereIsNoSingleLanguage(t *testing.T) { why anyLangReason want string }{ - {anyLangSeveral, "all languages (this scan has more than one)"}, + {anyLangSeveral, "all languages (the walked sources are in more than one)"}, + {anyLangNone, "all languages (no language was detected in this scan)"}, {anyLangReview, "all languages (a review is not ranked by language)"}, {anyLangUnknown, "all languages (no single language is known for this check)"}, } { @@ -522,3 +527,230 @@ func TestTerminalConfirmNotATerminal(t *testing.T) { t.Errorf("a pipe is not a terminal: accepted=%v interactive=%v err=%v out=%q", accepted, interactive, err, out.String()) } } + +// Pins are resolved as the members are, so the hard rule and the critic's +// provider preference compare the models that would run, not the spellings +// the operator typed (B3). +func TestSeatFromPoolResolvesPins(t *testing.T) { + t.Run("an agent-pinned critic that is the evidence-seated writer is refused", func(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{critic: "claude-code:claude-sonnet-5-5"} + confirm := &fakeConfirm{accepted: true, interactive: true} + _, exit, done, out, errs := runPool(t, root, "python", samplePool, vals, &fakeLoad{ev: sampleEvidence()}, confirm) + if exit != 2 || !done || confirm.calls != 0 { + t.Fatalf("exit=%d done=%v asked=%d, want the lineup refused before any question\n%s%s", exit, done, confirm.calls, out, errs) + } + if !strings.Contains(out, "refused: test-critic and test-writer are both claude-sonnet-5-5") || !strings.Contains(out, "--critic-model off") { + t.Errorf("the plan must name the broken pair and the flags that fix it:\n%s", out) + } + }) + t.Run("a writer pinned by alias is not proposed again as the critic", func(t *testing.T) { + root := poolEnv(t) + t.Setenv(models.EnvInline, `{"edge": {"provider": "ollama", "model": "localmodel-a:1", "endpoint": "http://127.0.0.1:11434"},`+ + `"edge2": {"provider": "ollama", "model": "localmodel-a:1", "endpoint": "http://127.0.0.1:11434"}}`) + vals := &certifyVals{derive: "edge", mutant: "edge", writer: "edge"} + confirm := &fakeConfirm{accepted: true, interactive: true} + seating, exit, done, out, errs := runPool(t, root, "python", "edge2,anthropic/claude-haiku-5-5", vals, &fakeLoad{ev: sampleEvidence()}, confirm) + if done || exit != 0 { + t.Fatalf("done=%v exit=%d\n%s%s", done, exit, out, errs) + } + if vals.critic != "anthropic/claude-haiku-5-5" || seating.SeatedBy[modelrank.SeatTestCritic] != "order" { + t.Errorf("critic = %q (%v), want the member that is not the writer's model\n%s", vals.critic, seating.SeatedBy, out) + } + }) + t.Run("a pinned writer outside the pool still steers the critic to another provider", func(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{derive: "anthropic/claude-sonnet-5-5", mutant: "anthropic/claude-sonnet-5-5", writer: "anthropic/claude-sonnet-5-5"} + confirm := &fakeConfirm{accepted: true, interactive: true} + _, exit, done, out, errs := runPool(t, root, "python", "anthropic/claude-haiku-5-5,codex:gpt-6", vals, &fakeLoad{ev: sampleEvidence()}, confirm) + if done || exit != 0 { + t.Fatalf("done=%v exit=%d\n%s%s", done, exit, out, errs) + } + if vals.critic != "codex:gpt-6" { + t.Errorf("critic = %q, want codex:gpt-6 (a provider other than the pinned writer's)\n%s", vals.critic, out) + } + }) + t.Run("a pin no source serves is refused before any plan", func(t *testing.T) { + root := poolEnv(t) + vals := &certifyVals{writer: "no-source-serves-this"} + load, confirm := &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{accepted: true, interactive: true} + _, exit, done, out, errs := runPool(t, root, "python", samplePool, vals, load, confirm) + if exit != 2 || !done || out != "" || confirm.calls != 0 || len(load.asked) != 0 { + t.Fatalf("exit=%d done=%v out=%q asked=%d load=%v, want a refusal before evidence or plan", exit, done, out, confirm.calls, load.asked) + } + if !strings.Contains(errs, "--writer-model") || !strings.Contains(errs, "no-source-serves-this") { + t.Errorf("stderr must carry the resolver's own refusal: %q", errs) + } + }) +} + +// The recorded evidence line names the source without a host path, while the +// printed plan keeps the directory the operator can open (B2). The record is +// signed and may be published; a home directory and a user name are not +// evidence. +func TestSeatEvidenceRecordsNoHostPath(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + dir := machineLedgerWith(t, [2]string{`{"test-writer":"claude-sonnet-5-5"}`, ""}) + ledger := filepath.Join(home, "state", "ledger") + if err := os.MkdirAll(filepath.Dir(ledger), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(dir, ledger); err != nil { + t.Fatal(err) + } + t.Setenv("CORRALAI_MACHINE_LEDGER", ledger) + + ev, err := machineLedgerRankEvidence() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ev.Source, ledger) { + t.Errorf("the printed source must name the directory: %q", ev.Source) + } + if ev.RecordSource != "the machine ledger (1 scan(s), 0 review(s))" { + t.Errorf("RecordSource = %q", ev.RecordSource) + } + + root := poolEnv(t) + vals := &certifyVals{derive: "anthropic/claude-haiku-5-5", mutant: "anthropic/claude-haiku-5-5", critic: "off"} + var out, errb bytes.Buffer + seating, exit, done := seatFromPool(context.Background(), "corral certify --repo", root, shadowpool.LangAny, anyLangSeveral, + poolFlags("anthropic/claude-sonnet-5-5"), vals.roles(), + poolDeps{load: defaultRankLoader, confirm: (&fakeConfirm{accepted: true, interactive: true}).fn, stdout: &out, stderr: &errb}) + if done || exit != 0 || seating.SeatedBy[modelrank.SeatTestWriter] != "evidence" { + t.Fatalf("done=%v exit=%d seating=%+v\n%s%s", done, exit, seating, out.String(), errb.String()) + } + if strings.Contains(seating.SeatEvidence, string(filepath.Separator)) || strings.Contains(seating.SeatEvidence, home) { + t.Errorf("SeatEvidence carries a host path: %q", seating.SeatEvidence) + } + if want := "the machine ledger (1 scan(s), 0 review(s)); all languages (the walked sources are in more than one)"; seating.SeatEvidence != want { + t.Errorf("SeatEvidence = %q, want %q", seating.SeatEvidence, want) + } + if !strings.Contains(out.String(), "evidence: the machine ledger "+ledger) { + t.Errorf("the printed plan must still name the directory:\n%s", out.String()) + } +} + +// A loader that gives no path-free name for the record is refused before +// anything is proposed: a recorded line built from the printed one would +// carry the path, and an empty one would record nothing about the evidence. +func TestSeatFromPoolRefusesEvidenceWithNoRecordName(t *testing.T) { + root := poolEnv(t) + ev := sampleEvidence() + ev.RecordSource = "" + confirm := &fakeConfirm{accepted: true, interactive: true} + _, exit, done, out, errs := runPool(t, root, "python", samplePool, &certifyVals{}, &fakeLoad{ev: ev}, confirm) + if exit != 2 || !done || out != "" || confirm.calls != 0 { + t.Fatalf("exit=%d done=%v out=%q asked=%d", exit, done, out, confirm.calls) + } + if !strings.Contains(errs, "record") { + t.Errorf("stderr = %q", errs) + } +} + +// With no observations at all, one plain line under the plan says why every +// measured seat is refused (C5). +func TestSeatFromPoolSaysWhenThereIsNoEvidenceAtAll(t *testing.T) { + root := poolEnv(t) + _, _, _, out, _ := runPool(t, root, "python", samplePool, &certifyVals{}, + &fakeLoad{ev: rankEvidence{Source: "the machine ledger /state/ledger (no entries yet)", RecordSource: "the machine ledger (no entries yet)"}}, + &fakeConfirm{accepted: true, interactive: true}) + const line = "no recorded evidence yet: every measured seat must be pinned until this machine has runs on its ledger\n" + if !strings.Contains(out, line) { + t.Errorf("plan lacks the no-evidence line:\n%s", out) + } + _, _, _, out, _ = runPool(t, root, "python", samplePool, &certifyVals{}, &fakeLoad{ev: sampleEvidence()}, + &fakeConfirm{accepted: true, interactive: true}) + if strings.Contains(out, "no recorded evidence yet") { + t.Errorf("the line is printed with evidence present:\n%s", out) + } +} + +// With no terminal the run audits nothing and exits 0; its last line says so +// in words a CI log reader cannot take for a pass (B7). +func TestSeatFromPoolNoTerminalSaysNothingWasAudited(t *testing.T) { + root := poolEnv(t) + _, exit, done, out, _ := runPool(t, root, "python", samplePool, &certifyVals{critic: "off"}, &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{interactive: false}) + if exit != 0 || !done { + t.Fatalf("exit=%d done=%v", exit, done) + } + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + if last := lines[len(lines)-1]; last != noTerminalLine { + t.Errorf("last line = %q, want %q", last, noTerminalLine) + } +} + +// A scan in which no language was detected says that, not "more than one" +// (B5, A5). +func TestSeatFromPoolNoLanguageDetected(t *testing.T) { + root := poolEnv(t) + var out, errb bytes.Buffer + _, exit, done := seatFromPool(context.Background(), "corral certify --repo", root, shadowpool.LangAny, anyLangNone, poolFlags(samplePool), (&certifyVals{}).roles(), + poolDeps{load: (&fakeLoad{ev: sampleEvidence()}).fn, confirm: (&fakeConfirm{accepted: true, interactive: true}).fn, stdout: &out, stderr: &errb}) + if exit != 0 || done || !strings.Contains(out.String(), "; all languages (no language was detected in this scan)\n") { + t.Errorf("exit=%d done=%v\n%s%s", exit, done, out.String(), errb.String()) + } +} + +// A model a placement alone serves cannot be a member, and the refusal says +// how to seat it instead (B11). +func TestSeatFromPoolNamesLocalEndpointForAnUnservedMember(t *testing.T) { + root := poolEnv(t) + _, exit, _, _, errs := runPool(t, root, "python", "anthropic/claude-sonnet-5-5,gpu-only-model:1", &certifyVals{}, &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{}) + if exit != 2 || !strings.Contains(errs, "--local-endpoint") || !strings.Contains(errs, "gpu-only-model:1") { + t.Errorf("exit=%d stderr=%q", exit, errs) + } +} + +// The prompt is written to the plan's own stream, so both ends must be a +// terminal: with stdout redirected, a "y" would accept a lineup nobody saw +// (B9). +func TestTerminalConfirmNeedsBothEndsToBeTerminals(t *testing.T) { + in, inW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer in.Close() + defer inW.Close() + outR, outW, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer outR.Close() + defer outW.Close() + orig := isTerminal + t.Cleanup(func() { isTerminal = orig }) + + isTerminal = func(fd int) bool { return fd == int(in.Fd()) } // #nosec G115 -- test fd + var buf bytes.Buffer + if acc, inter, err := terminalConfirm(in, &buf)("plan"); acc || inter || err != nil || buf.Len() != 0 { + t.Errorf("stdout a buffer: accepted=%v interactive=%v err=%v out=%q", acc, inter, err, buf.String()) + } + if acc, inter, err := terminalConfirm(in, outW)("plan"); acc || inter || err != nil { + t.Errorf("stdout a pipe: accepted=%v interactive=%v err=%v", acc, inter, err) + } + + isTerminal = func(int) bool { return true } + if _, err := inW.WriteString("y\n"); err != nil { + t.Fatal(err) + } + if acc, inter, err := terminalConfirm(in, outW)("plan"); !acc || !inter || err != nil { + t.Errorf("both terminals: accepted=%v interactive=%v err=%v", acc, inter, err) + } +} + +// An alias and the concrete name it declares are one model: naming both +// would make the ranking's row ambiguous between two members (C4). +func TestSeatFromPoolRefusesAnAliasBesideItsModel(t *testing.T) { + root := poolEnv(t) + t.Setenv(models.EnvInline, `{"sonnet": {"provider": "anthropic", "model": "claude-sonnet-5-5"}}`) + load, confirm := &fakeLoad{ev: sampleEvidence()}, &fakeConfirm{accepted: true, interactive: true} + _, exit, done, out, errs := runPool(t, root, "python", "sonnet,anthropic/claude-sonnet-5-5", &certifyVals{}, load, confirm) + if exit != 2 || !done || out != "" || confirm.calls != 0 || len(load.asked) != 0 { + t.Fatalf("exit=%d done=%v out=%q asked=%d load=%v", exit, done, out, confirm.calls, load.asked) + } + if !strings.Contains(errs, `members "sonnet" and "anthropic/claude-sonnet-5-5" are both the model "claude-sonnet-5-5"`) { + t.Errorf("stderr = %q", errs) + } +} diff --git a/cmd/corral/models_rank.go b/cmd/corral/models_rank.go index 9729dd27..045fb559 100644 --- a/cmd/corral/models_rank.go +++ b/cmd/corral/models_rank.go @@ -33,6 +33,12 @@ import ( type rankEvidence struct { Obs []modelrank.Observation Source string + // RecordSource names the same source with no host path, for a record: + // --models writes it into signed entries that may be published, where + // Source's directory would put a home directory and a user name. Set by + // the machine-ledger loader, the one source --models reads; "" from the + // others, which only `corral models rank` prints. + RecordSource string // Generations is which grading rules (VerdictGeneration) the scans behind // Obs were graded under. Zero when the store records none (the local // bugcatch ledger) or has no scans. @@ -774,11 +780,17 @@ func machineLedgerRankEvidence() (rankEvidence, error) { } if _, serr := os.Stat(dir); serr != nil { if os.IsNotExist(serr) { - return rankEvidence{Source: "the machine ledger " + dir + " (not created yet: no run on this machine has written an entry)"}, nil + return rankEvidence{ + Source: "the machine ledger " + dir + " (not created yet: no run on this machine has written an entry)", + RecordSource: "the machine ledger (not created yet)", + }, nil } return rankEvidence{}, fmt.Errorf("cannot read the machine ledger %s: %w", dir, serr) } - return rankFromView(dir, func() (*sql.DB, error) { + // The path-free name is built from the same counts as the printed one, + // never by cutting the directory out of the printed text. + var record string + ev, err := rankFromView(dir, func() (*sql.DB, error) { db, lerr := auditpush.LoadDirForRank(dir) if lerr != nil { return nil, fmt.Errorf("the machine ledger %s: %w\n run `corral verify --ledger %s` to see which entry fails; `--source scorecard` or `--db ` reads a different source deliberately — nothing falls back to one automatically", dir, lerr, dir) @@ -790,10 +802,17 @@ func machineLedgerRankEvidence() (rankEvidence, error) { return "", fmt.Errorf("counting entries in the machine ledger %s: %w", dir, err) } if scans == 0 && reviews == 0 { + record = "the machine ledger (no entries yet)" return "the machine ledger " + dir + " (no entries yet)", nil } + record = fmt.Sprintf("the machine ledger (%d scan(s), %d review(s))", scans, reviews) return fmt.Sprintf("the machine ledger %s (%d scan(s), %d review(s); a run is one SCAN)", dir, scans, reviews), nil }) + if err != nil { + return rankEvidence{}, err + } + ev.RecordSource = record + return ev, nil } // reviewRankEvidence grades the reviewer and verifier seats from the diff --git a/cmd/corral/review.go b/cmd/corral/review.go index 4ed57f4e..5b84ec85 100644 --- a/cmd/corral/review.go +++ b/cmd/corral/review.go @@ -126,7 +126,7 @@ func reviewFlagSet(out io.Writer) (*flag.FlagSet, *reviewFlags) { fs.StringVar(&f.scope, "scope", "", "the directory or file under --repo to review (required)") fs.StringVar(&f.model, "reviewer-model", "", "the reviewer seat — an alias from the registry, a hosted model as vendor/model (`anthropic/…`, `google/…`, `openai/…`; a bare cloud name is refused), a local model the OLLAMA_URL daemon lists, or an AGENTIC seat: an agent NAME, pinned to a model as `:`. `claude-code`, `codex` and `antigravity` are defined; ANY agent is defined the same way as CORRALAI_AGENT_=\"\" ({dir} the worktree, {out} a reply file, {model} / {model:FLAG} the pin; the brief on stdin, the reply on stdout). An agentic seat is a coding agent started in a disposable copy of the repository: it reads the whole scope itself (no --max-bytes cap) and hands back scripts, which corral runs — nothing it did itself is on the record. Required; corral has no default models") fs.StringVar(&f.verifier, "verifier-model", "", "a VERIFIER seat, adversarial to the reviewer: a different model that tries to refute every finding, by the same rules — a REPRODUCED refutation (a sh script that exits 0 iff the refutation is demonstrated) that holds demotes the finding on the record; a CODE-READ refutation is carried as opinion; a search that finds nothing is never a refutation. Must not be the reviewer's model. Off unless named: `off` names no verifier, the same word --critic-model off uses on certify. With --models a verifier is proposed unless you pin it with this flag or pin `off`") - f.models = registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool") + f.models = registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, reviews nothing and exits 0, so a pipeline must pass the seat flags itself. A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool") fs.StringVar(&f.ledger, "ledger", "", "the ledger directory the review entry is written to (default: /.corral/ledger, or $CORRAL_LEDGER). A copy of the entry also goes to this machine's own ledger (the state directory's ledger, see corral state paths; override CORRALAI_MACHINE_LEDGER); a machine ledger that resolves inside the checkout is skipped, with the reason printed") fs.StringVar(&f.attest, "attest", "", "write an in-toto statement (predicate https://corralai.dev/review/v1) to this path, and its DSSE envelope beside it when a certify key is configured: the REPRODUCTIONS — every finding's declared and recorded tier, the hash of its script and output, its exit, the verifier's refutation on the same terms — signed; the opinion bound by its hash and not carried. The ledger entry then names the statement. corral verify --attest --db recomputes the reproductions' hash from the entry") fs.StringVar(&f.push, "push", "", "also append the review's rows to a warehouse you own — a DuckDB path, or md: — as corral_reviews and corral_findings, keyed by the entry's hash so a later review adjudicate --push joins to them. Scripts and outputs travel as hashes unless --push-source") @@ -543,8 +543,9 @@ func recordSeatProvider(r *review.Review, seat string, rs agentbackend.ResolvedS // and under the same rule: only a seat that is on the entry is recorded (a // verifier that failed is not), and a run without --models records nothing, // so its entry keeps the shape it always had. The ranking's evidence line is -// recorded whenever any seat was recorded, since it explains the -// evidence-seated ones. +// recorded only while a seat on the entry was seated by evidence: it +// explains those seats, and a pinned reviewer beside a verifier that never +// made the entry has nothing for it to explain. func recordSeatedBy(r *review.Review, seat string, seating poolSeating) { by, ok := seating.SeatedBy[seat] if !ok { @@ -554,7 +555,7 @@ func recordSeatedBy(r *review.Review, seat string, seating poolSeating) { r.SeatedBy = map[string]string{} } r.SeatedBy[seat] = by - r.SeatEvidence = seating.SeatEvidence + r.SeatEvidence = seatEvidenceFor(r.SeatedBy, seating.SeatEvidence) } // reviewSeatTools are the tools an agent seat in review is granted: it reads diff --git a/cmd/corral/review_models_test.go b/cmd/corral/review_models_test.go index 65745690..3768cb54 100644 --- a/cmd/corral/review_models_test.go +++ b/cmd/corral/review_models_test.go @@ -4,6 +4,7 @@ package main import ( "bytes" + "errors" "io" "path/filepath" "strings" @@ -33,7 +34,7 @@ func reviewEvidence() rankEvidence { add("verifier-y", modelrank.SeatReviewer, 2, 1, 0, 0) add("verifier-y", modelrank.SeatVerifier, 0, 0, 2, 2) add("reviewer-x", modelrank.SeatVerifier, 0, 0, 2, 1) - return rankEvidence{Obs: obs, Source: "test evidence (6 review(s))", Generations: generationRange{Min: "48", Max: "48", Scans: 6}} + return rankEvidence{Obs: obs, Source: "test evidence (6 review(s))", RecordSource: "test evidence (6 review(s))", Generations: generationRange{Min: "48", Max: "48", Scans: 6}} } // stubReviewPool replaces the doors' seams with reviewEvidence and an @@ -60,12 +61,42 @@ func stubReviewPool(t *testing.T) *int { // seats and returns the single entry it wrote, so the tests read what was // recorded and not an intermediate. func reviewModelsRun(t *testing.T, args ...string) (rev review.Review, code int, out, errs string, seen map[string]int) { + t.Helper() + r := reviewModelsRunWith(t, reviewRunOpts{verifierReply: `{"opinion":"v","refutations":[]}`}, args...) + if r.code == 0 { + if len(r.entries) != 1 || r.entries[0].Review == nil { + t.Fatalf("want one review entry, got %d\n%s%s", len(r.entries), r.out, r.errs) + } + rev = *r.entries[0].Review + } + return rev, r.code, r.out, r.errs, r.seen +} + +// reviewRunOpts varies the canned review: what the verifier replies. +type reviewRunOpts struct{ verifierReply string } + +// reviewRun is everything one canned review left behind: the exit, both +// streams, the seats built (by model), the entries in the repo ledger and the +// machine ledger directory, which a test reads for what was or was not +// written. +type reviewRun struct { + code int + out, errs string + seen map[string]int + entries []auditpush.LedgerEntry + machineLedger string + stateDir string +} + +func reviewModelsRunWith(t *testing.T, opts reviewRunOpts, args ...string) reviewRun { t.Helper() requireReviewSandbox(t) declareLocalSeats(t, "reviewer-x", "verifier-y") t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") ledger := filepath.Join(t.TempDir(), "machine") t.Setenv("CORRALAI_MACHINE_LEDGER", ledger) + state := t.TempDir() + t.Setenv("CORRALAI_STATE_DIR", state) root := t.TempDir() gitRun := gitCmd(t, root) mustWrite(t, filepath.Join(root, "pkg", "a.go"), "package pkg\n\nfunc Add(a, b int) int { return a - b }\n") @@ -74,26 +105,20 @@ func reviewModelsRun(t *testing.T, args ...string) (rev review.Review, code int, gitRun("add", ".") gitRun("commit", "-q", "-m", "base", "--no-gpg-sign") - seen = map[string]int{} + seen := map[string]int{} orig := newReviewerBackend t.Cleanup(func() { newReviewerBackend = orig }) newReviewerBackend = func(rs agentbackend.ResolvedSeat, _ agentbackend.SeatOptions) (agentbackend.Backend, error) { seen[rs.Spec]++ if rs.Spec == "verifier-y" { - return cannedReviewer{reply: `{"opinion":"v","refutations":[]}`}, nil + return cannedReviewer{reply: opts.verifierReply}, nil } return cannedReviewer{reply: `{"opinion":"o","findings":[{"claim":"a caller might overflow","tier":"HYPOTHESIS"}],"sound":[]}`}, nil } var o, e bytes.Buffer - code = runReview(append([]string{"--repo", root, "--scope", "pkg"}, args...), &o, &e) + code := runReview(append([]string{"--repo", root, "--scope", "pkg"}, args...), &o, &e) entries, _ := auditpush.ReadLedgerDir(filepath.Join(root, ".corral", "ledger")) - if code == 0 { - if len(entries) != 1 || entries[0].Review == nil { - t.Fatalf("want one review entry, got %d\n%s%s", len(entries), o.String(), e.String()) - } - rev = *entries[0].Review - } - return rev, code, o.String(), e.String(), seen + return reviewRun{code: code, out: o.String(), errs: e.String(), seen: seen, entries: entries, machineLedger: ledger, stateDir: state} } func TestReviewSeatedByModels(t *testing.T) { @@ -200,7 +225,67 @@ func TestReviewModelsRefusesUsageBeforeAskingAnything(t *testing.T) { o.Reset() e.Reset() root := t.TempDir() - if code := runReview([]string{"--repo", root, "--scope", "x", "--no-ledger"}, &o, &e); code == 0 { - t.Errorf("no reviewer was accepted") + gitRun := gitCmd(t, root) + mustWrite(t, filepath.Join(root, "x", "a.go"), "package x\n") + gitRun("init", "-q") + gitRun("add", ".") + gitRun("commit", "-q", "-m", "base", "--no-gpg-sign") + code := runReview([]string{"--repo", root, "--scope", "x", "--no-ledger"}, &o, &e) + if code != 2 || *asked != 0 || o.Len() != 0 { + t.Errorf("no reviewer: exit %d asked %d stdout %q, want 2, nothing asked, nothing printed", code, *asked, o.String()) + } + if !strings.Contains(e.String(), "a reviewer: --reviewer-model, or --models to have one proposed") { + t.Errorf("stderr = %q", e.String()) + } +} + +// Every seat pinned (the reviewer named, the verifier pinned off): nothing is +// proposed or asked, the pool is still checked, and the entry records the +// reviewer as the operator's and nothing else new (C8). +func TestReviewModelsAllPinned(t *testing.T) { + asked := stubReviewPool(t) + rev, code, out, errs, seen := reviewModelsRun(t, "--models", reviewPool, "--reviewer-model", "anthropic/reviewer-x", "--verifier-model", "off") + if code != 0 { + t.Fatalf("exit %d\n%s%s", code, out, errs) + } + if *asked != 0 || strings.Contains(out, "seats (from --models") { + t.Errorf("asked %d time(s) or printed a plan with every seat pinned:\n%s", *asked, out) + } + if len(rev.SeatedBy) != 1 || rev.SeatedBy[modelrank.SeatReviewer] != "operator" || rev.SeatEvidence != "" { + t.Errorf("seated_by %v, seat_evidence %q; want the reviewer as the operator's only", rev.SeatedBy, rev.SeatEvidence) + } + if rev.VerifierModel != "" || seen["verifier-y"] != 0 { + t.Errorf("a verifier ran: %q %v", rev.VerifierModel, seen) + } + // The pool is still checked: a member that cannot run refuses the run. + shadowMemberRunnable = func(rs agentbackend.ResolvedSeat) error { + if rs.Spec == "verifier-y" { + return errors.New("no key for verifier-y") + } + return nil + } + _, code, _, errs, _ = reviewModelsRun(t, "--models", reviewPool, "--reviewer-model", "anthropic/reviewer-x", "--verifier-model", "off") + if code != 2 || !strings.Contains(errs, "no key for verifier-y") { + t.Errorf("a pool with a member that cannot run: exit %d stderr %q", code, errs) + } +} + +// A review records the evidence line only when a seat on the entry was +// seated by evidence (B12): with the reviewer pinned and the evidence-seated +// verifier recorded unverified (its reply does not parse), no recorded seat +// was chosen by the ranking, so the line would explain nothing on the entry. +func TestReviewRecordsNoEvidenceLineWithoutAnEvidenceSeat(t *testing.T) { + stubReviewPool(t) + r := reviewModelsRunWith(t, reviewRunOpts{verifierReply: "not a verdict at all"}, + "--models", reviewPool, "--reviewer-model", "anthropic/reviewer-x") + if r.code != 0 || len(r.entries) != 1 || r.entries[0].Review == nil { + t.Fatalf("exit %d, %d entries\n%s%s", r.code, len(r.entries), r.out, r.errs) + } + rev := *r.entries[0].Review + if rev.VerifierModel != "" || rev.SeatedBy[modelrank.SeatReviewer] != "operator" { + t.Fatalf("fixture did not drop the verifier: verifier %q seated_by %v", rev.VerifierModel, rev.SeatedBy) + } + if rev.SeatEvidence != "" { + t.Errorf("seat_evidence = %q on an entry whose only recorded seat was pinned", rev.SeatEvidence) } } diff --git a/cmd/corral/seated_by_record_test.go b/cmd/corral/seated_by_record_test.go index 14662191..9499ed3f 100644 --- a/cmd/corral/seated_by_record_test.go +++ b/cmd/corral/seated_by_record_test.go @@ -42,7 +42,7 @@ func recordEvidence() rankEvidence { for _, r := range runs { obs = append(obs, writerObs("claude-sonnet-5", "go", r, 5, 5)) } - return rankEvidence{Obs: obs, Source: recordSource, Generations: generationRange{Min: "48", Max: "48", Scans: 6}} + return rankEvidence{Obs: obs, Source: recordSource, RecordSource: recordSource, Generations: generationRange{Min: "48", Max: "48", Scans: 6}} } // stubDoorPool replaces the doors' production seams (the machine ledger and diff --git a/internal/advpool/driver.go b/internal/advpool/driver.go index dabcbc72..7c5c5bb5 100644 --- a/internal/advpool/driver.go +++ b/internal/advpool/driver.go @@ -15,6 +15,7 @@ import ( "github.com/pdbethke/corralai/internal/adequacy" "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/autoseat" "github.com/pdbethke/corralai/internal/clip" golang "github.com/pdbethke/corralai/internal/lang" "github.com/pdbethke/corralai/internal/matrix" @@ -1005,7 +1006,8 @@ type Verdict struct { // field existed reads the same. Set by signedSeats on both paths. SeatedBy map[string]string `json:"seated_by,omitempty"` // SeatEvidence is what the ranking behind the "evidence" seats read - // (RunSpec.SeatEvidence). "" when no seat was seated by evidence. + // (RunSpec.SeatEvidence). "" when no seat SeatedBy keeps was seated by + // evidence. SeatEvidence string `json:"seat_evidence,omitempty"` // MutantSources is where each mutant of the exam came from (graded, or // refused by the compile gate, or unmeasured — so waste can be counted @@ -3957,12 +3959,15 @@ func (r *runState) signedSeats(assign RoleAssignment) signedRoster { sels = append(sels, sel) } } + seatedBy := SeatedOnly(r.rs.SeatedBy, roster) return signedRoster{ roster: roster, selection: sels, tools: SeatedOnly(r.rs.SeatTools, roster), providers: SeatedOnly(r.rs.SeatProviders, roster), - seatedBy: SeatedOnly(r.rs.SeatedBy, roster), - evidence: r.rs.SeatEvidence, + seatedBy: seatedBy, + // The evidence line explains an evidence seat, so it goes with the + // last one the narrowing keeps, as the selection goes with its seat. + evidence: autoseat.EvidenceFor(seatedBy, r.rs.SeatEvidence), } } diff --git a/internal/advpool/run.go b/internal/advpool/run.go index 7046f026..d1672cc8 100644 --- a/internal/advpool/run.go +++ b/internal/advpool/run.go @@ -173,8 +173,10 @@ type RunSpec struct { // Carried to Verdict.SeatedBy by signedSeats on BOTH construction paths. SeatedBy map[string]string // SeatEvidence names what the ranking read to seat the "evidence" - // seats, for example "the machine ledger (31 scans)". "" when nothing - // was read. Carried to Verdict.SeatEvidence the same way. + // seats, with no host path, for example "the machine ledger (31 + // scan(s), 2 review(s)); language go". "" when nothing was read. + // Carried to Verdict.SeatEvidence the same way, and dropped there when + // the roster keeps no evidence seat. SeatEvidence string ShadowSelection []shadowpool.Selection diff --git a/internal/advpool/seated_by_test.go b/internal/advpool/seated_by_test.go index 8fcb08f6..dcc63308 100644 --- a/internal/advpool/seated_by_test.go +++ b/internal/advpool/seated_by_test.go @@ -53,3 +53,23 @@ func TestVerdictCarriesSeatedByOnBothConstructionPaths(t *testing.T) { } } } + +// The evidence line explains an evidence seat, so a verdict whose evidence +// seats the roster narrowed away keeps no line (B12): a critic seated by +// evidence and then pinned off leaves a verdict whose every recorded seat +// was the operator's. +func TestVerdictDropsSeatEvidenceWithItsLastEvidenceSeat(t *testing.T) { + rs := RunSpec{Repo: "r", Commit: "c", Lang: "go", + SeatedBy: map[string]string{RoleTestWriter: "operator", RoleTestCritic: "evidence", RoleMutantGenerator: "operator"}, + SeatEvidence: "the machine ledger (31 scan(s), 2 review(s)); language go", + } + assign := RoleAssignment{RoleTestWriter: "w", RoleTestCritic: "off", RoleMutantGenerator: "g"} + d := &Driver{Assign: assign} + converged := Verdict{} + (&runState{rs: rs}).signedSeats(assign).stamp(&converged) + for name, v := range map[string]Verdict{"converged": converged, "timeout": d.timeoutVerdict(&runState{rs: rs})} { + if v.SeatEvidence != "" { + t.Errorf("%s: SeatEvidence = %q with no evidence seat on the verdict (SeatedBy %v)", name, v.SeatEvidence, v.SeatedBy) + } + } +} diff --git a/internal/autoseat/autoseat.go b/internal/autoseat/autoseat.go index 57f1f48f..bcad19a6 100644 --- a/internal/autoseat/autoseat.go +++ b/internal/autoseat/autoseat.go @@ -34,6 +34,20 @@ const ( ByOrder = "order" ) +// EvidenceFor is the evidence line a record keeps beside seatedBy (role -> +// how it was chosen): the line when any seat seatedBy names was seated by +// evidence, and "" otherwise. A record narrows SeatedBy to the seats it +// holds, and the line explains only an evidence seat, so a record whose +// evidence seats were all narrowed away keeps no line. +func EvidenceFor(seatedBy map[string]string, evidence string) string { + for _, by := range seatedBy { + if by == ByEvidence { + return evidence + } + } + return "" +} + // PinFlag maps a role to the flag that pins it. It is the one table the // "pin --writer-model" text is built from, so a refusal names a flag that // exists. @@ -55,12 +69,23 @@ type Member struct { Rankable bool // false for a bare agent seat with no model } +// Pin is a seat the operator named with its own flag, resolved by the same +// resolver as the pool's members. The hard rule compares the models that +// would run: an alias, a vendor/model spelling and an agent pinned to a model +// can all name one model, and comparing the typed text would let a model +// grade its own work under a second spelling. +type Pin struct { + Given string // exactly what the operator typed; "off" disables the seat + Recorded string // the resolved name (ResolvedSeat.Spec); "" only for "off" + Provider string // the resolved provider; "" when the resolver names none +} + // Input is everything a proposal depends on. type Input struct { - Roles []string // in the order the verb meets them - Pool []Member // in the order the operator named it - Pins map[string]string // role -> the operator's flag value ("off" included) - Report modelrank.Report // already filtered to the pool and the language + Roles []string // in the order the verb meets them + Pool []Member // in the order the operator named it + Pins map[string]Pin // role -> the operator's resolved flag value ("off" included) + Report modelrank.Report // already filtered to the pool and the language Floor int } @@ -68,7 +93,8 @@ type Input struct { type Seat struct { Role string Given string // "" when unfilled - Recorded string + Recorded string // "" when unfilled or pinned "off" + Provider string By string Reason string Unfilled string // the fix, when the role could not be filled @@ -131,59 +157,95 @@ func Propose(in Input) Plan { plan.Refused = true } } - for role, other := range counterpart { + // The hard rule, on the whole lineup with pins included, comparing the + // resolved names. Walked in Roles order, never over the map, so the same + // input always gives the same text, and every broken pair is reported: + // an operator who fixes one should not discover the next on a rerun. + var why []string + for _, role := range in.Roles { + other, ok := counterpart[role] + if !ok { + continue + } a, b := done[role], done[other] - if a.Given == "" || b.Given == "" || strings.EqualFold(a.Given, off) { + // An unfilled seat or one pinned off has no Recorded name, and there + // is nothing for it to be correlated with. + if a.Recorded == "" || b.Recorded == "" { continue } - if agentbackend.SameSeat(a.Given, b.Given) { - plan.Refused = true - plan.Why = fmt.Sprintf("%s and %s are both %q; a model must not grade its own work", - role, other, a.Given) + if agentbackend.SameSeat(a.Recorded, b.Recorded) { + why = append(why, fmt.Sprintf( + "%s and %s are both %s; a model must not grade its own work: pin --%s or --%s to a different model, or pass --%s off", + role, other, agentbackend.SeatModelOf(a.Recorded), PinFlag[role], PinFlag[other], PinFlag[role])) } } + if len(why) > 0 { + plan.Refused = true + plan.Why = strings.Join(why, "; and ") + } return plan } func (in Input) seat(role string, done map[string]Seat) Seat { - if v, ok := in.Pins[role]; ok { - return Seat{Role: role, Given: v, Recorded: in.recordedFor(v), By: ByOperator, - Reason: fmt.Sprintf("pinned by --%s", PinFlag[role])} + if p, ok := in.Pins[role]; ok { + reason := fmt.Sprintf("pinned by --%s", PinFlag[role]) + if strings.EqualFold(strings.TrimSpace(p.Given), off) { + return Seat{Role: role, Given: p.Given, By: ByOperator, Reason: reason} + } + if strings.TrimSpace(p.Recorded) == "" { + // A pin the caller did not resolve cannot be checked against + // the hard rule, and treating its typed text as its model is + // the comparison this field exists to replace. + return Seat{Role: role, Unfilled: fmt.Sprintf( + "--%s %q reached the proposer unresolved, so the lineup cannot be checked; this is a corral defect", + PinFlag[role], p.Given)} + } + return Seat{Role: role, Given: p.Given, Recorded: p.Recorded, Provider: p.Provider, By: ByOperator, Reason: reason} } - avoid := in.avoid(role, done) - if row, ok := in.best(role, avoid); ok { + other := in.counterSeat(role, done) + if row, ok := in.best(role, other.Recorded); ok { mem, _ := in.member(row.Model) - return Seat{Role: role, Given: mem.Given, Recorded: mem.Recorded, By: ByEvidence, Reason: row.Evidence} + return Seat{Role: role, Given: mem.Given, Recorded: mem.Recorded, Provider: mem.Provider, By: ByEvidence, Reason: row.Evidence} } switch { case measured[role]: + if other.Recorded != "" { + if row, ok := in.best(role, ""); ok { + // Every member with enough evidence is the counterpart's + // model, so "no model has the evidence" would be false. + return Seat{Role: role, Unfilled: fmt.Sprintf( + "the only model in --models with %d+ %s as %s is %s, already the %s; name another model in --models, or pin --%s", + in.Floor, in.unit(role), role, row.Model, counterpart[role], PinFlag[role])} + } + } return Seat{Role: role, Unfilled: fmt.Sprintf( "no model in --models has %d+ %s as %s; pin --%s", in.Floor, in.unit(role), role, PinFlag[role])} case role == modelrank.SeatGoalDeriver, role == modelrank.SeatTestCritic: - return in.byOrder(role, avoid) + return in.byOrder(role, other) } return Seat{Role: role, Unfilled: fmt.Sprintf("no rule seats %s; pin --%s", role, PinFlag[role])} } // byOrder seats the first pool member that is not the counterpart, preferring // a different provider: two models from one vendor satisfy the letter of -// decorrelation and not its point. -func (in Input) byOrder(role, avoid string) Seat { +// decorrelation and not its point. The counterpart's provider is known for a +// pin as well as a member, because pins are resolved before they get here. +func (in Input) byOrder(role string, other Seat) Seat { const reason = "no measurement for this seat: taken from the order you named" var same *Member for i, mem := range in.Pool { - if avoid != "" && agentbackend.SameSeat(mem.Recorded, avoid) { + if other.Recorded != "" && agentbackend.SameSeat(mem.Recorded, other.Recorded) { continue } - if avoid != "" && same == nil && in.providerOf(avoid) != "" && mem.Provider == in.providerOf(avoid) { + if other.Provider != "" && same == nil && strings.EqualFold(mem.Provider, other.Provider) { same = &in.Pool[i] continue } - return Seat{Role: role, Given: mem.Given, Recorded: mem.Recorded, By: ByOrder, Reason: reason} + return Seat{Role: role, Given: mem.Given, Recorded: mem.Recorded, Provider: mem.Provider, By: ByOrder, Reason: reason} } if same != nil { - return Seat{Role: role, Given: same.Given, Recorded: same.Recorded, By: ByOrder, + return Seat{Role: role, Given: same.Given, Recorded: same.Recorded, Provider: same.Provider, By: ByOrder, Reason: reason + " (same provider as " + counterpart[role] + "; no other member was available)"} } fix := fmt.Sprintf("name a second model in --models, or pin --%s", PinFlag[role]) @@ -197,6 +259,11 @@ func (in Input) byOrder(role, avoid string) Seat { // best walks the group's rows in the order modelrank returned them and takes // the first sufficient, rankable pool member that is not the avoided model. // Members whose metric ties the best are resolved by pool position. +// +// A row must clear this proposal's own floor as well as be marked +// sufficient: Sufficient is modelrank's verdict at whatever floor that +// ranking ran with, and the proposer is the one place that enforces the +// floor it was given rather than trusting a caller to have ranked at it. func (in Input) best(role, avoid string) (modelrank.Row, bool) { var cands []modelrank.Row for _, g := range in.Report.Groups { @@ -205,7 +272,7 @@ func (in Input) best(role, avoid string) (modelrank.Row, bool) { } for _, r := range g.Rows { mem, ok := in.member(r.Model) - if !ok || !mem.Rankable || !r.Sufficient || r.Metric == nil { + if !ok || !mem.Rankable || !r.Sufficient || r.N < in.Floor || r.Metric == nil { continue } if avoid != "" && agentbackend.SameSeat(mem.Recorded, avoid) { @@ -227,21 +294,16 @@ func (in Input) best(role, avoid string) (modelrank.Row, bool) { return win, true } -// avoid is the recorded name the role must differ from, taken from the -// counterpart's seat (a pin included), or "". -func (in Input) avoid(role string, done map[string]Seat) string { +// counterSeat is the seat the role must differ from (a pin included), or +// the zero Seat when the role has no counterpart, or the counterpart is +// unfilled or pinned off. Its Recorded is what the role avoids and its +// Provider what the order rule prefers to avoid. +func (in Input) counterSeat(role string, done map[string]Seat) Seat { other, ok := counterpart[role] if !ok { - return "" - } - s := done[other] - if s.Given == "" || strings.EqualFold(s.Given, off) { - return "" + return Seat{} } - if s.Recorded != "" { - return s.Recorded - } - return s.Given + return done[other] } func (in Input) member(recorded string) (Member, bool) { @@ -262,24 +324,6 @@ func (in Input) position(recorded string) int { return len(in.Pool) } -// recordedFor resolves a pin to the name the record knows it by when it is a -// pool member; a pin outside the pool is recorded as typed. -func (in Input) recordedFor(pin string) string { - for _, m := range in.Pool { - if agentbackend.SameSeat(m.Given, pin) || agentbackend.SameSeat(m.Recorded, pin) { - return m.Recorded - } - } - return pin -} - -func (in Input) providerOf(recorded string) string { - if m, ok := in.member(recorded); ok { - return m.Provider - } - return "" -} - // unit names the seat's own evidence unit for the refusal text, read from a // row of the report so it is the unit modelrank counted in. func (in Input) unit(role string) string { diff --git a/internal/autoseat/autoseat_test.go b/internal/autoseat/autoseat_test.go index f0663b0d..704c6019 100644 --- a/internal/autoseat/autoseat_test.go +++ b/internal/autoseat/autoseat_test.go @@ -35,6 +35,24 @@ func m(name, provider string) Member { return Member{Given: name, Recorded: name, Provider: provider, Rankable: true} } +// testProviders is the provider each test model resolves to, so a pin built +// by pins carries what the door's resolver would have given it. +var testProviders = map[string]string{"a": "anthropic", "b": "anthropic", "c": "openai"} + +// pins builds resolved pins from role, name pairs, as seatFromPool hands them +// over: Recorded is the name (these tests have no aliases) and "off" has none. +func pins(kv ...string) map[string]Pin { + out := map[string]Pin{} + for i := 0; i+1 < len(kv); i += 2 { + if strings.EqualFold(kv[i+1], "off") { + out[kv[i]] = Pin{Given: kv[i+1]} + continue + } + out[kv[i]] = Pin{Given: kv[i+1], Recorded: kv[i+1], Provider: testProviders[strings.ToLower(kv[i+1])]} + } + return out +} + var certifyRoles = []string{modelrank.SeatGoalDeriver, modelrank.SeatMutantGenerator, modelrank.SeatTestWriter, modelrank.SeatTestCritic} func seatOf(t *testing.T, p Plan, role string) Seat { @@ -60,7 +78,7 @@ func TestPropose(t *testing.T) { reason string refused bool }{ - {"pin wins over a better row", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "x"}, + {"pin wins over a better row", Input{Roles: certifyRoles, Pool: three, Pins: pins(W, "x"), Report: report(row(W, "a", 0.99, true)), Floor: 5}, W, "x", ByOperator, "", false}, {"measured best", Input{Roles: certifyRoles, Pool: three, Report: report(row(W, "b", 0.99, true), row(W, "a", 0.80, true)), Floor: 5}, W, "b", ByEvidence, "evidence for b", false}, @@ -76,22 +94,22 @@ func TestPropose(t *testing.T) { {"unrankable member can take the deriver", Input{Roles: certifyRoles, Pool: []Member{{Given: "codex", Recorded: "codex", Provider: "openai"}, m("b", "anthropic")}, Floor: 5}, D, "codex", ByOrder, "no measurement for this seat: taken from the order you named", false}, - {"critic prefers a different provider", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "a"}, Floor: 5}, + {"critic prefers a different provider", Input{Roles: certifyRoles, Pool: three, Pins: pins(W, "a"), Floor: 5}, C, "c", ByOrder, "taken from the order", false}, {"critic with a sufficient row is by evidence", Input{Roles: certifyRoles, Pool: three, Report: report(row(C, "b", 0.7, true)), Floor: 5}, C, "b", ByEvidence, "evidence for b", false}, - {"critic evidence equal to writer is skipped", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "a"}, + {"critic evidence equal to writer is skipped", Input{Roles: certifyRoles, Pool: three, Pins: pins(W, "a"), Report: report(row(C, "a", 0.9, true), row(C, "b", 0.7, true)), Floor: 5}, C, "b", ByEvidence, "evidence for b", false}, - {"critic evidence only the writer falls back to order", Input{Roles: certifyRoles, Pool: three, Pins: map[string]string{W: "a"}, + {"critic evidence only the writer falls back to order", Input{Roles: certifyRoles, Pool: three, Pins: pins(W, "a"), Report: report(row(C, "a", 0.9, true)), Floor: 5}, C, "c", ByOrder, "", false}, - {"one model cannot seat a critic", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Pins: map[string]string{W: "a"}, Floor: 5}, + {"one model cannot seat a critic", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Pins: pins(W, "a"), Floor: 5}, C, "", "", "", true}, - {"critic off is a pin", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Pins: map[string]string{W: "a", C: "off"}, Floor: 5}, + {"critic off is a pin", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Pins: pins(W, "a", C, "off"), Floor: 5}, C, "off", ByOperator, "", false}, {"pinned writer b leaves critic a", Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic"), m("b", "anthropic")}, - Pins: map[string]string{W: "b"}, Floor: 5}, C, "a", ByOrder, "", false}, + Pins: pins(W, "b"), Floor: 5}, C, "a", ByOrder, "", false}, {"pinned critic equal to writer breaks the hard rule", Input{Roles: certifyRoles, Pool: three, - Pins: map[string]string{W: "a", C: "a"}, Floor: 5}, C, "a", ByOperator, "", true}, + Pins: pins(W, "a", C, "a"), Floor: 5}, C, "a", ByOperator, "", true}, {"verifier is never the reviewer", Input{Roles: []string{modelrank.SeatReviewer, modelrank.SeatVerifier}, Pool: three, Report: report(row(modelrank.SeatReviewer, "a", 0.9, true), row(modelrank.SeatVerifier, "a", 0.95, true), row(modelrank.SeatVerifier, "c", 0.6, true)), Floor: 5}, modelrank.SeatVerifier, "c", ByEvidence, "evidence for c", false}, @@ -113,6 +131,10 @@ func TestPropose(t *testing.T) { if failed != tc.refused || (failed && !p.Refused) { t.Fatalf("failed = %v, want %v (plan %+v)", failed, tc.refused, p) } + // A refused seat names its flag and the floor it missed (C3). + if tc.name == "below the floor" && (!strings.Contains(s.Unfilled, "5+") || !strings.Contains(s.Unfilled, "pin --writer-model")) { + t.Errorf("unfilled %q must name the floor (5+) and the flag", s.Unfilled) + } if len(p.Seats) != len(tc.in.Roles) { t.Fatalf("%d seats for %d roles", len(p.Seats), len(tc.in.Roles)) } @@ -123,7 +145,7 @@ func TestPropose(t *testing.T) { func TestProposeUnfilledNamesTheFix(t *testing.T) { W, C := modelrank.SeatTestWriter, modelrank.SeatTestCritic p := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, - Pins: map[string]string{W: "a"}, Report: report(row(W, "a", 0.99, false)), Floor: 5}) + Pins: pins(W, "a"), Report: report(row(W, "a", 0.99, false)), Floor: 5}) w := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic")}, Report: report(row(W, "a", 0.99, false)), Floor: 5}) if s := seatOf(t, w, W); !strings.Contains(s.Unfilled, "pin --writer-model") || !strings.Contains(s.Unfilled, "5") || !w.Refused { @@ -136,7 +158,7 @@ func TestProposeUnfilledNamesTheFix(t *testing.T) { func TestProposeHardRuleSetsWhy(t *testing.T) { p := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "x"), m("b", "x")}, - Pins: map[string]string{modelrank.SeatTestWriter: "a", modelrank.SeatTestCritic: "A"}, Floor: 5}) + Pins: pins(modelrank.SeatTestWriter, "a", modelrank.SeatTestCritic, "A"), Floor: 5}) if !p.Refused || p.Why == "" { t.Fatalf("plan = %+v, want refused with Why", p) } @@ -148,3 +170,102 @@ func TestProposeRefusesAMissingFloor(t *testing.T) { t.Fatalf("plan = %+v, want a refusal naming the floor", p) } } + +// The floor is the proposer's own rule, not only modelrank's: a row marked +// sufficient by a ranking run at a lower floor is still below this one. +func TestProposeEnforcesTheFloorItself(t *testing.T) { + W := modelrank.SeatTestWriter + r := row(W, "a", 0.99, true) + r.N = 3 + p := Propose(Input{Roles: certifyRoles, Pool: []Member{m("a", "anthropic"), m("b", "openai")}, + Report: report(r), Floor: 5}) + if s := seatOf(t, p, W); s.Given != "" || !strings.Contains(s.Unfilled, "5+") || !p.Refused { + t.Fatalf("writer seated from a row with N=3 under a floor of 5: %+v", s) + } +} + +// A measured seat whose only qualified member is its counterpart says so, +// rather than claiming no member has the evidence. +func TestProposeSaysWhenTheOnlyQualifiedMemberIsTheCounterpart(t *testing.T) { + R, V := modelrank.SeatReviewer, modelrank.SeatVerifier + p := Propose(Input{Roles: []string{R, V}, Pool: []Member{m("a", "anthropic"), m("c", "openai")}, + Report: report(row(R, "a", 0.9, true), row(V, "a", 0.95, true)), Floor: 5}) + s := seatOf(t, p, V) + if !strings.Contains(s.Unfilled, "already the reviewer") || !strings.Contains(s.Unfilled, "pin --verifier-model") { + t.Fatalf("verifier unfilled text = %q", s.Unfilled) + } +} + +// Every broken pair is reported, in a fixed order, and each names the flags +// that resolve it. +func TestProposeHardRuleReportsEveryPairInOrder(t *testing.T) { + W, C, R, V := modelrank.SeatTestWriter, modelrank.SeatTestCritic, modelrank.SeatReviewer, modelrank.SeatVerifier + in := Input{Roles: []string{W, C, R, V}, Pool: []Member{m("a", "x"), m("b", "y")}, + Pins: pins(W, "a", C, "a", R, "b", V, "b"), Floor: 5} + first := Propose(in).Why + for i := 0; i < 20; i++ { + if got := Propose(in).Why; got != first { + t.Fatalf("Why differs between runs:\n%q\n%q", first, got) + } + } + for _, want := range []string{"test-critic and test-writer", "verifier and reviewer", + "--critic-model", "--writer-model", "--critic-model off", "--verifier-model", "--reviewer-model"} { + if !strings.Contains(first, want) { + t.Errorf("Why %q lacks %q", first, want) + } + } + if strings.Index(first, "test-critic") > strings.Index(first, "verifier and") { + t.Errorf("pairs not in Roles order: %q", first) + } +} + +// Pins are compared by the model they resolve to, not the text the operator +// typed: an alias, a vendor/model spelling and an agent pinned to a model can +// name one model (B3). Every case here has Given != Recorded somewhere. +func TestProposeComparesResolvedPins(t *testing.T) { + W, C := modelrank.SeatTestWriter, modelrank.SeatTestCritic + sonnet := Member{Given: "anthropic/claude-sonnet-5-5", Recorded: "claude-sonnet-5-5", Provider: "anthropic", Rankable: true} + haiku := Member{Given: "anthropic/claude-haiku-5-5", Recorded: "claude-haiku-5-5", Provider: "anthropic", Rankable: true} + codex := Member{Given: "codex:gpt-6", Recorded: "codex:gpt-6", Provider: "openai", Rankable: true} + edge := Member{Given: "edge2", Recorded: "localmodel-a:1", Provider: "ollama", Rankable: true} + + t.Run("agent-pinned critic against the evidence-seated vendor/model writer", func(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{sonnet, codex}, + Pins: map[string]Pin{C: {Given: "claude-code:claude-sonnet-5-5", Recorded: "claude-code:claude-sonnet-5-5", Provider: "anthropic"}}, + Report: report(row(W, "claude-sonnet-5-5", 0.9, true)), Floor: 5}) + if !p.Refused || !strings.Contains(p.Why, "both claude-sonnet-5-5") { + t.Fatalf("plan = %+v, want the hard rule to refuse one model in both seats", p) + } + }) + t.Run("bare pin against vendor/model pin", func(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{codex}, + Pins: map[string]Pin{ + W: {Given: "anthropic/claude-sonnet-5-5", Recorded: "claude-sonnet-5-5", Provider: "anthropic"}, + C: {Given: "sonnet-alias", Recorded: "claude-sonnet-5-5", Provider: "anthropic"}, + }, Floor: 5}) + if !p.Refused || p.Why == "" { + t.Fatalf("plan = %+v, want refused", p) + } + }) + t.Run("alias-pinned writer is not proposed again as the critic", func(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{edge, haiku}, + Pins: map[string]Pin{W: {Given: "edge", Recorded: "localmodel-a:1", Provider: "ollama"}}, Floor: 5}) + if s := seatOf(t, p, C); s.Given != haiku.Given || s.By != ByOrder { + t.Fatalf("critic = %+v, want %s by order", s, haiku.Given) + } + }) + t.Run("a pinned writer outside the pool still steers the critic's provider", func(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{haiku, codex}, + Pins: map[string]Pin{W: {Given: "anthropic/claude-sonnet-5-5", Recorded: "claude-sonnet-5-5", Provider: "anthropic"}}, Floor: 5}) + if s := seatOf(t, p, C); s.Given != codex.Given { + t.Fatalf("critic = %+v, want %s (a provider other than the writer's)", s, codex.Given) + } + }) + t.Run("an unresolved pin is refused, not compared by its typed text", func(t *testing.T) { + p := Propose(Input{Roles: certifyRoles, Pool: []Member{haiku, codex}, + Pins: map[string]Pin{W: {Given: "anthropic/claude-sonnet-5-5"}}, Floor: 5}) + if s := seatOf(t, p, W); s.Unfilled == "" || !p.Refused { + t.Fatalf("writer = %+v, want refused", s) + } + }) +} diff --git a/internal/ledger/row.go b/internal/ledger/row.go index 4bc2cc8f..a11e48ec 100644 --- a/internal/ledger/row.go +++ b/internal/ledger/row.go @@ -56,10 +56,16 @@ type Row struct { // like ModelsByRole: "operator" (named by flag), "evidence" (the recorded // ranking picked it) or "order" (first of the pool the operator named). // "" on an entry written without --models and before the field existed, - // both of which mean every seat was named by flag. + // both of which mean every seat was named by flag. That is a statement + // about the row: a row served from the verdict cache embeds a verdict + // document (VerdictJSON) that keeps the seating it was measured under, + // which can come from another run, with or without --models, because + // the cache key does not name how a seat was chosen. SeatedBy string `json:",omitempty"` - // SeatEvidence names what the ranking read to seat the "evidence" seats - // (for example "the machine ledger (31 scans)"); "" when nothing was read. + // SeatEvidence names what the ranking read to seat the "evidence" seats, + // with no host path (for example "the machine ledger (31 scan(s), 2 + // review(s)); language go"); "" when no seat on the row was seated by + // evidence. SeatEvidence string `json:",omitempty"` MinKillRate *float64 MaxProvenMissed *int diff --git a/internal/reviewrecord/reviewrecord.go b/internal/reviewrecord/reviewrecord.go index 44c95b64..b29d07b5 100644 --- a/internal/reviewrecord/reviewrecord.go +++ b/internal/reviewrecord/reviewrecord.go @@ -88,7 +88,9 @@ type Review struct { // "verifier" like SeatProviders: operator, evidence or order. Absent // means named by flag. SeatedBy map[string]string `json:"seated_by,omitempty"` - // SeatEvidence names what the ranking read to seat the "evidence" seats. + // SeatEvidence names what the ranking read to seat the "evidence" seats, + // with no host path. Absent when no seat on the entry was seated by + // evidence. SeatEvidence string `json:"seat_evidence,omitempty"` // ReviewerModelResolved / VerifierModelResolved report whether the model // field beside them names an actual MODEL, or merely the tool that ran From 81c843566ea0ac5e4fcf810ffecf39544127e9c3 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 13:10:24 -0500 Subject: [PATCH 11/14] Start a --models run's clock when the operator accepts, refuse flag-only errors before the question, and say plainly what doctor and a run with no terminal do, because the record counted a person reading the plan as scan time and certify --repo could still refuse a lineup the operator had already accepted. certify --local and certify --repo restart started_at, total_millis and the host-load sampler at the answer (review's started_at was already taken after it); a --repo scan's total_millis therefore no longer covers the walk that told it the scan's language. certify --repo now refuses --scope-tests and a malformed --local-endpoint before the walk, hands the placements to --models so a pin served only by its placement resolves there too, and names the zero-language case on the evidence line. The draw test's after-the-draw refusal for --repo is now a primary seat no source serves, which the seat resolution that follows the draw refuses. Doctor no longer claims to run nothing: it asks nothing and spends nothing on a seat it proposes, and a --derive-model the operator names is still probed with one real call. The --models help at each door says that with no terminal it audits nothing and exits 0, so a pipeline must pass the seat flags itself, and the top-level usage now says corral models rank reads this machine's ledger unless --db or --source names another source. The consent property gets door-level tests: a declined run and a run with no terminal write no ledger entry, open no store and make no model call at certify --local, certify --repo, demo (through its forward) and review. The gates now also fail a door that registers --models and never hands it to seatFromPool, and an evidence reader or ranking call outside the files that legitimately hold one. Co-Authored-By: Claude Fable 5.2 --- cmd/corral/certify_local.go | 24 +- cmd/corral/certify_repo.go | 70 ++++-- cmd/corral/demo.go | 2 +- cmd/corral/doctor.go | 8 +- cmd/corral/main.go | 6 +- cmd/corral/models_consent_test.go | 274 +++++++++++++++++++++++ cmd/corral/models_doors_test.go | 78 ++++++- cmd/corral/models_flags_test.go | 20 +- cmd/corral/shadow_draw_test.go | 19 +- docs/cli/corral.md | 16 +- site/src/content/docs/docs/cli/corral.md | 16 +- 11 files changed, 475 insertions(+), 58 deletions(-) create mode 100644 cmd/corral/models_consent_test.go diff --git a/cmd/corral/certify_local.go b/cmd/corral/certify_local.go index 277bd375..b9314e53 100644 --- a/cmd/corral/certify_local.go +++ b/cmd/corral/certify_local.go @@ -101,11 +101,14 @@ func runCertifyLocal(args []string, stdout, stderr io.Writer) int { // would file the demo's measurements under that project's origin and HEAD. func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Writer) int { // The run's start, for the ledger entry's started_at and total_millis. + // Restarted when the operator accepts a --models plan (below), so the + // record never counts the time someone spent reading it. startedAt := time.Now() // Host load and the run's own consumption, from here to the entry. The - // deferred Stop ends the sampler on every return; Stop is idempotent. + // deferred Stop ends the sampler on every return (the one running then, + // since a --models acceptance replaces it); Stop is idempotent. hostSampler := hoststat.Start(hostSampleInterval) - defer hostSampler.Stop() + defer func() { hostSampler.Stop() }() flagArgs, checkArgv := splitCertifyArgs(args) fs := flag.NewFlagSet("certify --local", flag.ContinueOnError) @@ -143,7 +146,7 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write writerPool: "a comma-separated POOL of challenger WRITER models, drawn per run exactly as --shadow-pool is. Mutually exclusive with --shadow-writer-model. OFF unless named; NEVER gates the verdict", seed: shadowSeedReplayHelp, }) - modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool") + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, audits nothing and exits 0, so a pipeline must pass the seat flags itself. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool") writerModeFlag := fs.String("writer-mode", "", "how the test-writer attacks this file's survivors: `per-survivor` (the default) makes ONE call per survivor — each carrying the file once as a cacheable shared prefix plus that survivor's diff, each repaired on its own budget and each PROVEN ALONE against its own mutant — or `batched`, the original shape: one call carrying every survivor, one repair budget, one proof pass over all of them. Nothing measured changes between them (a survivor is proven iff an authored test kills it alone and passes on the original, either way); what changes is that one unbuildable test no longer spends the whole file's retries and takes every other survivor down with it. Each survivor's proof in per-survivor mode runs its OWN compliant baseline (a compliant pass plus a canary, per seat), so a file with N survivors pays N baselines where batched paid one: on a repo whose suite takes a minute, prefer --writer-mode batched or expect N baselines' worth of wall clock.") matrixFlag := fs.Bool("matrix", false, "opt into the tests×mutants matrix: after the primary pass, re-score EVERY dev test ALONE against the run's mutants — a per-test adequacy readout + a delete-candidate list, instead of one dev-suite-wide number. COSTLY: T tests × M mutants extra jail runs (T×M, on top of the primary pass), so leave off by default on a big suite") var localEndpointFlag stringSlice @@ -232,7 +235,9 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write // an operator must never accept a lineup for a run that was always going // to be refused. They also precede the shadow draw and seat resolution, // which read stores and a daemon; checking the command line first is the - // cheaper order, and the order certify --repo keeps. + // cheaper order. certify --repo keeps the same rule, with its walk + // between its flag refusals and its question, because the walk is what + // tells it the scan's language. // // --local-endpoint placements are parsed BEFORE the seats resolve: a // placement is itself a declared source for a seat with a bare local name. @@ -297,14 +302,23 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write // seat resolution, so a proposed member is drawn against and resolved // exactly as the same model typed into its flag would be. The flags the // operator did type are pins. + seatDeps := doorPoolDeps(stdout, stderr) + seatDeps.placed = localEndpoints seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, "", modelsPool, []poolRole{ {role: advpool.RoleMutantGenerator, val: mutantModel}, {role: advpool.RoleTestWriter, val: writerModel}, {role: advpool.RoleTestCritic, val: criticModel}, - }, doorPoolDeps(stdout, stderr)) + }, seatDeps) if poolDone { return poolExit } + if seating.proposed() { + // The run starts when the operator accepts: the scan time and the + // host-load samples must not include a person reading the plan, or + // the evidence read behind it. What came before (flag checks, the + // language detection) is milliseconds and is not the run either. + startedAt, hostSampler = restartRunClock(hostSampler) + } // The bug-catching scorecard store: a --local run's per-seat metrics // (including the shadow head-to-head) persist next to the signed build // ledger and feed the same `corral scorecard` surface the daemon's runs feed. diff --git a/cmd/corral/certify_repo.go b/cmd/corral/certify_repo.go index 464a3847..90d6158f 100644 --- a/cmd/corral/certify_repo.go +++ b/cmd/corral/certify_repo.go @@ -83,7 +83,7 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { writerPool: "a comma-separated POOL of challenger WRITER models, drawn once per scan exactly as --shadow-pool is. Mutually exclusive with --shadow-writer-model. OFF unless named; NEVER gates the verdict", seed: shadowSeedReplayHelp, }) - modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it spans several), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits; --dry-run prints the plan and asks nothing). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool") + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it has none or several), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, audits nothing and exits 0, so a pipeline must pass the seat flags itself; --dry-run prints the plan and asks nothing. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool") owner := fs.String("owner", "local", "owning account for the scan (tenant identifier)") commit := fs.String("commit", "", "commit SHA the report is bound to") swarmFlag := fs.Int("swarm", 0, "max concurrent audit workers (0 = auto-size to this host's cores); on --substrate workspace it also sizes the private trees that score one file's mutants at once (budget/4, min 1), so --swarm 4 is one tree") @@ -363,16 +363,35 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { maxProvenMissed = &v } + // Refusals the command line alone decides, before the walk and before + // --models asks anything: an operator must never accept a lineup for a + // run that was always going to be refused. + if *scopeTestsFlag { + fmt.Fprintln(stderr, "corral certify --repo: --scope-tests was removed. Its paired-FILE scoping inverted verdicts (requests/adapters.py 1.00 -> 0.00). Selection is now by coverage evidence and on by default; pass --whole-suite to grade against the whole suite. See docs/design/test-selection.md") + return 2 + } + // --local-endpoint placements are parsed here, before the seats resolve: + // a placement is itself a declared source for a seat with a bare local + // name, so --models resolves a pin with them and the preflight below + // (resolveAuditRoles) sees them folded in. + localEndpoints, lerr := parseLocalEndpoints(localEndpointFlag) + if lerr != nil { + fmt.Fprintf(stderr, "corral certify --repo: %v\n", lerr) + return 2 + } + // Captured here, before enumeration: this is the header row's // provenance timestamp for the whole invocation, not just the audit - // portion of it. Read only if --record is given (see the fail-open call - // site near the bottom of this function). + // portion of it, except that a --models run restarts it when the + // operator accepts the plan (below). Read only if --record is given (see + // the fail-open call site near the bottom of this function). startedAt := time.Now() // The host's load and the run's own consumption, sampled until the scan // header is built. The deferred Stop ends the sampler on every early - // return; Stop is idempotent, so the one before the header is the read. + // return (the one running then, since a --models acceptance replaces + // it); Stop is idempotent, so the one before the header is the read. hostSampler := hoststat.Start(hostSampleInterval) - defer hostSampler.Stop() + defer func() { hostSampler.Stop() }() var testMap *reposcan.TestMap if *testsPath != "" { @@ -427,14 +446,18 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { // the one the evidence pass instruments (preflightLanguages over the // enumerated sources): a superset of what the scan audits, so a scan // ranked by one language never audits another. One language ranks by it; - // none or several rank across all of them. + // none or several rank across all of them, and the evidence line says + // which of the two it was. // // A dry run spends nothing and demands no credential for any seat, so // there the pool is not credential-checked and the plan is shown without // a question. Under --json the plan goes to stderr: not one byte of prose // may precede the inventory. - scanLang := shadowpool.LangAny - if langs := preflightLanguages(enumeratedSourcePaths(cands, excl)); len(langs) == 1 { + scanLang, anyWhy := shadowpool.LangAny, anyLangSeveral + switch langs := preflightLanguages(enumeratedSourcePaths(cands, excl)); len(langs) { + case 0: + anyWhy = anyLangNone + case 1: for l := range langs { scanLang = l } @@ -453,13 +476,23 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { planOut = stderr } seatDeps := doorPoolDeps(planOut, stderr) + seatDeps.placed = localEndpoints if *dryRun { seatDeps.confirm, seatDeps.noCredentialCheck = acceptUnasked, true } - seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --repo", *repoDir, scanLang, anyLangSeveral, modelsPool, poolRoles, seatDeps) + seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --repo", *repoDir, scanLang, anyWhy, modelsPool, poolRoles, seatDeps) if poolDone { return poolExit } + if seating.proposed() && !*dryRun { + // The scan starts when the operator accepts: started_at, + // total_millis and the host-load samples must not include a person + // reading the plan, or the evidence read behind it. The walk above + // (which the plan needed for the scan's language) is excluded with + // them, and is the one part of the invocation a --models scan's + // total_millis does not cover. + startedAt, hostSampler = restartRunClock(hostSampler) + } if *dryRun && seating.proposed() { fmt.Fprintln(planOut, "a dry run asks nothing and runs nothing: this is the lineup a real run would propose") } @@ -501,15 +534,8 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { // Resolved HERE, before any seat value is read by the preflight, the cache // key, the ledger or the signed statement, so every one of them records // the CONCRETE model. An alias is a label for humans and is never - // authoritative. - // --local-endpoint placements are parsed BEFORE the seats resolve: a - // placement is itself a declared source for a seat with a bare local name, - // and the preflight below (resolveAuditRoles) must see it folded in. - localEndpoints, lerr := parseLocalEndpoints(localEndpointFlag) - if lerr != nil { - fmt.Fprintf(stderr, "corral certify --repo: %v\n", lerr) - return 2 - } + // authoritative. The --local-endpoint placements, parsed with the other + // flag refusals above, are folded in here. // context.Background(): no cancellable context exists here. The SIGINT // handler (auditContext) is installed after the goal derivation, on // purpose — an interrupt before any audit has nothing to record, and @@ -521,10 +547,6 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "corral certify --repo: %v\n", regErr) return 2 } - if *scopeTestsFlag { - fmt.Fprintln(stderr, "corral certify --repo: --scope-tests was removed. Its paired-FILE scoping inverted verdicts (requests/adapters.py 1.00 -> 0.00). Selection is now by coverage evidence and on by default; pass --whole-suite to grade against the whole suite. See docs/design/test-selection.md") - return 2 - } fmt.Fprintf(stdout, "corral certify --repo %s\n", *repoDir) if scope != "" { @@ -1987,8 +2009,10 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { SeatProviders: roleMapJSON(seatProviders), // How the roster's seats were chosen, narrowed to the roster as // its providers are (the deriver is on it only when it ran). + // The evidence line goes with the last evidence seat the + // narrowing keeps, as the verdict's does (B12). SeatedBy: roleMapJSON(advpool.SeatedOnly(seating.SeatedBy, models())), - SeatEvidence: seating.SeatEvidence, + SeatEvidence: seatEvidenceFor(advpool.SeatedOnly(seating.SeatedBy, models()), seating.SeatEvidence), MinKillRate: minKillRate, MaxProvenMissed: maxProvenMissed, Passed: boolPtr(exitCode == 0), PushedBy: auditpush.PushedByCertify, diff --git a/cmd/corral/demo.go b/cmd/corral/demo.go index ce9d7c42..52880373 100644 --- a/cmd/corral/demo.go +++ b/cmd/corral/demo.go @@ -51,7 +51,7 @@ func runDemo(args []string, stdout, stderr io.Writer) int { // seats (ranked for the demo's language, Go) and records how they were // chosen. Proposing here and passing plain seat flags on would record // a lineup the evidence chose as one the operator named. - modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused") + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal (with no terminal it audits nothing and exits 0). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused") if err := fs.Parse(args); err != nil { return flagParseExit(err) } diff --git a/cmd/corral/doctor.go b/cmd/corral/doctor.go index a5f2886b..5d67cf68 100644 --- a/cmd/corral/doctor.go +++ b/cmd/corral/doctor.go @@ -96,7 +96,7 @@ func runDoctor(args []string, stdout, stderr io.Writer) int { }) deriveModel := fs.String("derive-model", "", "the goal-derivation model a `certify --repo` run will name, if any. Naming it makes doctor SPEND ONE small real derive call through certify's own deriver, on a tiny built-in file with an obvious property, and fail the check when the seat errors or returns an empty reply; nothing is called unless this is named") repoDir := fs.String("repo", ".", "the repository the run will audit — where its .corral/models.json registry is read from, exactly as certify reads it") - modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code) and checks those seats. doctor asks nothing and runs nothing (certify would run nothing until you accept the plan), and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused") + modelsPool := registerModelsFlags(fs, "a comma-separated `POOL` of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code or when its language cannot be detected) and checks those seats. doctor asks nothing and spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. certify would run nothing until you accept the plan, and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused") if err := fs.Parse(args); err != nil { return flagParseExit(err) } @@ -106,7 +106,9 @@ func runDoctor(args []string, stdout, stderr io.Writer) int { deriveNamed := strings.TrimSpace(*deriveModel) != "" // --models: the lineup certify would propose, written into the seat // flags so every check below is about the seats that would run. doctor - // spends nothing, so it accepts the plan without asking and says so. + // spends nothing on a seat the pool proposes, so it accepts the plan + // without asking and says so; its one paid check is the derive probe, + // and only on a deriver the operator named. poolLang := shadowpool.LangAny if c := strings.TrimSpace(*code); c != "" { if p, ok := lang.Detect(c); ok { @@ -125,7 +127,7 @@ func runDoctor(args []string, stdout, stderr io.Writer) int { return poolExit } if seating.proposed() { - fmt.Fprintln(stdout, "doctor runs nothing: this is the lineup `certify` would propose") + fmt.Fprintln(stdout, "doctor asks nothing: this is the lineup `certify` would propose") } // Every member of every challenger pool, checked exactly as certify checks // it before its draw — but never drawn: doctor answers "would this run diff --git a/cmd/corral/main.go b/cmd/corral/main.go index 2c4c9300..7b50c08a 100644 --- a/cmd/corral/main.go +++ b/cmd/corral/main.go @@ -266,8 +266,10 @@ Usage: THE OPERATOR NAMED ON THAT RUN: this command writes no config, changes no default and staffs no seat — corral has no default models. - flags: --db (a pushed warehouse instead of the local - bug-catching ledger; unreachable REFUSES, never falls back) + flags: --db (a ledger directory or pushed warehouse + instead of this machine's ledger; unreachable REFUSES, never + falls back) --source ledger|scorecard (without --db: this + machine's ledger, or the old local bug-catching store) --seat --lang --min-runs n --json corral criticscore list list execution-checked test-critic findings still awaiting human adjudication — the local store certify --local writes, or a diff --git a/cmd/corral/models_consent_test.go b/cmd/corral/models_consent_test.go new file mode 100644 index 00000000..2b7f764a --- /dev/null +++ b/cmd/corral/models_consent_test.go @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: Elastic-2.0 + +package main + +import ( + "bytes" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pdbethke/corralai/internal/agentbackend" + "github.com/pdbethke/corralai/internal/auditpush" +) + +// The consent property, door by door: a --models run the operator declines, +// or one with nobody to ask, writes no ledger entry, opens no store for +// write and makes no model call. seatFromPool's own tests prove it returns +// "done"; these prove each door stops on it, so a door that carried on past +// a decline (or wired the flag and ignored it) fails here. + +// consentPool names two wired agent seats, each pinned to a model so the +// record can rank them: fakea:m1 is measured as generator and writer, and +// fakec takes the critic by pool order. Every seat is an agent whose script +// logs each call, so "no model call" is read from two files. +const consentPool = "fakea:m1,fakec" + +// consentEvidence measures fakea:m1 over six Go runs in both measured seats. +func consentEvidence() rankEvidence { + runs := []string{"r1", "r2", "r3", "r4", "r5", "r6"} + obs := genObs("fakea:m1", "go", runs...) + for _, r := range runs { + obs = append(obs, writerObs("fakea:m1", "go", r, 5, 5)) + } + return rankEvidence{Obs: obs, Source: "test evidence (6 scan(s))", RecordSource: "test evidence (6 scan(s))", + Generations: generationRange{Min: "48", Max: "48", Scans: 6}} +} + +// consentAnswer is how the stubbed operator answers. +type consentAnswer struct { + name string + accepted, interactive bool +} + +var consentAnswers = []consentAnswer{ + {"declined", false, true}, + {"no terminal", false, false}, +} + +// stubConsent wires the doors' seams to consentEvidence and an operator who +// answers as a says, wires both agents, isolates the state directory and the +// machine ledger, and returns the two agents' call logs and the state dir. +func stubConsent(t *testing.T, a consentAnswer) (logs []string, state, machine string) { + t.Helper() + asked := new(int) + orig := doorPoolDeps + t.Cleanup(func() { doorPoolDeps = orig }) + doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{ + load: func(rankSource) (rankEvidence, error) { return consentEvidence(), nil }, + confirm: func(string) (bool, bool, error) { *asked++; return a.accepted, a.interactive, nil }, + stdout: stdout, stderr: stderr, + } + } + origRun := shadowMemberRunnable + t.Cleanup(func() { shadowMemberRunnable = origRun }) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } + t.Cleanup(func() { + if *asked != 1 { + t.Errorf("the operator was asked %d time(s), want once", *asked) + } + }) + noKeys(t) + clearSeatEnv(t) + state = t.TempDir() + t.Setenv("CORRALAI_STATE_DIR", state) + t.Setenv("CORRALAI_BUGCATCH_DB", "") + machine = withMachineLedger(t) + logs = []string{wiredAgent(t, "fakea", "claude-json", claudeJSONReply), wiredAgent(t, "fakec", "claude-json", claudeJSONReply)} + // fakea is seated pinned to a model (fakea:m1), so its definition must + // carry the pin, or the door refuses the seat before it could run. + t.Setenv("CORRALAI_AGENT_FAKEA", filepath.Join(filepath.Dir(logs[0]), "agent-fakea")+" {model:--model}") + return logs, state, machine +} + +// assertNothingHappened is the property: no call logged by either agent, no +// entry in any ledger named, and no DuckDB store anywhere under the state +// directory (the bug-catch store is the one a certify run opens for write). +func assertNothingHappened(t *testing.T, code int, out, errs string, logs []string, state string, ledgers ...string) { + t.Helper() + if code != 0 { + t.Errorf("exit %d, want 0 (a decline and a missing terminal are not failures)\n%s%s", code, out, errs) + } + for _, l := range logs { + if b, err := os.ReadFile(l); err == nil && len(b) > 0 { // #nosec G304 -- test temp path + t.Errorf("a model call was made (%s):\n%s", l, b) + } + } + for _, dir := range ledgers { + if _, err := os.Stat(dir); err == nil { + entries, _ := auditpush.ReadLedgerDir(dir) + t.Errorf("ledger %s exists with %d entr(ies)", dir, len(entries)) + } + } + _ = filepath.WalkDir(state, func(p string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && strings.HasSuffix(p, ".duckdb") { + t.Errorf("a store was opened under the state directory: %s", p) + } + return nil + }) + if !strings.Contains(out, "seats (from --models "+consentPool+")") { + t.Errorf("the door never reached the plan, so the test proves nothing:\n%s%s", out, errs) + } +} + +func TestDeclinedOrUnaskedLocalRunDoesNothing(t *testing.T) { + for _, a := range consentAnswers { + t.Run(a.name, func(t *testing.T) { + root, setPath, _ := localEntryFixture(t) + logs, state, machine := stubConsent(t, a) + var out, errb bytes.Buffer + code := runCertifyLocal([]string{ + "--repo-dir", root, "--code", "pkg/a.go", "--goal", "A returns 1", + "--models", consentPool, "--mutants", setPath, "--quiet", "--", "go", "test", "./...", + }, &out, &errb) + assertNothingHappened(t, code, out.String(), errb.String(), logs, state, machine) + }) + } +} + +func TestDeclinedOrUnaskedRepoRunDoesNothing(t *testing.T) { + for _, a := range consentAnswers { + t.Run(a.name, func(t *testing.T) { + root, setPath, goals := telemetryRepoFixture(t) + logs, state, machine := stubConsent(t, a) + var out, errb bytes.Buffer + code := runCertifyRepo([]string{ + "--repo", root, "--models", consentPool, + "--goals", goals, "--substrate", substrateWorkspace, "--all", "--mutants", setPath, + }, &out, &errb) + assertNothingHappened(t, code, out.String(), errb.String(), logs, state, machine, filepath.Join(root, ".corral", "ledger")) + }) + } +} + +// demo reaches certify --local by its argv, so the property must hold +// through the forward too. +func TestDeclinedOrUnaskedDemoDoesNothing(t *testing.T) { + for _, a := range consentAnswers { + t.Run(a.name, func(t *testing.T) { + logs, state, machine := stubConsent(t, a) + var out, errb bytes.Buffer + code := runDemo([]string{"--dir", t.TempDir(), "--models", consentPool}, &out, &errb) + assertNothingHappened(t, code, out.String(), errb.String(), logs, state, machine) + }) + } +} + +// review builds its seats through newReviewerBackend, so "no backend built" +// is read directly there as well as from the ledgers. +func TestDeclinedOrUnaskedReviewDoesNothing(t *testing.T) { + for _, a := range consentAnswers { + t.Run(a.name, func(t *testing.T) { + asked := new(int) + orig := doorPoolDeps + t.Cleanup(func() { doorPoolDeps = orig }) + doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{ + load: func(rankSource) (rankEvidence, error) { return reviewEvidence(), nil }, + confirm: func(string) (bool, bool, error) { *asked++; return a.accepted, a.interactive, nil }, + stdout: stdout, stderr: stderr, + } + } + origRun := shadowMemberRunnable + t.Cleanup(func() { shadowMemberRunnable = origRun }) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } + r := reviewModelsRunWith(t, reviewRunOpts{verifierReply: `{"opinion":"v","refutations":[]}`}, "--models", reviewPool) + if *asked != 1 { + t.Errorf("asked %d time(s), want once", *asked) + } + if r.code != 0 || len(r.seen) != 0 || len(r.entries) != 0 { + t.Errorf("exit %d, seats built %v, %d entries; want 0, none, none\n%s%s", r.code, r.seen, len(r.entries), r.out, r.errs) + } + if _, err := os.Stat(r.machineLedger); err == nil { + t.Errorf("the machine ledger %s was written", r.machineLedger) + } + if !strings.Contains(r.out, "seats (from --models "+reviewPool+")") { + t.Errorf("the door never reached the plan:\n%s%s", r.out, r.errs) + } + }) + } +} + +// Scan time and host load are measured from the moment the operator +// accepted, never from before the question (B1): a run's recorded duration +// is what the run took, not how long someone read the plan. +func TestAcceptedRunTimesFromTheAnswer(t *testing.T) { + slowAccept := func(t *testing.T) *time.Time { + t.Helper() + at := new(time.Time) + orig := doorPoolDeps + t.Cleanup(func() { doorPoolDeps = orig }) + doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + return poolDeps{ + load: func(rankSource) (rankEvidence, error) { return recordEvidence(), nil }, + confirm: func(string) (bool, bool, error) { + time.Sleep(1500 * time.Millisecond) + *at = time.Now() + return true, true, nil + }, + stdout: stdout, stderr: stderr, + } + } + origRun := shadowMemberRunnable + t.Cleanup(func() { shadowMemberRunnable = origRun }) + shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } + return at + } + check := func(t *testing.T, e auditpush.LedgerEntry, accepted time.Time, finished time.Time) { + t.Helper() + s := e.Bundle.Scan + if s.StartedAt == nil || s.StartedAt.Before(accepted.Truncate(time.Microsecond)) { + t.Errorf("started_at %v is before the operator accepted at %v", s.StartedAt, accepted) + } + if s.TotalMillis == nil { + t.Fatalf("no total_millis recorded") + } + if *s.TotalMillis > finished.Sub(accepted).Milliseconds() { + t.Errorf("total_millis %d includes the wait (the run after acceptance took at most %dms)", *s.TotalMillis, finished.Sub(accepted).Milliseconds()) + } + } + t.Run("certify --local", func(t *testing.T) { + root, setPath, _ := localEntryFixture(t) + at := slowAccept(t) + ml := withMachineLedger(t) + var out, errb bytes.Buffer + code := runCertifyLocal([]string{ + "--repo-dir", root, "--code", "pkg/a.go", "--goal", "A returns 1", + "--models", recordPool, "--mutants", setPath, "--quiet", "--", "go", "test", "./...", + }, &out, &errb) + finished := time.Now() + localRunFinished(t, code, out.String(), errb.String()) + entries := scanEntriesIn(t, ml) + if len(entries) != 1 { + t.Fatalf("%d entries", len(entries)) + } + check(t, entries[0], *at, finished) + }) + t.Run("certify --repo", func(t *testing.T) { + root, setPath, goals := telemetryRepoFixture(t) + at := slowAccept(t) + machine := withMachineLedger(t) + noKeys(t) + t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") + wiredAgent(t, "fakec", "claude-json", claudeJSONReply) + var out, errb bytes.Buffer + code := runCertifyRepo([]string{ + "--repo", root, "--models", recordPool, + "--goals", goals, "--substrate", substrateWorkspace, "--all", "--mutants", setPath, + }, &out, &errb) + finished := time.Now() + if code != 0 { + t.Fatalf("exit=%d\n%s%s", code, out.String(), errb.String()) + } + entries, err := auditpush.ReadLedgerDir(machine) + if err != nil || len(entries) != 1 { + t.Fatalf("%d entries, %v", len(entries), err) + } + check(t, entries[0], *at, finished) + }) +} diff --git a/cmd/corral/models_doors_test.go b/cmd/corral/models_doors_test.go index 03386445..14e446f1 100644 --- a/cmd/corral/models_doors_test.go +++ b/cmd/corral/models_doors_test.go @@ -4,6 +4,7 @@ package main import ( "bytes" + "encoding/json" "errors" "io" "os" @@ -39,7 +40,7 @@ func TestDoctorModelsPrintsThePlanAndAsksNothing(t *testing.T) { } text := out.String() plan := strings.Index(text, "seats (from --models "+recordPool+")") - note := strings.Index(text, "doctor runs nothing: this is the lineup `certify` would propose") + note := strings.Index(text, "doctor asks nothing: this is the lineup `certify` would propose") if plan < 0 || note < plan { t.Fatalf("want the plan, then the doctor note under it:\n%s\nstderr:\n%s", text, errb.String()) } @@ -150,8 +151,8 @@ func TestModelsHelpIsTrueAtEachDoor(t *testing.T) { must []string }{ {"certify --local", runCertifyLocal, []string{"not a default", "accept", "language"}}, - {"certify --repo", runCertifyRepo, []string{"not a default", "accept", "goal-deriver", "--dry-run"}}, - {"doctor", runDoctor, []string{"not a default", "accept", "runs nothing"}}, + {"certify --repo", runCertifyRepo, []string{"not a default", "accept", "goal-deriver", "--dry-run", "audits nothing and exits 0", "none or several"}}, + {"doctor", runDoctor, []string{"not a default", "accept", "doctor asks nothing and spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call"}}, {"demo", runDemo, []string{"not a default", "accept", "Go"}}, } for _, d := range doors { @@ -163,3 +164,74 @@ func TestModelsHelpIsTrueAtEachDoor(t *testing.T) { } } } + +// certify --repo refuses what its flags alone decide before --models asks +// anything (B4): --scope-tests and a malformed --local-endpoint used to be +// refused after the operator had accepted a lineup. +func TestRepoModelsRefusesFlagErrorsBeforeAsking(t *testing.T) { + for _, c := range []struct { + name string + args []string + want string + }{ + {"--scope-tests", []string{"--scope-tests"}, "--scope-tests was removed"}, + {"--local-endpoint", []string{"--local-endpoint", "no-such-role=http://localhost:1"}, "unknown role"}, + } { + t.Run(c.name, func(t *testing.T) { + root, setPath, goals := telemetryRepoFixture(t) + asked := stubDoorPool(t) + var out, errb bytes.Buffer + code := runCertifyRepo(append([]string{"--repo", root, "--models", recordPool, "--goals", goals, + "--substrate", substrateWorkspace, "--all", "--mutants", setPath}, c.args...), &out, &errb) + if code != 2 || !strings.Contains(errb.String(), c.want) { + t.Fatalf("exit %d, want 2 naming %q:\n%s", code, c.want, errb.String()) + } + if *asked != 0 || strings.Contains(out.String(), "seats (from --models") { + t.Errorf("asked %d time(s) and printed a plan for a run its flags refuse:\n%s", *asked, out.String()) + } + }) + } +} + +// The top-level usage names what `corral models rank` reads without --db: +// this machine's ledger, with --source to choose the old scorecard store +// (B14). It said "instead of the local bug-catching ledger" after the default +// had moved. +func TestTopLevelUsageNamesTheRankSources(t *testing.T) { + u := usageText() + if strings.Contains(u, "instead of the local\n") || strings.Contains(u, "bug-catching ledger; unreachable") { + t.Errorf("usage still says --db replaces the local bug-catching ledger") + } + for _, want := range []string{"instead of this machine's ledger", "--source ledger|scorecard"} { + if !strings.Contains(u, want) { + t.Errorf("usage lacks %q", want) + } + } +} + +// Under --json a dry run's stdout is the inventory document and nothing +// else; the --models plan and the dry-run note go to stderr (C6). +func TestRepoJSONDryRunModelsKeepsStdoutJSON(t *testing.T) { + root, _, goals := telemetryRepoFixture(t) + asked := stubDoorPool(t) + noKeys(t) + clearSeatEnv(t) + wiredAgent(t, "fakec", "claude-json", claudeJSONReply) + var out, errb bytes.Buffer + code := runCertifyRepo([]string{"--repo", root, "--models", recordPool, "--goals", goals, "--dry-run", "--json"}, &out, &errb) + if code != 0 { + t.Fatalf("exit=%d\n%s%s", code, out.String(), errb.String()) + } + if *asked != 0 { + t.Errorf("asked %d time(s)", *asked) + } + if !json.Valid(out.Bytes()) || !strings.HasPrefix(strings.TrimSpace(out.String()), "{") { + t.Errorf("stdout is not exactly one JSON document:\n%s", out.String()) + } + if strings.Contains(out.String(), "seats (from --models") { + t.Errorf("the plan reached stdout") + } + if !strings.Contains(errb.String(), "seats (from --models "+recordPool+")") || !strings.Contains(errb.String(), "a dry run asks nothing") { + t.Errorf("the plan and the dry-run note must be on stderr:\n%s", errb.String()) + } +} diff --git a/cmd/corral/models_flags_test.go b/cmd/corral/models_flags_test.go index 363691a2..8aa93d77 100644 --- a/cmd/corral/models_flags_test.go +++ b/cmd/corral/models_flags_test.go @@ -65,6 +65,12 @@ func TestEveryDoorWithSeatFlagsRegistersModelsFlags(t *testing.T) { if !wired { t.Errorf("%s registers a seat flag and not --models; call registerModelsFlags", f) } + // Registered is not used: a door that parses --models and never + // hands it to seatFromPool (or, for demo, to the command it + // forwards to) accepts a pool and silently ignores it. + if wired && !strings.Contains(src, "seatFromPool(") && !strings.Contains(src, `"--models"`) { + t.Errorf("%s registers --models and neither calls seatFromPool nor forwards \"--models\"", f) + } } // certify --local, certify --repo, doctor, demo and review. if doors < 5 { @@ -82,6 +88,18 @@ func TestOnlyAutoseatDecidesASeatFromEvidence(t *testing.T) { // models_rank.go is the disclosure verb; review.go reads the // ranking's seat names and predates the proposer. `"github.com/pdbethke/corralai/internal/modelrank"`: {"models_pool.go": true, "models_rank.go": true, "review.go": true}, + // The evidence readers and the ranking itself. An import gate alone + // misses a door that reads the evidence with the package's own + // loader and ranks it, so the calls are held to their places too. + // main.go hands defaultRankLoader to `corral models rank`, which + // discloses and seats nothing; models_pool.go wires it into + // doorPoolDeps, the one reader seatFromPool uses; models_rank.go + // defines both loaders and runs the disclosure verb's ranking. + "defaultRankLoader": {"main.go": true, "models_pool.go": true, "models_rank.go": true}, + "machineLedgerRankEvidence(": {"models_rank.go": true}, + // models_pool.go ranks for the proposer (rankPool); models_rank.go + // ranks for the disclosure verb. + "modelrank.Rank(": {"models_pool.go": true, "models_rank.go": true}, } files, err := filepath.Glob("*.go") if err != nil || len(files) == 0 { @@ -97,7 +115,7 @@ func TestOnlyAutoseatDecidesASeatFromEvidence(t *testing.T) { } for imp, ok := range allowed { if strings.Contains(string(b), imp) && !ok[f] { - t.Errorf("%s imports %s; only %v may", f, imp, allowedFiles(ok)) + t.Errorf("%s uses %s; only %v may", f, imp, allowedFiles(ok)) } } } diff --git a/cmd/corral/shadow_draw_test.go b/cmd/corral/shadow_draw_test.go index c39cb5a0..43638257 100644 --- a/cmd/corral/shadow_draw_test.go +++ b/cmd/corral/shadow_draw_test.go @@ -163,17 +163,21 @@ func TestEveryDoorDrawsFromItsPoolFlags(t *testing.T) { t.Setenv("CORRALAI_BUGCATCH_DB", filepath.Join(t.TempDir(), "bc.duckdb")) pools := []string{"--shadow-pool", "localmodel-a:1,localmodel-b:1", "--shadow-writer-pool", "localmodel-c:1,localmodel-d:1", "--shadow-seed", "0x2a"} doors := []struct { - name string - run func(args []string, stdout, stderr io.Writer) int - args []string + name string + run func(args []string, stdout, stderr io.Writer) int + args []string + refusal string // the after-the-draw refusal the door must reach }{ // A --prior that does not exist: refused after the draw and the // seat resolution, before any jail or spend. (Missing --code was the // trigger until the flag refusals moved ahead of the draw, so that // --models never asks about a run the command line already refuses.) - {"certify --local", runCertifyLocal, append([]string{"--lang", "go", "--code", "a.go", "--goal", "g", "--prior", filepath.Join(t.TempDir(), "no-such-prior")}, pools...)}, - // --scope-tests is refused right after the registry, which runs after the draw. - {"certify --repo", runCertifyRepo, append([]string{"--repo", t.TempDir(), "--scope-tests"}, pools...)}, + {"certify --local", runCertifyLocal, append([]string{"--lang", "go", "--code", "a.go", "--goal", "g", "--prior", filepath.Join(t.TempDir(), "no-such-prior")}, pools...), "no-such-prior"}, + // A primary seat no source serves: refused by the seat resolution, + // which runs after the draw and needs it (the drawn challengers are + // resolved in the same call). --scope-tests was the trigger until it + // moved ahead of --models with the other flag-only refusals. + {"certify --repo", runCertifyRepo, append([]string{"--repo", t.TempDir(), "--writer-model", "no-source-serves-this"}, pools...), "no-source-serves-this"}, } for _, d := range doors { t.Run(d.name, func(t *testing.T) { @@ -181,6 +185,9 @@ func TestEveryDoorDrawsFromItsPoolFlags(t *testing.T) { if rc := d.run(d.args, &out, &errb); rc != 2 { t.Fatalf("rc = %d, want the usage refusal after the draw\n%s", rc, errb.String()) } + if !strings.Contains(errb.String(), d.refusal) { + t.Errorf("refused, but not by the after-the-draw refusal naming %q:\n%s", d.refusal, errb.String()) + } for _, want := range []string{"mutant-generator challenger drawn from a pool of 2", "test-writer challenger drawn from a pool of 2", "0x000000000000002a"} { if !strings.Contains(errb.String(), want) { t.Errorf("stderr lacks %q:\n%s", want, errb.String()) diff --git a/docs/cli/corral.md b/docs/cli/corral.md index 92afb003..ded16899 100644 --- a/docs/cli/corral.md +++ b/docs/cli/corral.md @@ -91,8 +91,10 @@ Usage: THE OPERATOR NAMED ON THAT RUN: this command writes no config, changes no default and staffs no seat — corral has no default models. - flags: --db (a pushed warehouse instead of the local - bug-catching ledger; unreachable REFUSES, never falls back) + flags: --db (a ledger directory or pushed warehouse + instead of this machine's ledger; unreachable REFUSES, never + falls back) --source ledger|scorecard (without --db: this + machine's ledger, or the old local bug-catching store) --seat --lang --min-runs n --json corral criticscore list list execution-checked test-critic findings still awaiting human adjudication — the local store certify --local writes, or a @@ -305,7 +307,7 @@ Usage of certify --local: -max-tokens int cap on model TOKENS for the whole run, input + output, every seat (0 = no cap; an agent seat that reports no token usage is refused, since the cap could not see it). Checked before each call and charged after it, so one in-flight call can overshoot by its own size. Once reached: a generator seat that has not run makes its file ungradable (executor-error naming the cap), a writer or critic seat is skipped and the file flagged as it is for a provider failure — the dev kill rate already measured stands. The cost line says the cap was reached and after how many calls. Corral has bounded mutants, shards and wall clock and never money; this is the money bound -models POOL - a comma-separated POOL of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool + a comma-separated POOL of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, audits nothing and exits 0, so a pipeline must pass the seat flags itself. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string @@ -418,7 +420,7 @@ Usage of demo: -dir string where to write the demo project (default: a new temp dir, printed and kept) -models POOL - a comma-separated POOL of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused + a comma-separated POOL of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal (with no terminal it audits nothing and exits 0). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_) -writer-model anthropic/… @@ -438,7 +440,7 @@ Usage of doctor: -jail string sandbox backend (default: auto-detect). "container" needs CORRALAI_EXEC_IMAGE set to a toolchain image, e.g. CORRALAI_EXEC_IMAGE=python:3.12-bookworm -models POOL - a comma-separated POOL of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code) and checks those seats. doctor asks nothing and runs nothing (certify would run nothing until you accept the plan), and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused + a comma-separated POOL of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code or when its language cannot be detected) and checks those seats. doctor asks nothing and spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. certify would run nothing until you accept the plan, and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model string the mutant-generator model whose credential to check -repo string @@ -983,7 +985,7 @@ flags of `corral review`: -max-bytes int how many bytes of the scope the reviewer is shown; files past the cap are listed by name and the review records them as unshown (default 200000) -models POOL - a comma-separated POOL of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool + a comma-separated POOL of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, reviews nothing and exits 0, so a pipeline must pass the seat flags itself. A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool -no-ledger print the review and write no entry, in either ledger -push string @@ -1318,7 +1320,7 @@ Usage of certify --repo: -min-kill-rate string fail the scan (exit 1) if ANY audited file's kill rate is below this value (0.0-1.0 inclusive; a minimum, so a file exactly at the threshold passes). Opt-in: unset by default, so exit codes are unchanged unless this is given. Applies PER FILE, not to the aggregate — a well-tested file must not mask a weak one -models POOL - a comma-separated POOL of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it spans several), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits; --dry-run prints the plan and asks nothing). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool + a comma-separated POOL of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it has none or several), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, audits nothing and exits 0, so a pipeline must pass the seat flags itself; --dry-run prints the plan and asks nothing. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string diff --git a/site/src/content/docs/docs/cli/corral.md b/site/src/content/docs/docs/cli/corral.md index 92afb003..ded16899 100644 --- a/site/src/content/docs/docs/cli/corral.md +++ b/site/src/content/docs/docs/cli/corral.md @@ -91,8 +91,10 @@ Usage: THE OPERATOR NAMED ON THAT RUN: this command writes no config, changes no default and staffs no seat — corral has no default models. - flags: --db (a pushed warehouse instead of the local - bug-catching ledger; unreachable REFUSES, never falls back) + flags: --db (a ledger directory or pushed warehouse + instead of this machine's ledger; unreachable REFUSES, never + falls back) --source ledger|scorecard (without --db: this + machine's ledger, or the old local bug-catching store) --seat --lang --min-runs n --json corral criticscore list list execution-checked test-critic findings still awaiting human adjudication — the local store certify --local writes, or a @@ -305,7 +307,7 @@ Usage of certify --local: -max-tokens int cap on model TOKENS for the whole run, input + output, every seat (0 = no cap; an agent seat that reports no token usage is refused, since the cap could not see it). Checked before each call and charged after it, so one in-flight call can overshoot by its own size. Once reached: a generator seat that has not run makes its file ungradable (executor-error naming the cap), a writer or critic seat is skipped and the file flagged as it is for a provider failure — the dev kill rate already measured stands. The cost line says the cap was reached and after how many calls. Corral has bounded mutants, shards and wall clock and never money; this is the money bound -models POOL - a comma-separated POOL of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool + a comma-separated POOL of models you name; corral proposes which of them seats the mutant-generator, test-writer and test-critic for this file, ranked by the recorded evidence for its language (--lang, or detected from --code), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, audits nothing and exits 0, so a pipeline must pass the seat flags itself. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string @@ -418,7 +420,7 @@ Usage of demo: -dir string where to write the demo project (default: a new temp dir, printed and kept) -models POOL - a comma-separated POOL of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused + a comma-separated POOL of models you name; certify --local proposes which of them seats the demo's mutant-generator, test-writer and test-critic, ranked by the recorded evidence for Go, prints that plan, and nothing runs until you accept it on a terminal (with no terminal it audits nothing and exits 0). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_) -writer-model anthropic/… @@ -438,7 +440,7 @@ Usage of doctor: -jail string sandbox backend (default: auto-detect). "container" needs CORRALAI_EXEC_IMAGE set to a toolchain image, e.g. CORRALAI_EXEC_IMAGE=python:3.12-bookworm -models POOL - a comma-separated POOL of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code) and checks those seats. doctor asks nothing and runs nothing (certify would run nothing until you accept the plan), and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused + a comma-separated POOL of models you name; doctor prints the lineup certify would propose from it for the goal-deriver, mutant-generator, test-writer and test-critic (ranked by the recorded evidence for --code's language, or across all languages without --code or when its language cannot be detected) and checks those seats. doctor asks nothing and spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. certify would run nothing until you accept the plan, and a pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused -mutant-model string the mutant-generator model whose credential to check -repo string @@ -983,7 +985,7 @@ flags of `corral review`: -max-bytes int how many bytes of the scope the reviewer is shown; files past the cap are listed by name and the review records them as unshown (default 200000) -models POOL - a comma-separated POOL of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits). A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool + a comma-separated POOL of models you name; corral proposes which of them seats the reviewer and the verifier, ranked by the recorded review evidence (all languages: a review is not ranked by language), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, reviews nothing and exits 0, so a pipeline must pass the seat flags itself. A pool is not a default: a seat you also name by its own flag is pinned (--verifier-model off pins no verifier), and a seat no member qualifies for is refused, never filled from outside the pool -no-ledger print the review and write no entry, in either ledger -push string @@ -1318,7 +1320,7 @@ Usage of certify --repo: -min-kill-rate string fail the scan (exit 1) if ANY audited file's kill rate is below this value (0.0-1.0 inclusive; a minimum, so a file exactly at the threshold passes). Opt-in: unset by default, so exit codes are unchanged unless this is given. Applies PER FILE, not to the aggregate — a well-tested file must not mask a weak one -models POOL - a comma-separated POOL of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it spans several), prints that plan, and nothing runs until you accept it on a terminal (with no terminal it prints the equivalent seat flags and exits; --dry-run prints the plan and asks nothing). A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool + a comma-separated POOL of models you name; corral proposes which of them seats the goal-deriver (unless --goals is given), mutant-generator, test-writer and test-critic for the whole scan, ranked by the recorded evidence for the scan's language (across all languages when it has none or several), prints that plan, and nothing runs until you accept it on a terminal. With no terminal it prints the equivalent seat flags, audits nothing and exits 0, so a pipeline must pass the seat flags itself; --dry-run prints the plan and asks nothing. A pool is not a default: a seat you also name by its own flag is pinned, and a seat no member qualifies for is refused, never filled from outside the pool -mutant-model anthropic/… model for the mutant-generator role — REQUIRED, corral has no default models. Takes a registry alias (.corral/models.json), a hosted model as vendor/model (anthropic/…, `google/…`, `openai/…` — a bare cloud name is refused, corral never guesses a vendor), a local model the OLLAMA_URL daemon lists, or an agent seat (`codex`, `claude-code`, `antigravity`, or your own CORRALAI_AGENT_). "off" leaves the seat empty, and is allowed only with --mutants-from, whose tool mutants are then the whole exam -mutants string From 24e5978e83d33bb58b7d08d437240cb09ff8279c Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 13:12:53 -0500 Subject: [PATCH 12/14] Bring the --models documents in line with the code as it now is, because the claims review found sentences that were false or would have become false: the recorded evidence line was said to be the printed one, the critic's provider preference was stated for a pinned writer it did not reach, the machine ledger was said to carry no critic adjudications when its rows do, and an entry without --models was said to carry no seating even where its cached verdict document does. agent-seats.md, AGENTS.md, the CHANGELOG entry and the README now say that pins are resolved and compared by the model each would run, that the prompt needs stdin and stdout to be terminals, that a run with no terminal audits nothing and exits 0 so a pipeline must pass the seat flags, that review --models refuses on a machine without five verdicts unless the verifier is pinned, that a model only a --local-endpoint placement serves cannot be a member, that the recorded SeatEvidence names the source without its directory and only beside an evidence seat, that a cached verdict keeps the seating it was measured under, that the --repo roster names the deriver whenever --goals is absent, that doctor still probes a --derive-model the operator names, and that a run's recorded time starts at the answer. The refusal text the CHANGELOG quotes is marked as an example, model-ranking.md says the proposer is the only code that decides a verdict seat from the ranking and that the shadow draw is separate, and the CHANGELOG records the corrected top-level usage. Co-Authored-By: Claude Fable 5.2 --- AGENTS.md | 37 ++++++++++------ CHANGELOG.md | 9 ++-- README.md | 3 +- docs/corral/agent-seats.md | 84 +++++++++++++++++++++++++----------- docs/design/model-ranking.md | 7 +-- 5 files changed, 96 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8acee8cb..10953efa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -312,23 +312,36 @@ checks none for any seat); a seat named by its own flag is a pin and wins; the m member with at least `modelrank.DefaultMinRuns` (5) observations in that seat's own unit; the goal-deriver takes the first member in the order the operator typed, and the critic, unless it has such a row, the first member that is not -the writer, preferring one from a different provider. A seat nothing +the writer, preferring one from a different provider. Pins are resolved by the +same resolver as the members (`resolvePins` in `models_pool.go`), so the hard +rule (critic is not writer, verifier is not reviewer) and the provider +preference compare resolved models, pins included; comparing the typed text let +`anthropic/claude-sonnet-5-5` and `claude-code:claude-sonnet-5-5` sit in both +seats. A seat nothing qualifies for is refused with the flag that would fill it, never filled from outside the pool and never by lowering the floor. The plan is printed and -nothing runs until the operator answers `y` or `yes` on a terminal (anything -else declines, exit 0); with no terminal it -prints the equivalent seat flags and exits 0. There is no `--yes`. `doctor` and -`certify --repo --dry-run` print the plan and ask nothing. The evidence is the -machine ledger only, with no fallback to another source. The proposer is +nothing runs until the operator answers `y` or `yes` with both stdin and stdout +on a terminal (anything else declines, exit 0); otherwise it prints the +equivalent seat flags and a last line saying nothing was audited, and exits 0, +so a pipeline must pass the seat flags itself. There is no `--yes`. `doctor` and +`certify --repo --dry-run` print the plan and ask nothing. The evidence is +the machine ledger only, with no fallback to another source. The proposer is `internal/autoseat` (a pure function) and it is the one door: `TestOnlyAutoseatDecidesASeatFromEvidence` fails any other file in `cmd/corral` -that imports it, and `models_pool.go` is the only caller. How each seat was +that imports it or calls the evidence readers or `modelrank.Rank` outside the +files it lists, `TestEveryDoorWithSeatFlagsRegistersModelsFlags` fails a door +that registers `--models` and never calls `seatFromPool` (or forwards it, for +`demo`), and `models_pool.go` is the only caller. How each seat was chosen is recorded on the entry as `SeatedBy` (`operator`, `evidence` or -`order`, keyed by role) beside `SeatEvidence` (what the ranking read), both -tier DECLARED; a seat filled by an accepted proposal is recorded as `evidence` -or `order`, never as `operator`, because a person pressing `y` did not choose -the model. The machine ledger carries no critic adjudications, so on this -evidence the critic is always seated by order. +`order`, keyed by role) beside `SeatEvidence` (what the ranking read, named +without the ledger's directory: the entry is signed and may be published), +both tier DECLARED; `SeatEvidence` is kept only while a seat the entry records +was seated by evidence (`autoseat.EvidenceFor`). A seat filled by an accepted +proposal is recorded as `evidence` or `order`, never as `operator`, because a +person pressing `y` did not choose the model. A run's clock and host sampler +restart when the operator accepts, so a recorded duration never includes the +plan being read. The ranking reads no critic adjudications from the machine +ledger, so the critic is always seated by order unless pinned. `corral models rank` itself still seats nothing. `--autoseat` (seat without asking) and exploration (seating below-floor members in the shadow seats) are NOT built; do not document them as features. diff --git a/CHANGELOG.md b/CHANGELOG.md index a4a5f079..4a618353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,10 +13,10 @@ history of any release, `git log v0.3.4..v0.3.5`. ### Added -- **`--models` proposes which of the models you name takes each seat, from this machine's ledger.** `certify --local`, `certify --repo`, `doctor`, `demo` and `review` take `--models a,b,c`, a pool written the way a seat flag takes a model (`vendor/model`, a registry alias, a local model, an agent seat). A pool is not a default: only a model you named on the run can be seated. A seat you also name by its own flag is pinned and keeps your value. The measured seats (mutant-generator, test-writer, and on `review` the reviewer and verifier) go to the pool member `corral models rank` puts first among those with at least 5 observations in the seat's own unit: runs for the generator, survivors attempted for the writer, claims checked for the reviewer, verdicts for the verifier; equal rates go to the member named first. The goal-deriver goes to the first member, and the test-critic to the first member that is not the writer, preferring one from a different provider. The critic would be seated by its adjudication record if it had 5 adjudications, but the machine ledger carries none, so today it is always seated by order. A bare agent seat (`codex`) is never ranked; `codex:` is. - - *What you see.* Before anything is spent, the plan: one row per seat with the model, how it was chosen (`operator`, `evidence` or `order`) and why, then the evidence line (the machine ledger, its scan and review counts, and the language the ranking was read for) and the range of grading rules it spans. On a terminal corral asks `run with these seats? [y/N]`; only `y` or `yes` runs, and anything else prints `not run; nothing was spent` and exits 0. With no terminal nothing runs: corral prints the plan and the lineup as seat flags (`to run this lineup: --mutant-model … --writer-model …`) and exits 0. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask nothing; the dry run checks no member's credential, as it checks none for any seat. `demo` hands the pool to `certify --local`, which proposes the seats for Go. Under `certify --repo --json` the plan goes to stderr. - - *What refuses, exit 2, nothing spent.* A pool with an empty member, a member named twice, two members that resolve to one model, or `off`; `--models` given twice; a member that cannot run (every member is resolved and credential-checked first, whether or not it would be seated); a measured seat no member has 5 observations for, shown in the plan as `REFUSED` with its fix (`no model in --models has 5+ survivors attempted as test-writer; pin --writer-model`); a critic that would be the writer, or a verifier that would be the reviewer, including the critic of a one-model pool (`--critic-model off` satisfies it); a machine ledger that cannot be read or does not verify; and on `certify --local` a file whose language cannot be detected (pass `--lang`). The floor is never lowered, no model outside the pool is used, and no other evidence source is read in place of the machine ledger. A machine whose ledger has no entries yet refuses every measured seat you did not pin. When every seat is pinned, `--models` reads no evidence and asks nothing. - - *What is recorded.* Each entry written with `--models` carries `SeatedBy`, how each seat was chosen keyed by seat, and `SeatEvidence`, the evidence line, when any seat was seated by evidence. Both are declared fields. A seat you accepted from the plan is recorded as `evidence` or `order`, never as `operator`. An entry written without `--models` carries neither, as before. +- **`--models` proposes which of the models you name takes each seat, from this machine's ledger.** `certify --local`, `certify --repo`, `doctor`, `demo` and `review` take `--models a,b,c`, a pool written the way a seat flag takes a model (`vendor/model`, a registry alias, a local model, an agent seat). A pool is not a default: only a model you named on the run can be seated. A seat you also name by its own flag is pinned and keeps your value. The measured seats (mutant-generator, test-writer, and on `review` the reviewer and verifier) go to the pool member `corral models rank` puts first among those with at least 5 observations in the seat's own unit: runs for the generator, survivors attempted for the writer, claims checked for the reviewer, verdicts for the verifier; equal rates go to the member named first. The goal-deriver goes to the first member, and the test-critic to the first member that is not the writer, preferring one from a different provider. A pin is resolved the way a member is before anything is proposed, so "not the writer" and the provider preference compare the models that would run, pins included. The critic would be seated by its adjudication record if it had 5 adjudications, but the ranking reads no critic adjudications from the machine ledger, so the critic is always seated by order unless pinned. On `review` the reviewer and the verifier are both measured seats, so `review --models` on a machine with fewer than 5 claims checked or 5 verdicts for every member refuses unless that seat is pinned (`--verifier-model off` is a pin). A bare agent seat (`codex`) is never ranked; `codex:` is. A model that only a `--local-endpoint` placement serves cannot be a member; name it with its own seat flag and placement instead. + - *What you see.* Before anything is spent, the plan: one row per seat with the model, how it was chosen (`operator`, `evidence` or `order`) and why, then the evidence line (the machine ledger, its scan and review counts, and the language the ranking was read for) and the range of grading rules it spans. When the evidence holds no observation at all, one more line says `no recorded evidence yet: every measured seat must be pinned until this machine has runs on its ledger`. When stdin and stdout are both terminals corral asks `run with these seats? [y/N]`; only `y` or `yes` runs, and anything else prints `not run; nothing was spent` and exits 0. Otherwise nothing runs: corral prints the plan, the lineup as seat flags (`to run this lineup: --mutant-model … --writer-model …`) and a last line saying nothing was audited, and exits 0. A pipeline that reads only the exit code passes without an audit, so a pipeline must pass the seat flags itself. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask nothing; the dry run checks no member's credential, as it checks none for any seat. `doctor` spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. `demo` hands the pool to `certify --local`, which proposes the seats for Go. Under `certify --repo --json` the plan goes to stderr. + - *What refuses, exit 2, nothing spent.* A pool with an empty member, a member named twice, two members that resolve to one model, or `off`; `--models` given twice; a member that cannot run (every member is resolved and credential-checked first, whether or not it would be seated); a pin no source serves; a measured seat no member has 5 observations for, shown in the plan as `REFUSED` with its fix (for example `no model in --models has 5+ survivors attempted as test-writer; pin --writer-model`); a critic that would be the writer's model, or a verifier that would be the reviewer's, pins included, including the critic of a one-model pool, with the flags that fix it (`--critic-model off` satisfies it); a machine ledger that cannot be read or does not verify; and on `certify --local` a file whose language cannot be detected (pass `--lang`). The floor is never lowered, no model outside the pool is used, and no other evidence source is read in place of the machine ledger. A machine whose ledger has no entries yet refuses every measured seat you did not pin. When every seat is pinned, `--models` reads no evidence and asks nothing. + - *What is recorded.* Each entry written with `--models` carries `SeatedBy`, how each seat was chosen keyed by seat, and `SeatEvidence`, the evidence line, when a seat the entry records was seated by evidence. Both are declared fields. `SeatEvidence` names the source without the ledger's directory (`the machine ledger (31 scan(s), 2 review(s)); language python`), because the entry is signed and may be published; the printed plan keeps the directory. A seat you accepted from the plan is recorded as `evidence` or `order`, never as `operator`. An entry written without `--models` carries neither at the scan and row level, as before. A file's embedded verdict served from the verdict cache keeps the seating it was measured under, which can come from another run. The run's `started_at`, `total_millis` and host-load figures begin when you accept the plan, so they never count the time spent reading it. - *Not in this release.* Seating a proposal without asking (`--autoseat`), and seating pool members below the floor in the shadow seats so that they gather evidence. - **`review --verifier-model off` names no verifier**, the word `--critic-model off` already uses on `certify`. The entry is the same as one written with no verifier flag. With `--models` it is a pin: without it, `--models` proposes a verifier. @@ -29,6 +29,7 @@ history of any release, `git log v0.3.4..v0.3.5`. ### Fixed +- **The top-level usage says what `corral models rank` reads.** It said `--db` reads a warehouse "instead of the local bug-catching ledger", which stopped being the default source in rc.34. It now says `--db` reads a ledger directory or warehouse instead of this machine's ledger, and names `--source ledger|scorecard`. - **`corral mcp` starts in a second agent session.** The findings server opened the local findings store read-write and held it for as long as it ran. DuckDB allows one read-write process per file, so the first session's server locked the store: every later session's server failed at start-up with `Could not set lock on file`, and `certify --local` could not record findings while any agent session stayed open. The server now opens the store read-only for each tool call and closes it before answering, so an idle server holds no lock. - *What still fails.* DuckDB does not let a reader share the file with a process that is writing it. A tool call made while `certify --local` or `corral criticscore confirm|refute` is writing fails, with an error that names the cause and says to retry. - *No store yet.* The server no longer creates the state directory or an empty store at start-up. `list_audit_findings` answers `total: 0` and nothing is created. diff --git a/README.md b/README.md index d9df0cd6..ffba1609 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,8 @@ go by the order you named the pool, because the ledger holds nothing that ranks them, the critic never to the writer's model and preferably to another provider. corral prints the lineup with the reason for each seat and runs nothing until you answer `y` or `yes` on a terminal. With no terminal -it prints the lineup as seat flags and exits 0; there is no `--yes`. `doctor` +it prints the lineup as seat flags, audits nothing and exits 0, so a pipeline +must pass the seat flags itself; there is no `--yes`. `doctor` and `certify --repo --dry-run` print it and ask nothing. A seat no member qualifies for is refused with the flag that would fill it, never filled from outside the pool. The entry records how each seat was chosen (`SeatedBy`) and diff --git a/docs/corral/agent-seats.md b/docs/corral/agent-seats.md index cc6065ae..770d038e 100644 --- a/docs/corral/agent-seats.md +++ b/docs/corral/agent-seats.md @@ -257,21 +257,36 @@ default either: corral seats only models you named on the run. How each seat is filled: - A seat you also name with its own flag (`--writer-model`, `--critic-model off` - and so on) is pinned, and the pin wins. + and so on) is pinned, and the pin wins. A pin is resolved the way a member + is, before anything is proposed, so the rules below compare the models that + would run and not the words you typed: a critic pinned as + `claude-code:claude-sonnet-5-5` beside a writer seated as + `anthropic/claude-sonnet-5-5` is one model in both seats, and is refused. A + pin no source serves is refused there, with the resolver's own message. - The measured seats (mutant-generator, test-writer, and on `review` the reviewer and verifier) go to the pool member that `corral models rank` puts first among those with at least 5 observations in that seat's own unit: runs for the generator (in the machine ledger a run is one scan), survivors attempted for the writer, claims checked for the reviewer, verdicts for the - verifier. Equal rates go to the member you named first. + verifier. Equal rates go to the member you named first. On `review` both + seats are measured, so on a machine whose ledger has fewer than 5 claims + checked or 5 verdicts for every member, `review --models` refuses unless + you pin the seat it cannot fill (`--verifier-model off` is a pin). - The goal-deriver (on `certify --repo` without `--goals`, and on `doctor`) is not measured, so it goes to the first member you named. The critic goes to - the first member that is not the writer, preferring one from a different - provider. A critic is measured only by human adjudication, which the machine - ledger does not carry, so on this evidence it is always seated this way. + the first member that is not the writer's model, preferring one from a + different provider than the writer, whether the writer was proposed or + pinned (an operator-defined agent declares no provider, so beside one + there is no preference to apply). A critic is measured only by human adjudication, and the ranking + reads no critic adjudications from the machine ledger, so the critic is + always seated this way unless pinned. - A bare agent seat (`codex`) names a tool, not a model, so nothing is ranked under it: it can take a seat by pin or by pool order, never by evidence. `codex:gpt-6` is ranked like any other model. +- A model that only a `--local-endpoint` placement serves cannot be a pool + member: a placement belongs to one seat, and a member is not yet any + seat's. Name it with its own seat flag and place it with `--local-endpoint`; + the pin is then resolved with the placement. Before anything is proposed, every member is resolved and its credential (or its CLI) checked, whether or not it will be seated. `certify --repo --dry-run` @@ -279,7 +294,9 @@ checks no member's credential, as it checks none for any seat. Then the plan is printed on stdout (on stderr under `certify --repo --json`), one row per seat: the seat, the model, how it was chosen (`operator`, `evidence` or `order`) and why, then what the ranking read and, when the evidence records them, the -grading rules it spans. An illustration of a `certify --repo` plan for a +grading rules it spans. When the evidence holds no observation at all, one +more line says so: `no recorded evidence yet: every measured seat must be +pinned until this machine has runs on its ledger`. An illustration of a `certify --repo` plan for a Python scan (the numbers are a test's fixture, not a measurement; `certify --local` and `demo` have no goal-deriver row): @@ -293,13 +310,18 @@ evidence: the machine ledger /state/ledger (31 scan(s), 2 review(s); a run is on grading rules: generations 40 to 43 across 6 scan(s) with evidence ``` -On a terminal corral then asks `run with these seats? [y/N]`. Only `y` or `yes` -runs; any other answer prints `not run; nothing was spent` and exits 0. With no -terminal (a pipe, a CI job) nothing runs: corral prints the plan and the same -lineup as seat flags (`to run this lineup: --mutant-model ... --writer-model ...`) -and exits 0, so you can run it deliberately. There is no `--yes`. `doctor` and -`certify --repo --dry-run` print the plan, say it is the lineup a real run -would propose, and ask nothing. +When stdin and stdout are both terminals, corral then asks +`run with these seats? [y/N]`. Only `y` or `yes` runs; any other answer prints +`not run; nothing was spent` and exits 0. Otherwise (a pipe, a CI job, or +stdout redirected to a file, where a `y` would accept a plan nobody saw) +nothing runs: corral prints the plan, the same lineup as seat flags +(`to run this lineup: --mutant-model ... --writer-model ...`) and a last line +saying nothing was audited, and exits 0. A pipeline that reads only the exit +code passes without an audit, so a pipeline must pass the seat flags itself, +not `--models`. There is no `--yes`. `doctor` and `certify --repo --dry-run` +print the plan, say it is the lineup a real run would propose, and ask +nothing. `doctor` spends nothing on a seat it proposes; a `--derive-model` you +name is still probed with one real call, as without `--models`. What refuses, before anything is spent (exit 2): @@ -313,7 +335,10 @@ What refuses, before anything is spent (exit 2): The floor is never lowered and no model outside the pool is used. A machine with no entries yet refuses every measured seat that is not pinned; - a critic or verifier that would be the same model as its writer or reviewer, - including a one-model pool's critic (`--critic-model off` satisfies it); + pins included and compared by the model each resolves to, including a + one-model pool's critic. The plan names the pair and the flags that fix it + (`--critic-model off` satisfies it); +- a pin no source serves; - a machine ledger that cannot be read or does not verify. Nothing falls back to another source; - on `certify --local`, a file whose language cannot be detected (pass `--lang`). @@ -322,18 +347,29 @@ When every seat is pinned, `--models` seats nothing, reads no evidence and asks nothing. On `review`, `--verifier-model off` pins no verifier. The ranking is read for the audited file's language on `certify --local` and -`demo` (Go), for the scan's language on `certify --repo` when it has exactly -one (otherwise across all languages), for `--code`'s language on -`doctor` (across all languages without it), and across all languages on -`review`. The evidence line says which. +`demo` (Go), for the scan's language on `certify --repo` when its walked +sources are in exactly one (across all languages when none is detected or +several are), for `--code`'s language on `doctor` (across all languages +without `--code` or when its language cannot be detected), and across all +languages on `review`. The evidence line says which, and why. The entry records how each seat was chosen as `SeatedBy`, keyed by seat, and, -when a seat was seated by evidence, what the ranking read as `SeatEvidence`; -both are declared fields -([record-fields.md](record-fields.md)). A seat you accepted from the plan is -recorded as `evidence` or `order`, not `operator`. On `certify --repo` without -`--goals` the entry's roster also names the goal-deriver, so its seating is -attributed to a model. `corral models rank` itself still seats nothing. +when a seat the entry records was seated by evidence, what the ranking read as +`SeatEvidence`; both are declared fields +([record-fields.md](record-fields.md)). `SeatEvidence` names the source +without the ledger's directory, for example +`the machine ledger (31 scan(s), 2 review(s)); language python`, because the +entry is signed and may be published and a home directory is not evidence; +the printed plan keeps the directory. A seat you accepted from the plan is +recorded as `evidence` or `order`, not `operator`. On `certify --repo` the +entry's roster also names the goal-deriver whenever `--goals` is absent, +including when every goal came from the goal cache, so its seating is +attributed to a model. A file's verdict served from the verdict cache keeps +the seating it was measured under, which can come from another run, because +the cache key does not name how a seat was chosen. A run's `started_at`, +`total_millis` and host-load figures begin when you accept the plan, so they +do not count the time spent reading it. `corral models rank` itself still +seats nothing. Not in this release: seating a proposal without asking (`--autoseat`), and seating members below the floor in the shadow seats to gather evidence on them. diff --git a/docs/design/model-ranking.md b/docs/design/model-ranking.md index e70b5bdb..0595ab21 100644 --- a/docs/design/model-ranking.md +++ b/docs/design/model-ranking.md @@ -116,14 +116,15 @@ that has already acted. There is one code path from the ranking to a seat, and it is built against the incident above. `--models a,b,c` on `certify --local`, `certify --repo`, `doctor`, `demo` and `review` hands the ranking, filtered to the pool, to -`internal/autoseat`, a pure function and the only code that decides a seat from -evidence (`TestOnlyAutoseatDecidesASeatFromEvidence` holds that). It can seat +`internal/autoseat`, a pure function and the only code that decides a verdict +seat from the ranking (in `cmd/corral`, `TestOnlyAutoseatDecidesASeatFromEvidence` +holds that); the shadow draw is separate. It can seat only a model the operator named in that run's pool, so a model that ranks well but was not named is never chosen; it uses this page's floor and never lowers it, refusing a seat no member clears; it prints the lineup with each seat's reason, and nothing runs until the operator accepts it on a terminal. The entry records how each seat was chosen (`SeatedBy`) and what the ranking read -(`SeatEvidence`). The router in the incident re-selected a model nobody had +(`SeatEvidence`, named without the ledger's directory). The router in the incident re-selected a model nobody had named on the run; the guard here is that the operator names the candidates every time. The operator-facing description is in [docs/corral/agent-seats.md](../corral/agent-seats.md). From 4d6c48bd8a3baf37b7b6ade33dfb51af38e8efd5 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 13:27:05 -0500 Subject: [PATCH 13/14] Leave the --models plan interval out of a run's recorded duration and host figures instead of restarting the clock at the answer, because a restarted clock made total_millis and started_at mean something different on a --models scan (a --repo scan stopped counting its walk), and two entries for one repository stopped being comparable. seatFromPool measures the interval once, from the evidence load to the operator's answer, returns it as poolSeating.PlanWait, and pauses the door's host sampler across exactly that interval. The certify doors keep started_at at the run's real start and subtract PlanWait from total_millis. The new hoststat Pause/Resume takes no load sample while paused and leaves the paused wall time and CPU out of WallMillis and CPUMillis, so LoadStart and MemAvailableStartBytes stay the readings at the run's start; a restart would have moved them to the answer. The test now checks that started_at is the real start, that exactly the plan interval is left out, and that work before the question is still counted. The documents and the ledger field comments say the time spent on the plan is excluded rather than that the clock starts at acceptance. Co-Authored-By: Claude Fable 5.2 --- AGENTS.md | 8 ++-- CHANGELOG.md | 2 +- cmd/corral/certify_local.go | 18 +++---- cmd/corral/certify_local_entry.go | 6 ++- cmd/corral/certify_repo.go | 26 ++++------ cmd/corral/models_consent_test.go | 60 ++++++++++++++--------- cmd/corral/models_pool.go | 30 ++++++++---- docs/corral/agent-seats.md | 10 ++-- internal/hoststat/hoststat.go | 76 +++++++++++++++++++++++++++--- internal/hoststat/hoststat_test.go | 55 +++++++++++++++++++++ internal/ledger/bundle.go | 10 +++- 11 files changed, 223 insertions(+), 78 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 10953efa..86b222f4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -338,9 +338,11 @@ without the ledger's directory: the entry is signed and may be published), both tier DECLARED; `SeatEvidence` is kept only while a seat the entry records was seated by evidence (`autoseat.EvidenceFor`). A seat filled by an accepted proposal is recorded as `evidence` or `order`, never as `operator`, because a -person pressing `y` did not choose the model. A run's clock and host sampler -restart when the operator accepts, so a recorded duration never includes the -plan being read. The ranking reads no critic adjudications from the machine +person pressing `y` did not choose the model. `seatFromPool` measures the plan +interval (evidence load to the answer) as `poolSeating.PlanWait` and pauses the +door's host sampler across it; the doors subtract it from `total_millis` and +keep `started_at` at the run's real start, so a `--models` entry's figures are +comparable with any other entry's. The ranking reads no critic adjudications from the machine ledger, so the critic is always seated by order unless pinned. `corral models rank` itself still seats nothing. `--autoseat` (seat without asking) and exploration (seating below-floor members in the shadow seats) are diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a618353..d0cb9d91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ history of any release, `git log v0.3.4..v0.3.5`. - **`--models` proposes which of the models you name takes each seat, from this machine's ledger.** `certify --local`, `certify --repo`, `doctor`, `demo` and `review` take `--models a,b,c`, a pool written the way a seat flag takes a model (`vendor/model`, a registry alias, a local model, an agent seat). A pool is not a default: only a model you named on the run can be seated. A seat you also name by its own flag is pinned and keeps your value. The measured seats (mutant-generator, test-writer, and on `review` the reviewer and verifier) go to the pool member `corral models rank` puts first among those with at least 5 observations in the seat's own unit: runs for the generator, survivors attempted for the writer, claims checked for the reviewer, verdicts for the verifier; equal rates go to the member named first. The goal-deriver goes to the first member, and the test-critic to the first member that is not the writer, preferring one from a different provider. A pin is resolved the way a member is before anything is proposed, so "not the writer" and the provider preference compare the models that would run, pins included. The critic would be seated by its adjudication record if it had 5 adjudications, but the ranking reads no critic adjudications from the machine ledger, so the critic is always seated by order unless pinned. On `review` the reviewer and the verifier are both measured seats, so `review --models` on a machine with fewer than 5 claims checked or 5 verdicts for every member refuses unless that seat is pinned (`--verifier-model off` is a pin). A bare agent seat (`codex`) is never ranked; `codex:` is. A model that only a `--local-endpoint` placement serves cannot be a member; name it with its own seat flag and placement instead. - *What you see.* Before anything is spent, the plan: one row per seat with the model, how it was chosen (`operator`, `evidence` or `order`) and why, then the evidence line (the machine ledger, its scan and review counts, and the language the ranking was read for) and the range of grading rules it spans. When the evidence holds no observation at all, one more line says `no recorded evidence yet: every measured seat must be pinned until this machine has runs on its ledger`. When stdin and stdout are both terminals corral asks `run with these seats? [y/N]`; only `y` or `yes` runs, and anything else prints `not run; nothing was spent` and exits 0. Otherwise nothing runs: corral prints the plan, the lineup as seat flags (`to run this lineup: --mutant-model … --writer-model …`) and a last line saying nothing was audited, and exits 0. A pipeline that reads only the exit code passes without an audit, so a pipeline must pass the seat flags itself. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask nothing; the dry run checks no member's credential, as it checks none for any seat. `doctor` spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. `demo` hands the pool to `certify --local`, which proposes the seats for Go. Under `certify --repo --json` the plan goes to stderr. - *What refuses, exit 2, nothing spent.* A pool with an empty member, a member named twice, two members that resolve to one model, or `off`; `--models` given twice; a member that cannot run (every member is resolved and credential-checked first, whether or not it would be seated); a pin no source serves; a measured seat no member has 5 observations for, shown in the plan as `REFUSED` with its fix (for example `no model in --models has 5+ survivors attempted as test-writer; pin --writer-model`); a critic that would be the writer's model, or a verifier that would be the reviewer's, pins included, including the critic of a one-model pool, with the flags that fix it (`--critic-model off` satisfies it); a machine ledger that cannot be read or does not verify; and on `certify --local` a file whose language cannot be detected (pass `--lang`). The floor is never lowered, no model outside the pool is used, and no other evidence source is read in place of the machine ledger. A machine whose ledger has no entries yet refuses every measured seat you did not pin. When every seat is pinned, `--models` reads no evidence and asks nothing. - - *What is recorded.* Each entry written with `--models` carries `SeatedBy`, how each seat was chosen keyed by seat, and `SeatEvidence`, the evidence line, when a seat the entry records was seated by evidence. Both are declared fields. `SeatEvidence` names the source without the ledger's directory (`the machine ledger (31 scan(s), 2 review(s)); language python`), because the entry is signed and may be published; the printed plan keeps the directory. A seat you accepted from the plan is recorded as `evidence` or `order`, never as `operator`. An entry written without `--models` carries neither at the scan and row level, as before. A file's embedded verdict served from the verdict cache keeps the seating it was measured under, which can come from another run. The run's `started_at`, `total_millis` and host-load figures begin when you accept the plan, so they never count the time spent reading it. + - *What is recorded.* Each entry written with `--models` carries `SeatedBy`, how each seat was chosen keyed by seat, and `SeatEvidence`, the evidence line, when a seat the entry records was seated by evidence. Both are declared fields. `SeatEvidence` names the source without the ledger's directory (`the machine ledger (31 scan(s), 2 review(s)); language python`), because the entry is signed and may be published; the printed plan keeps the directory. A seat you accepted from the plan is recorded as `evidence` or `order`, never as `operator`. An entry written without `--models` carries neither at the scan and row level, as before. A file's embedded verdict served from the verdict cache keeps the seating it was measured under, which can come from another run. The time spent on the plan (reading the evidence and waiting for your answer) is excluded from the run's `total_millis` and host-load figures, so they mean what they mean on a run without `--models`; `started_at` stays the run's real start, and the `certify --repo` walk before the question is still counted. - *Not in this release.* Seating a proposal without asking (`--autoseat`), and seating pool members below the floor in the shadow seats so that they gather evidence. - **`review --verifier-model off` names no verifier**, the word `--critic-model off` already uses on `certify`. The entry is the same as one written with no verifier flag. With `--models` it is a pin: without it, `--models` proposes a verifier. diff --git a/cmd/corral/certify_local.go b/cmd/corral/certify_local.go index b9314e53..0728e9b1 100644 --- a/cmd/corral/certify_local.go +++ b/cmd/corral/certify_local.go @@ -101,14 +101,13 @@ func runCertifyLocal(args []string, stdout, stderr io.Writer) int { // would file the demo's measurements under that project's origin and HEAD. func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Writer) int { // The run's start, for the ledger entry's started_at and total_millis. - // Restarted when the operator accepts a --models plan (below), so the - // record never counts the time someone spent reading it. + // A --models plan's interval is left out of total_millis and of the host + // figures (poolSeating.PlanWait), never by moving this start. startedAt := time.Now() // Host load and the run's own consumption, from here to the entry. The - // deferred Stop ends the sampler on every return (the one running then, - // since a --models acceptance replaces it); Stop is idempotent. + // deferred Stop ends the sampler on every return; Stop is idempotent. hostSampler := hoststat.Start(hostSampleInterval) - defer func() { hostSampler.Stop() }() + defer hostSampler.Stop() flagArgs, checkArgv := splitCertifyArgs(args) fs := flag.NewFlagSet("certify --local", flag.ContinueOnError) @@ -304,6 +303,7 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write // operator did type are pins. seatDeps := doorPoolDeps(stdout, stderr) seatDeps.placed = localEndpoints + seatDeps.sampler = hostSampler seating, poolExit, poolDone := seatFromPool(context.Background(), "corral certify --local", *repoDirFlag, drawLang, "", modelsPool, []poolRole{ {role: advpool.RoleMutantGenerator, val: mutantModel}, {role: advpool.RoleTestWriter, val: writerModel}, @@ -312,13 +312,6 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write if poolDone { return poolExit } - if seating.proposed() { - // The run starts when the operator accepts: the scan time and the - // host-load samples must not include a person reading the plan, or - // the evidence read behind it. What came before (flag checks, the - // language detection) is milliseconds and is not the run either. - startedAt, hostSampler = restartRunClock(hostSampler) - } // The bug-catching scorecard store: a --local run's per-seat metrics // (including the shadow head-to-head) persist next to the signed build // ledger and feed the same `corral scorecard` surface the daemon's runs feed. @@ -642,6 +635,7 @@ func runCertifyLocalAs(args []string, unattributed bool, stdout, stderr io.Write seedSkipped: seedSkippedJSON(seedSkipped), toolMutants: toolMutantsJSON(toolCounts), host: hostSampler.Stop(), + planWait: seating.PlanWait, }); berr != nil { fmt.Fprintf(stderr, "corral certify --local: no ledger entry written: %v\n", berr) } else { diff --git a/cmd/corral/certify_local_entry.go b/cmd/corral/certify_local_entry.go index 043877a7..2bfa7d65 100644 --- a/cmd/corral/certify_local_entry.go +++ b/cmd/corral/certify_local_entry.go @@ -59,6 +59,10 @@ type localRunFacts struct { repoIdentity string // host is what hoststat observed over the run (Sampler.Stop). host hoststat.Conditions + // planWait is the --models plan interval (poolSeating.PlanWait), left + // out of total_millis so the figure means what it means without + // --models; zero otherwise. + planWait time.Duration // sandbox is the jail the audit ran in (localSandboxIdentity). sandbox sandbox.Identity // seedSkipped is what the jail's copy left out as not text, as the @@ -166,7 +170,7 @@ func localEntryBundle(in localAuditInput, v advpool.Verdict, t advpool.RunTeleme StartedAt: started, FinishedAt: finished, Author: party.Author, Committer: party.Committer, CoAuthors: party.CoAuthors, GeneratedWith: party.GeneratedWith, CorralVersion: version, Host: host, Cores: runtime.NumCPU(), - TotalMillis: finished.Sub(started).Milliseconds(), + TotalMillis: (finished.Sub(started) - run.planWait).Milliseconds(), InputTokens: inTokens, OutputTokens: outTokens, ModelCalls: callCount, } // The test command and runner, and the settings --local has. It has no diff --git a/cmd/corral/certify_repo.go b/cmd/corral/certify_repo.go index 90d6158f..8eaae116 100644 --- a/cmd/corral/certify_repo.go +++ b/cmd/corral/certify_repo.go @@ -382,16 +382,16 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { // Captured here, before enumeration: this is the header row's // provenance timestamp for the whole invocation, not just the audit - // portion of it, except that a --models run restarts it when the - // operator accepts the plan (below). Read only if --record is given (see - // the fail-open call site near the bottom of this function). + // portion of it. A --models plan's interval is left out of total_millis + // and of the host figures (poolSeating.PlanWait), never by moving this + // start. Read only if --record is given (see the fail-open call site + // near the bottom of this function). startedAt := time.Now() // The host's load and the run's own consumption, sampled until the scan // header is built. The deferred Stop ends the sampler on every early - // return (the one running then, since a --models acceptance replaces - // it); Stop is idempotent, so the one before the header is the read. + // return; Stop is idempotent, so the one before the header is the read. hostSampler := hoststat.Start(hostSampleInterval) - defer func() { hostSampler.Stop() }() + defer hostSampler.Stop() var testMap *reposcan.TestMap if *testsPath != "" { @@ -477,6 +477,7 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { } seatDeps := doorPoolDeps(planOut, stderr) seatDeps.placed = localEndpoints + seatDeps.sampler = hostSampler if *dryRun { seatDeps.confirm, seatDeps.noCredentialCheck = acceptUnasked, true } @@ -484,15 +485,6 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { if poolDone { return poolExit } - if seating.proposed() && !*dryRun { - // The scan starts when the operator accepts: started_at, - // total_millis and the host-load samples must not include a person - // reading the plan, or the evidence read behind it. The walk above - // (which the plan needed for the scan's language) is excluded with - // them, and is the one part of the invocation a --models scan's - // total_millis does not cover. - startedAt, hostSampler = restartRunClock(hostSampler) - } if *dryRun && seating.proposed() { fmt.Fprintln(planOut, "a dry run asks nothing and runs nothing: this is the lineup a real run would propose") } @@ -1913,7 +1905,9 @@ func runCertifyRepo(args []string, stdout, stderr io.Writer) int { } return 0 }(), - TotalMillis: finishedAt.Sub(startedAt).Milliseconds(), + // Less the --models plan interval, so a scan with --models and + // one without measure the same thing (the walk included). + TotalMillis: (finishedAt.Sub(startedAt) - seating.PlanWait).Milliseconds(), // The instrumented coverage run, at the grain it HAPPENS at: once per // scan. Every file's verdict carries the same duration too (so a // per-file readout can name each phase of that file's audit), but diff --git a/cmd/corral/models_consent_test.go b/cmd/corral/models_consent_test.go index 2b7f764a..9afa7218 100644 --- a/cmd/corral/models_consent_test.go +++ b/cmd/corral/models_consent_test.go @@ -194,21 +194,33 @@ func TestDeclinedOrUnaskedReviewDoesNothing(t *testing.T) { } } -// Scan time and host load are measured from the moment the operator -// accepted, never from before the question (B1): a run's recorded duration -// is what the run took, not how long someone read the plan. -func TestAcceptedRunTimesFromTheAnswer(t *testing.T) { - slowAccept := func(t *testing.T) *time.Time { +// A --models scan's recorded time means what it means on any other scan: +// started_at is the run's real start, and total_millis is the run's wall +// clock less exactly the interval spent on the plan (reading the evidence and +// waiting for the operator's answer). Work the door does before the question +// (on --repo, the walk) is still counted, or two entries for one repository +// would stop being comparable (B1, corrected). +func TestRecordedTimeExcludesOnlyThePlan(t *testing.T) { + const before = 400 * time.Millisecond + const wait = 1500 * time.Millisecond + // stub: the door's seam constructor runs after the door's own work before + // the question, so sleeping in it stands in for a slow walk; the confirm + // sleeps for the operator. It reports when the run reached the question + // and how long the plan interval took from the evidence load to the answer. + stub := func(t *testing.T) (reached *time.Time, planned *time.Duration) { t.Helper() - at := new(time.Time) + reached, planned = new(time.Time), new(time.Duration) orig := doorPoolDeps t.Cleanup(func() { doorPoolDeps = orig }) doorPoolDeps = func(stdout, stderr io.Writer) poolDeps { + time.Sleep(before) + *reached = time.Now() + var loadedAt time.Time return poolDeps{ - load: func(rankSource) (rankEvidence, error) { return recordEvidence(), nil }, + load: func(rankSource) (rankEvidence, error) { loadedAt = time.Now(); return recordEvidence(), nil }, confirm: func(string) (bool, bool, error) { - time.Sleep(1500 * time.Millisecond) - *at = time.Now() + time.Sleep(wait) + *planned = time.Since(loadedAt) return true, true, nil }, stdout: stdout, stderr: stderr, @@ -217,41 +229,44 @@ func TestAcceptedRunTimesFromTheAnswer(t *testing.T) { origRun := shadowMemberRunnable t.Cleanup(func() { shadowMemberRunnable = origRun }) shadowMemberRunnable = func(agentbackend.ResolvedSeat) error { return nil } - return at + return reached, planned } - check := func(t *testing.T, e auditpush.LedgerEntry, accepted time.Time, finished time.Time) { + check := func(t *testing.T, e auditpush.LedgerEntry, reached time.Time, planned time.Duration) { t.Helper() s := e.Bundle.Scan - if s.StartedAt == nil || s.StartedAt.Before(accepted.Truncate(time.Microsecond)) { - t.Errorf("started_at %v is before the operator accepted at %v", s.StartedAt, accepted) + if s.StartedAt == nil || s.FinishedAt == nil || s.TotalMillis == nil { + t.Fatalf("started_at %v finished_at %v total_millis %v", s.StartedAt, s.FinishedAt, s.TotalMillis) } - if s.TotalMillis == nil { - t.Fatalf("no total_millis recorded") + if !s.StartedAt.Before(reached.Add(-before)) { + t.Errorf("started_at %v is not the run's real start (the door's work before the question began by %v)", s.StartedAt, reached.Add(-before)) } - if *s.TotalMillis > finished.Sub(accepted).Milliseconds() { - t.Errorf("total_millis %d includes the wait (the run after acceptance took at most %dms)", *s.TotalMillis, finished.Sub(accepted).Milliseconds()) + excluded := s.FinishedAt.Sub(*s.StartedAt).Milliseconds() - *s.TotalMillis + if excluded < wait.Milliseconds() || excluded > planned.Milliseconds()+50 { + t.Errorf("total_millis leaves out %dms; want the plan interval (%dms, at least the %dms wait) and nothing else", excluded, planned.Milliseconds(), wait.Milliseconds()) + } + if *s.TotalMillis < before.Milliseconds() { + t.Errorf("total_millis %d does not include the %v of work before the question", *s.TotalMillis, before) } } t.Run("certify --local", func(t *testing.T) { root, setPath, _ := localEntryFixture(t) - at := slowAccept(t) + reached, planned := stub(t) ml := withMachineLedger(t) var out, errb bytes.Buffer code := runCertifyLocal([]string{ "--repo-dir", root, "--code", "pkg/a.go", "--goal", "A returns 1", "--models", recordPool, "--mutants", setPath, "--quiet", "--", "go", "test", "./...", }, &out, &errb) - finished := time.Now() localRunFinished(t, code, out.String(), errb.String()) entries := scanEntriesIn(t, ml) if len(entries) != 1 { t.Fatalf("%d entries", len(entries)) } - check(t, entries[0], *at, finished) + check(t, entries[0], *reached, *planned) }) t.Run("certify --repo", func(t *testing.T) { root, setPath, goals := telemetryRepoFixture(t) - at := slowAccept(t) + reached, planned := stub(t) machine := withMachineLedger(t) noKeys(t) t.Setenv("ANTHROPIC_API_KEY", "test-placeholder-not-a-real-key") @@ -261,7 +276,6 @@ func TestAcceptedRunTimesFromTheAnswer(t *testing.T) { "--repo", root, "--models", recordPool, "--goals", goals, "--substrate", substrateWorkspace, "--all", "--mutants", setPath, }, &out, &errb) - finished := time.Now() if code != 0 { t.Fatalf("exit=%d\n%s%s", code, out.String(), errb.String()) } @@ -269,6 +283,6 @@ func TestAcceptedRunTimesFromTheAnswer(t *testing.T) { if err != nil || len(entries) != 1 { t.Fatalf("%d entries, %v", len(entries), err) } - check(t, entries[0], *at, finished) + check(t, entries[0], *reached, *planned) }) } diff --git a/cmd/corral/models_pool.go b/cmd/corral/models_pool.go index 07279af6..2647abca 100644 --- a/cmd/corral/models_pool.go +++ b/cmd/corral/models_pool.go @@ -54,6 +54,13 @@ type poolSeating struct { // host path (rankEvidence.RecordSource), then the language scope exactly // as the plan printed it. "" when no seat was seated by evidence. SeatEvidence string + // PlanWait is the interval seatFromPool spent on the plan: from reading + // the evidence to the operator's answer. A door subtracts it from the + // run's recorded duration, so total_millis means the same on a --models + // run as on any other; started_at stays the run's real start. Zero when + // no evidence was read (no --models, or every seat pinned). Not recorded: + // it is a correction to a recorded figure, not a measurement of the run. + PlanWait time.Duration } // seatEvidenceFor is the evidence line a record keeps beside seatedBy: the @@ -94,6 +101,10 @@ type poolDeps struct { confirm func(plan string) (accepted bool, interactive bool, err error) stdout io.Writer stderr io.Writer + // sampler is the door's host-load sampler, paused across the plan + // interval (PlanWait) so its figures leave out the same interval the + // duration does. nil where the door samples nothing (doctor, review). + sampler *hoststat.Sampler // placed is the door's --local-endpoint placements (role -> daemon // URL, as parseLocalEndpoints returns them), or nil where the door // takes none. A pin is resolved with them, exactly as the door's own @@ -216,6 +227,13 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an return refuse("%v", err) } + // The plan interval starts here: the evidence load, the plan and the + // question are not the run, so the duration a door records and its host + // sampler both leave them out. Measured once, here, so no door times it + // separately. + planStart := time.Now() + deps.sampler.Pause() + defer deps.sampler.Resume() ev, err := deps.load(rankSource{Kind: rankSourceLedger}) if err != nil { return refuse("cannot read the evidence: %v — nothing was proposed", err) @@ -238,6 +256,7 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an } accepted, interactive, err := deps.confirm(text) + planWait := time.Since(planStart) switch { case err != nil: return refuse("reading the answer: %v", err) @@ -259,6 +278,7 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an return poolSeating{}, 0, true } + seating.PlanWait = planWait seating.SeatedBy = map[string]string{} for i, s := range plan.Seats { seating.SeatedBy[s.Role] = s.By @@ -276,16 +296,6 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an return seating, 0, false } -// restartRunClock stops the sampler a door started before its --models -// question and starts the clock and a new sampler at the operator's answer. -// A door calls it only when a plan was accepted (poolSeating.proposed), so a -// run without --models, or with every seat pinned, is timed exactly as it -// always was. -func restartRunClock(old *hoststat.Sampler) (time.Time, *hoststat.Sampler) { - old.Stop() - return time.Now(), hoststat.Start(hostSampleInterval) -} - // noTerminalLine is the last line a --models run with no terminal prints. It // exits 0, as a decline does (spec 5.3), so a pipeline that only reads the // exit code goes green; this line is what a person reading the log sees. diff --git a/docs/corral/agent-seats.md b/docs/corral/agent-seats.md index 770d038e..d466f39e 100644 --- a/docs/corral/agent-seats.md +++ b/docs/corral/agent-seats.md @@ -366,10 +366,12 @@ entry's roster also names the goal-deriver whenever `--goals` is absent, including when every goal came from the goal cache, so its seating is attributed to a model. A file's verdict served from the verdict cache keeps the seating it was measured under, which can come from another run, because -the cache key does not name how a seat was chosen. A run's `started_at`, -`total_millis` and host-load figures begin when you accept the plan, so they -do not count the time spent reading it. `corral models rank` itself still -seats nothing. +the cache key does not name how a seat was chosen. The time spent on the +plan (reading the evidence and waiting for your answer) is excluded from the +run's `total_millis` and its host-load figures, so they mean what they mean +on a run without `--models`; `started_at` stays the run's real start, and on +`certify --repo` the repository walk before the question is still counted. +`corral models rank` itself still seats nothing. Not in this release: seating a proposal without asking (`--autoseat`), and seating members below the floor in the shadow seats to gather evidence on them. diff --git a/internal/hoststat/hoststat.go b/internal/hoststat/hoststat.go index ad682802..5c25f7ca 100644 --- a/internal/hoststat/hoststat.go +++ b/internal/hoststat/hoststat.go @@ -31,7 +31,8 @@ const procRoot = "/proc" type Conditions struct { // LoadStart, LoadEnd and LoadPeak are the 1-minute load average at Start, // at Stop and the largest of every sample taken between (the two ends - // included), so the peak is never below either end. + // included; no sample is taken while paused), so the peak is never below + // either end. LoadStart, LoadEnd, LoadPeak *float64 // MemAvailableStartBytes is /proc/meminfo's MemAvailable at Start. MemAvailableStartBytes *int64 @@ -40,9 +41,11 @@ type Conditions struct { // process-lifetime maximum, not a delta: the kernel offers no other. PeakRSSBytes *int64 // CPUMillis is user plus system CPU time of this process and its waited - // children consumed between Start and Stop. + // children consumed between Start and Stop, less any consumed between a + // Pause and its Resume. CPUMillis *int64 - // WallMillis is Stop minus Start. It needs no platform support. + // WallMillis is Stop minus Start, less every paused interval. It needs + // no platform support. WallMillis int64 // NotMeasured says, in a sentence, what could not be read and why; "" // when everything was. @@ -61,6 +64,15 @@ type Sampler struct { start Conditions peak *float64 notes []string + // paused is set between Pause and Resume; pausedAt and cpuAtPause are + // the readings at Pause. excludedWall and excludedCPU accumulate every + // paused interval, which finish subtracts. + paused bool + pausedAt time.Time + cpuAtPause int64 + cpuPauseOK bool + excludedWall time.Duration + excludedCPU int64 quit chan struct{} done chan struct{} @@ -159,8 +171,57 @@ func firstOf(ctx context.Context, quit <-chan struct{}) <-chan struct{} { return c } -// sample reads the load once and raises the peak. +// Pause stops sampling until Resume, and leaves the wall and CPU time spent +// in between out of what Stop reports. It exists for an interval that is not +// the run: a --models plan being read by the operator. Restarting the sampler +// there would move LoadStart and MemAvailableStartBytes to the answer and drop +// the CPU spent before the question, so the fields would mean something else +// on a run with --models than on one without; a pause keeps every field's +// meaning and removes only the interval. A no-op on a nil sampler and when +// already paused. +func (s *Sampler) Pause() { + if s == nil { + return + } + cpu, _, err := usage() + s.mu.Lock() + defer s.mu.Unlock() + if s.paused { + return + } + s.paused, s.pausedAt = true, time.Now() + s.cpuAtPause, s.cpuPauseOK = cpu, err == nil +} + +// Resume ends a Pause, adding its interval to what finish subtracts. The CPU +// spent while paused is subtracted only when it was read at both ends; when +// it could not be, nothing is subtracted and NotMeasured would already say +// the CPU reading failed. A no-op on a nil sampler and when not paused. +func (s *Sampler) Resume() { + if s == nil { + return + } + cpu, _, err := usage() + s.mu.Lock() + defer s.mu.Unlock() + if !s.paused { + return + } + s.paused = false + s.excludedWall += time.Since(s.pausedAt) + if s.cpuPauseOK && err == nil { + s.excludedCPU += cpu - s.cpuAtPause + } +} + +// sample reads the load once and raises the peak, unless paused. func (s *Sampler) sample() { + s.mu.Lock() + paused := s.paused + s.mu.Unlock() + if paused { + return + } v, err := readLoad(s.root) s.mu.Lock() defer s.mu.Unlock() @@ -187,7 +248,10 @@ func (s *Sampler) Stop() Conditions { } func (s *Sampler) finish() Conditions { - out := Conditions{WallMillis: time.Since(s.begin).Milliseconds()} + // A Stop while paused (a door returning from inside the paused + // interval) closes the interval first, so it is left out as well. + s.Resume() + out := Conditions{WallMillis: (time.Since(s.begin) - s.excludedWall).Milliseconds()} if !supported { out.NotMeasured = strings.Join(s.notes, "; ") return out @@ -209,7 +273,7 @@ func (s *Sampler) finish() Conditions { s.note(err) } else { if s.cpuOK { - d := cpu - s.cpu0 + d := cpu - s.cpu0 - s.excludedCPU out.CPUMillis = &d } out.PeakRSSBytes = &rss diff --git a/internal/hoststat/hoststat_test.go b/internal/hoststat/hoststat_test.go index 7bec1ee1..6045c23c 100644 --- a/internal/hoststat/hoststat_test.go +++ b/internal/hoststat/hoststat_test.go @@ -135,3 +135,58 @@ func TestOtherPlatformsSayNotMeasured(t *testing.T) { t.Errorf("%+v", c) } } + +// A paused sampler takes no load sample, and the CPU and wall time spent +// while paused are left out of what Stop reports, so a run that paused for +// someone to read a plan reports what a run without the pause reports. +// LoadStart and MemAvailableStartBytes stay the readings at Start. +func TestPausedIntervalIsNotSampledOrCounted(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the readers are Linux's") + } + root := t.TempDir() + write := func(name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + write("loadavg", "1.00 1.00 1.00 1/1 1\n") + write("meminfo", "MemAvailable: 1024 kB\n") + s := startAt(context.Background(), root, 5*time.Millisecond) + s.Pause() + write("loadavg", "99.00 1.00 1.00 1/1 1\n") + spinCPU(t, 300*time.Millisecond) + time.Sleep(200 * time.Millisecond) + write("loadavg", "2.00 1.00 1.00 1/1 1\n") + s.Resume() + time.Sleep(30 * time.Millisecond) + c := s.Stop() + if c.LoadStart == nil || *c.LoadStart != 1 { + t.Errorf("LoadStart = %v, want the reading at Start (1)", c.LoadStart) + } + if c.LoadPeak == nil || *c.LoadPeak != 2 { + t.Errorf("LoadPeak = %v, want 2: the 99 written during the pause must not be sampled", c.LoadPeak) + } + if c.CPUMillis == nil || *c.CPUMillis >= 200 { + t.Errorf("CPUMillis = %v, want the 300ms spent while paused left out", c.CPUMillis) + } + if c.WallMillis >= 300 { + t.Errorf("WallMillis = %d, want the paused interval left out", c.WallMillis) + } +} + +// Pause and Resume are safe on a nil sampler (a door with none) and are +// no-ops when repeated. +func TestPauseResumeAreSafe(t *testing.T) { + var none *Sampler + none.Pause() + none.Resume() + s := Start(5 * time.Millisecond) + s.Resume() + s.Pause() + s.Pause() + s.Resume() + s.Resume() + _ = s.Stop() +} diff --git a/internal/ledger/bundle.go b/internal/ledger/bundle.go index e70c38ec..a98e13a0 100644 --- a/internal/ledger/bundle.go +++ b/internal/ledger/bundle.go @@ -69,8 +69,12 @@ type ScanRow struct { // push time, so re-pushing an old scan made its superseded verdict // current. nil for rows written before this column existed. StartedAt *time.Time - // TotalMillis is the scan's own wall clock. *int64 so a run that did not - // time itself is NULL rather than a scan that took no time. + // TotalMillis is the scan's own wall clock, less the time spent on a + // --models plan (reading its evidence and waiting for the operator's + // answer), so it measures the same thing with and without --models; on + // such a scan it is therefore shorter than FinishedAt minus StartedAt. + // *int64 so a run that did not time itself is NULL rather than a scan + // that took no time. TotalMillis *int64 // SelectionMillis is the scan's ONE instrumented coverage run — the pass // that decides which tests execute which file. It belongs at the SCAN @@ -221,6 +225,8 @@ type ScanRow struct { // MemAvailable at the start; PeakRSSBytes the children's largest resident // set (getrusage RUSAGE_CHILDREN, a process-lifetime maximum); CPUMillis // the user plus system CPU of the process and its children over the run. + // A --models plan's interval is left out of the samples and of CPUMillis + // (hoststat's Pause), as it is out of TotalMillis. // A load average decides how much of the evidence bound a run can spend, // so two runs at load 38 and load 13 are not the same run. nil, never 0, // where a figure could not be read; HostNotMeasured says why. All absent From e0b7729a8aaa0b9c3f59c99599d6732790818d88 Mon Sep 17 00:00:00 2001 From: Peter D Bethke Date: Thu, 19 Nov 2026 13:35:03 -0500 Subject: [PATCH 14/14] Make four --models statements exact, because the re-review found them true in most cases and false in one: the critic's provider preference was said to be absent beside any operator-defined agent, the evidence-reader gate was said to cover readers it did not hold, the engine's dependency on the proposer's package was unexplained, and the no-terminal line said "audited" on corral review, which audits nothing. agent-seats.md now gives an agent seat's provider as the code computes it: the vendor of a pinned model the model-name table recognises, else a built-in agent's declared maker, and none for any other agent seat. TestOnlyAutoseatDecidesASeatFromEvidence now also holds scorecardRankEvidence, bugcatchRankEvidence and warehouseRankEvidence to models_rank.go, and AGENTS.md names exactly what the gate holds. The verdict's call to autoseat.EvidenceFor says why the engine imports the proposer's package: the rule has one definition, shared with the doors. The no-terminal line now reads "nothing was run", and the AGENTS.md, CHANGELOG and agent-seats.md sentences that quote it follow. Co-Authored-By: Claude Fable 5.2 --- AGENTS.md | 7 ++++--- CHANGELOG.md | 2 +- cmd/corral/models_flags_test.go | 5 +++++ cmd/corral/models_pool.go | 2 +- docs/corral/agent-seats.md | 11 ++++++++--- internal/advpool/driver.go | 4 ++++ 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 86b222f4..8e18c72f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -322,14 +322,15 @@ qualifies for is refused with the flag that would fill it, never filled from outside the pool and never by lowering the floor. The plan is printed and nothing runs until the operator answers `y` or `yes` with both stdin and stdout on a terminal (anything else declines, exit 0); otherwise it prints the -equivalent seat flags and a last line saying nothing was audited, and exits 0, +equivalent seat flags and a last line saying nothing was run, and exits 0, so a pipeline must pass the seat flags itself. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan and ask nothing. The evidence is the machine ledger only, with no fallback to another source. The proposer is `internal/autoseat` (a pure function) and it is the one door: `TestOnlyAutoseatDecidesASeatFromEvidence` fails any other file in `cmd/corral` -that imports it or calls the evidence readers or `modelrank.Rank` outside the -files it lists, `TestEveryDoorWithSeatFlagsRegistersModelsFlags` fails a door +that imports it, or that uses `modelrank.Rank` or an evidence reader +(`defaultRankLoader`, `machineLedgerRankEvidence`, `scorecardRankEvidence`, +`bugcatchRankEvidence`, `warehouseRankEvidence`) outside the files it lists, `TestEveryDoorWithSeatFlagsRegistersModelsFlags` fails a door that registers `--models` and never calls `seatFromPool` (or forwards it, for `demo`), and `models_pool.go` is the only caller. How each seat was chosen is recorded on the entry as `SeatedBy` (`operator`, `evidence` or diff --git a/CHANGELOG.md b/CHANGELOG.md index d0cb9d91..8cdf81e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ history of any release, `git log v0.3.4..v0.3.5`. ### Added - **`--models` proposes which of the models you name takes each seat, from this machine's ledger.** `certify --local`, `certify --repo`, `doctor`, `demo` and `review` take `--models a,b,c`, a pool written the way a seat flag takes a model (`vendor/model`, a registry alias, a local model, an agent seat). A pool is not a default: only a model you named on the run can be seated. A seat you also name by its own flag is pinned and keeps your value. The measured seats (mutant-generator, test-writer, and on `review` the reviewer and verifier) go to the pool member `corral models rank` puts first among those with at least 5 observations in the seat's own unit: runs for the generator, survivors attempted for the writer, claims checked for the reviewer, verdicts for the verifier; equal rates go to the member named first. The goal-deriver goes to the first member, and the test-critic to the first member that is not the writer, preferring one from a different provider. A pin is resolved the way a member is before anything is proposed, so "not the writer" and the provider preference compare the models that would run, pins included. The critic would be seated by its adjudication record if it had 5 adjudications, but the ranking reads no critic adjudications from the machine ledger, so the critic is always seated by order unless pinned. On `review` the reviewer and the verifier are both measured seats, so `review --models` on a machine with fewer than 5 claims checked or 5 verdicts for every member refuses unless that seat is pinned (`--verifier-model off` is a pin). A bare agent seat (`codex`) is never ranked; `codex:` is. A model that only a `--local-endpoint` placement serves cannot be a member; name it with its own seat flag and placement instead. - - *What you see.* Before anything is spent, the plan: one row per seat with the model, how it was chosen (`operator`, `evidence` or `order`) and why, then the evidence line (the machine ledger, its scan and review counts, and the language the ranking was read for) and the range of grading rules it spans. When the evidence holds no observation at all, one more line says `no recorded evidence yet: every measured seat must be pinned until this machine has runs on its ledger`. When stdin and stdout are both terminals corral asks `run with these seats? [y/N]`; only `y` or `yes` runs, and anything else prints `not run; nothing was spent` and exits 0. Otherwise nothing runs: corral prints the plan, the lineup as seat flags (`to run this lineup: --mutant-model … --writer-model …`) and a last line saying nothing was audited, and exits 0. A pipeline that reads only the exit code passes without an audit, so a pipeline must pass the seat flags itself. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask nothing; the dry run checks no member's credential, as it checks none for any seat. `doctor` spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. `demo` hands the pool to `certify --local`, which proposes the seats for Go. Under `certify --repo --json` the plan goes to stderr. + - *What you see.* Before anything is spent, the plan: one row per seat with the model, how it was chosen (`operator`, `evidence` or `order`) and why, then the evidence line (the machine ledger, its scan and review counts, and the language the ranking was read for) and the range of grading rules it spans. When the evidence holds no observation at all, one more line says `no recorded evidence yet: every measured seat must be pinned until this machine has runs on its ledger`. When stdin and stdout are both terminals corral asks `run with these seats? [y/N]`; only `y` or `yes` runs, and anything else prints `not run; nothing was spent` and exits 0. Otherwise nothing runs: corral prints the plan, the lineup as seat flags (`to run this lineup: --mutant-model … --writer-model …`) and a last line saying nothing was run, and exits 0. A pipeline that reads only the exit code passes without an audit, so a pipeline must pass the seat flags itself. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask nothing; the dry run checks no member's credential, as it checks none for any seat. `doctor` spends nothing on a seat it proposes; a `--derive-model` you name is still probed with one real call. `demo` hands the pool to `certify --local`, which proposes the seats for Go. Under `certify --repo --json` the plan goes to stderr. - *What refuses, exit 2, nothing spent.* A pool with an empty member, a member named twice, two members that resolve to one model, or `off`; `--models` given twice; a member that cannot run (every member is resolved and credential-checked first, whether or not it would be seated); a pin no source serves; a measured seat no member has 5 observations for, shown in the plan as `REFUSED` with its fix (for example `no model in --models has 5+ survivors attempted as test-writer; pin --writer-model`); a critic that would be the writer's model, or a verifier that would be the reviewer's, pins included, including the critic of a one-model pool, with the flags that fix it (`--critic-model off` satisfies it); a machine ledger that cannot be read or does not verify; and on `certify --local` a file whose language cannot be detected (pass `--lang`). The floor is never lowered, no model outside the pool is used, and no other evidence source is read in place of the machine ledger. A machine whose ledger has no entries yet refuses every measured seat you did not pin. When every seat is pinned, `--models` reads no evidence and asks nothing. - *What is recorded.* Each entry written with `--models` carries `SeatedBy`, how each seat was chosen keyed by seat, and `SeatEvidence`, the evidence line, when a seat the entry records was seated by evidence. Both are declared fields. `SeatEvidence` names the source without the ledger's directory (`the machine ledger (31 scan(s), 2 review(s)); language python`), because the entry is signed and may be published; the printed plan keeps the directory. A seat you accepted from the plan is recorded as `evidence` or `order`, never as `operator`. An entry written without `--models` carries neither at the scan and row level, as before. A file's embedded verdict served from the verdict cache keeps the seating it was measured under, which can come from another run. The time spent on the plan (reading the evidence and waiting for your answer) is excluded from the run's `total_millis` and host-load figures, so they mean what they mean on a run without `--models`; `started_at` stays the run's real start, and the `certify --repo` walk before the question is still counted. - *Not in this release.* Seating a proposal without asking (`--autoseat`), and seating pool members below the floor in the shadow seats so that they gather evidence. diff --git a/cmd/corral/models_flags_test.go b/cmd/corral/models_flags_test.go index 8aa93d77..a89ce5e5 100644 --- a/cmd/corral/models_flags_test.go +++ b/cmd/corral/models_flags_test.go @@ -100,6 +100,11 @@ func TestOnlyAutoseatDecidesASeatFromEvidence(t *testing.T) { // models_pool.go ranks for the proposer (rankPool); models_rank.go // ranks for the disclosure verb. "modelrank.Rank(": {"models_pool.go": true, "models_rank.go": true}, + // The other readers serve only the disclosure verb's other sources + // (--source scorecard, --db), which --models never reads. + "scorecardRankEvidence(": {"models_rank.go": true}, + "bugcatchRankEvidence(": {"models_rank.go": true}, + "warehouseRankEvidence(": {"models_rank.go": true}, } files, err := filepath.Glob("*.go") if err != nil || len(files) == 0 { diff --git a/cmd/corral/models_pool.go b/cmd/corral/models_pool.go index 2647abca..a65f5d87 100644 --- a/cmd/corral/models_pool.go +++ b/cmd/corral/models_pool.go @@ -299,7 +299,7 @@ func seatFromPool(ctx context.Context, cmdName, repoRoot, lang string, anyWhy an // noTerminalLine is the last line a --models run with no terminal prints. It // exits 0, as a decline does (spec 5.3), so a pipeline that only reads the // exit code goes green; this line is what a person reading the log sees. -const noTerminalLine = "nothing was audited: --models without a terminal only prints the plan and exits 0; pass the seat flags above to run this lineup" +const noTerminalLine = "nothing was run: --models without a terminal only prints the plan and exits 0; pass the seat flags above to run this lineup" // resolvePins resolves every pin but "off" with the resolver the members // use, sharing their daemon listing, and with the door's --local-endpoint diff --git a/docs/corral/agent-seats.md b/docs/corral/agent-seats.md index d466f39e..9c761c31 100644 --- a/docs/corral/agent-seats.md +++ b/docs/corral/agent-seats.md @@ -276,8 +276,13 @@ How each seat is filled: not measured, so it goes to the first member you named. The critic goes to the first member that is not the writer's model, preferring one from a different provider than the writer, whether the writer was proposed or - pinned (an operator-defined agent declares no provider, so beside one - there is no preference to apply). A critic is measured only by human adjudication, and the ranking + pinned. An agent seat's provider is the vendor of its pinned model when + corral's model-name table recognises that model (so `myagent:claude-…` + has one), else, for a built-in agent whose definition you have not + replaced with `CORRALAI_AGENT_`, the maker declared on it. Any other + agent seat (one you defined, or a built-in you redefined) that is unpinned + or pinned to a model the table does not recognise has no known provider, + and beside it there is no preference to apply. A critic is measured only by human adjudication, and the ranking reads no critic adjudications from the machine ledger, so the critic is always seated this way unless pinned. - A bare agent seat (`codex`) names a tool, not a model, so nothing is ranked @@ -316,7 +321,7 @@ When stdin and stdout are both terminals, corral then asks stdout redirected to a file, where a `y` would accept a plan nobody saw) nothing runs: corral prints the plan, the same lineup as seat flags (`to run this lineup: --mutant-model ... --writer-model ...`) and a last line -saying nothing was audited, and exits 0. A pipeline that reads only the exit +saying nothing was run, and exits 0. A pipeline that reads only the exit code passes without an audit, so a pipeline must pass the seat flags itself, not `--models`. There is no `--yes`. `doctor` and `certify --repo --dry-run` print the plan, say it is the lineup a real run would propose, and ask diff --git a/internal/advpool/driver.go b/internal/advpool/driver.go index 7c5c5bb5..16c3bf15 100644 --- a/internal/advpool/driver.go +++ b/internal/advpool/driver.go @@ -3967,6 +3967,10 @@ func (r *runState) signedSeats(assign RoleAssignment) signedRoster { seatedBy: seatedBy, // The evidence line explains an evidence seat, so it goes with the // last one the narrowing keeps, as the selection goes with its seat. + // The engine imports the proposer's package for this one rule + // because the rule has one definition, autoseat.EvidenceFor, used + // here for the verdict and by the doors for the --repo row and the + // review entry; a second copy here could disagree with theirs. evidence: autoseat.EvidenceFor(seatedBy, r.rs.SeatEvidence), } }