From 3e4d5ab1138d53d47a0f34b19fb04df0f6aee6d7 Mon Sep 17 00:00:00 2001 From: ShocOne <62835948+ShocOne@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:27:39 +0100 Subject: [PATCH] feat: a blocked entity says why on disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An observation says what a run learned about one property, and a request body says what a create looked like when it worked. Neither says why an entity produced nothing: that sentence reached only the operator's terminal, so an entity blocked by a refusal nobody was watching for left no trace to act on. The run summary is now committed at audit/summary.json, beside the observations and the request bodies the same run wrote, and is written whatever the run did — a run where every entity blocked is exactly the run whose reasons are worth keeping. Each entity result carries the redacted refusal behind a status that is not audited, so the API's own words survive the run. A minimal create that the additive search could not heal now reports the search's own last refusal rather than the earlier one that started it, and names the fields it added: the status alone does not distinguish a document that understates the create from an API refusing it for another reason. The aggregate counts stay as they were. docs/emittance_tracker.md remains the only place a count of what the toolkit emits or refuses may live. Co-Authored-By: Claude Opus 5 (1M context) --- docs/glossary.md | 1 + internal/audit/run/adaptive_test.go | 10 +++ internal/audit/run/adjust.go | 4 ++ internal/audit/run/entity.go | 2 +- internal/audit/run/run.go | 8 +++ internal/audit/run/steps_create.go | 52 ++++++++++++--- internal/audit/run/summary_file.go | 47 ++++++++++++++ internal/audit/run/summary_file_test.go | 86 +++++++++++++++++++++++++ internal/cli/audit.go | 16 ++++- 9 files changed, 215 insertions(+), 11 deletions(-) create mode 100644 internal/audit/run/summary_file.go create mode 100644 internal/audit/run/summary_file_test.go diff --git a/docs/glossary.md b/docs/glossary.md index 36a4a4e..2678851 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -11,6 +11,7 @@ a binder that drafts a call or a loader that merges `allOf` is prose. |---|---| | **audit** | The credentialed stage that exercises a live API to learn its true behaviour — minimum and maximum valid configuration, field dependencies, value-conditional rules. `tfpfgen audit run`. The only stage that touches a network. | | **observation** | One recorded finding of an audit: what the live API actually accepted or rejected, with a redacted request/response excerpt as proof. Committed per entity in `audit/observations/.observations.json`, stamped with the spec hash it was observed against. Deliberately not replayable. | +| **audit summary** | How far a run got with each entity: its status, the reason it stopped, and the redacted refusal behind that reason, committed at `audit/summary.json` beside the observations and the request bodies. An observation says what a run learned about one property and a request body what a create looked like when it worked; neither says why an entity produced nothing, and an entity blocked by a refusal nobody watched for otherwise leaves no trace to act on. Written whatever the run did, so a run where every entity blocked still records why. The aggregate counts it also carries are a fact about one run against one document — `docs/emittance_tracker.md` remains the only place a count of what the toolkit emits or refuses may live. | | **request bodies** | The create request bodies a run got the API to accept, committed per entity in `audit/request_bodies/.request_bodies.json` with the status each was answered and the response it was answered with. An observation says something about one property; these say what a whole create looked like when it worked, which is the one thing a generated acceptance test cannot derive — the document describes what should be accepted, and only a run knows what was. Acceptance fixtures replay these values rather than deriving them again. Named for the request half deliberately: the response is carried as evidence of what the API echoed, and it is the request that a configuration has to reproduce. | | **correction** | One committed correction to the imported OpenAPI document: RFC 6902 operations plus a required justification and an optional evidence pointer to an observation. Lives in `spec/corrections/`; proposed ones await a human in `spec/corrections/proposed/`; rejected ones leave a marker in `spec/corrections/rejected/`. Kinds listed in config `audit.auto_accept` skip `proposed/` and land accepted directly, named with an `auto-NNN-` prefix. | | **revise** | To fold observations into proposed corrections and apply accepted ones — `tfpfgen spec revise`. The spec is revised based on audit observations; the output is the revised spec (`spec/revised.yaml`), the single source of truth for all generation. | diff --git a/internal/audit/run/adaptive_test.go b/internal/audit/run/adaptive_test.go index 57246a0..74fe6a4 100644 --- a/internal/audit/run/adaptive_test.go +++ b/internal/audit/run/adaptive_test.go @@ -449,6 +449,16 @@ func TestUnit_Adaptive_UnintelligibleRefusalIsBoundedAndContinues(t *testing.T) if got := entityStatus(t, sum, "gadget"); got.Status != StatusAudited { t.Fatalf("gadget = %+v, want audited after the blocked entity", got) } + // The block carries the API's own words and names what the search asked + // for: a status alone cannot be acted on, and an entity that produced no + // request body leaves no other trace of either. + blocked := entityStatus(t, sum, "thing") + if blocked.Refusal == nil || blocked.Refusal.Status != 400 { + t.Errorf("thing = %+v, want the refusal behind the block", blocked) + } + if !strings.Contains(blocked.Reason, "refused with status 400") { + t.Errorf("reason = %q, want the refusing status", blocked.Reason) + } // Bounded: the unintelligible create was not retried into the ground. if spent := s.Requests() - before; spent > 20 { t.Errorf("the run spent %d requests; an unhealable create must not spin the loop", spent) diff --git a/internal/audit/run/adjust.go b/internal/audit/run/adjust.go index 402cb39..93f98dd 100644 --- a/internal/audit/run/adjust.go +++ b/internal/audit/run/adjust.go @@ -64,6 +64,10 @@ type adjustResult struct { // that failed with adjusted true was partly built and then stuck, which is // a weaker signal about the body it started from. adjusted bool + // tried names the fields the additive search added, in the order it added + // them. Set only where the search ran and failed: it is what the block + // reason needs to say more than that a status came back. + tried []string // gaveUp reports that the loop ended without success: the refusal could // not be classified into an action, an action made no progress, a borrow // found nothing, or the iteration bound was hit. diff --git a/internal/audit/run/entity.go b/internal/audit/run/entity.go index f263b85..94436dc 100644 --- a/internal/audit/run/entity.go +++ b/internal/audit/run/entity.go @@ -74,7 +74,7 @@ func (r *runner) runEntity(ctx context.Context, ep *plan.EntityPlan) { } r.summary.RequestBodies = append(r.summary.RequestBodies, recordedRequestBodies(ep.Entity, ent)) r.summary.Entities = append(r.summary.Entities, EntityResult{ - Entity: ep.Entity, Status: ent.status, Reason: ent.reason, + Entity: ep.Entity, Status: ent.status, Reason: ent.reason, Refusal: ent.cause, }) r.log.Info().Str("entity", ep.Entity).Str("status", ent.status).Str("reason", ent.reason).Int("requests", ent.requests).Msg("entity finished") } diff --git a/internal/audit/run/run.go b/internal/audit/run/run.go index eabefee..ef8f892 100644 --- a/internal/audit/run/run.go +++ b/internal/audit/run/run.go @@ -143,6 +143,14 @@ type EntityResult struct { Status string `json:"status"` // Reason is set when the status is not audited. Reason string `json:"reason,omitempty"` + // Refusal is the redacted request and response behind a status that is + // not audited, carrying the API's own words for why it stopped. + // + // The reason names the step and the status; only the excerpt says what + // the API objected to, and an entity that produced no request body + // leaves no other trace of it. Absent where nothing was refused — a + // budget exhausted, a path parameter with no value. + Refusal *observe.Excerpt `json:"refusal,omitempty"` } // Summary is what a run did, for the operator's table. diff --git a/internal/audit/run/steps_create.go b/internal/audit/run/steps_create.go index 8999573..90dd1f2 100644 --- a/internal/audit/run/steps_create.go +++ b/internal/audit/run/steps_create.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strings" "time" "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/audit/observe" @@ -27,11 +28,20 @@ func (r *runner) runCreateMinimal(ctx context.Context, ent *entityState, step *p // only that the request was bad names nothing to act on, and the document // that produced this body is the same document that understated it — so // ask the API instead, one field at a time. + // A failed search still learned something: its last refusal answers a + // wider body than the one that started it, and the fields it added name + // what it asked for. Both are carried for the block reason alone — the + // grammar's own result still decides what happens next. + var searched adjustResult if rr.obj == nil && rr.res != nil && rr.res.refused() { - if searched, serr := r.searchMinimal(ctx, ent, ent.recipe, rr.body, rr.res); serr != nil { + found, serr := r.searchMinimal(ctx, ent, ent.recipe, rr.body, rr.res) + if serr != nil { return serr - } else if searched.obj != nil { - rr = searched + } + if found.obj != nil { + rr = found + } else { + searched = found } } if rr.obj != nil { @@ -66,13 +76,37 @@ func (r *runner) runCreateMinimal(ctx context.Context, ent *entityState, step *p // the other variants and probes it can still run, so continue instead. return nil } - if rr.res != nil { - ent.cause = &rr.res.excerpt - return blockedError{reason: fmt.Sprintf("the minimal create was refused with status %d", rr.res.status)} + if refused := lastRefusal(rr, searched); refused != nil { + ent.cause = &refused.excerpt + return blockedError{reason: minimalRefusedReason(refused.status, searched.tried)} } return blockedError{reason: "the minimal create produced no object"} } +// lastRefusal answers the later-informed of the two refusals a minimal create +// can end on: the additive search's, when it ran, and the grammar's otherwise. +func lastRefusal(grammar, searched adjustResult) *httpResult { + if searched.res != nil { + return searched.res + } + return grammar.res +} + +// minimalRefusedReason spells why an entity produced no object, naming the +// fields the search asked the API for. +// +// The status alone cannot be acted on. An operator reading a blocked entity +// needs to know whether the document's body was refused as written or whether +// a search widened it and was refused anyway, and which fields it widened it +// with. +func minimalRefusedReason(status int, tried []string) string { + if len(tried) == 0 { + return fmt.Sprintf("the minimal create was refused with status %d", status) + } + return fmt.Sprintf("the minimal create was refused with status %d, and adding %s did not heal it", + status, strings.Join(tried, ", ")) +} + // runCreateMaximal creates with every writable field populated, refining the // body against a 4xx: a field the API says is not valid for this variant is // removed and the create retried, feeding the validWhen evidence. Accepted, it @@ -357,9 +391,11 @@ func (r *runner) searchMinimal(ctx context.Context, ent *entityState, rec *entit candidates := r.searchCandidates(ent, body, refusal) allowance := searchAllowance(len(candidates)) last := refusal + var tried []string for i := 0; i < allowance; i++ { field := candidates[i] + tried = append(tried, field) body[field] = r.synthField(ent, field) obj, res, err := r.createObject(ctx, ent, rec, body) @@ -377,7 +413,7 @@ func (r *runner) searchMinimal(ctx context.Context, ent *entityState, rec *entit return adjustResult{obj: obj, res: res, body: body, adjusted: true}, nil } if res == nil || !res.refused() { - return adjustResult{res: res, body: body, adjusted: true, gaveUp: true}, nil + return adjustResult{res: res, body: body, adjusted: true, gaveUp: true, tried: tried}, nil } // The API now objects to the field just added, so it is not one this // create wants; the ones before it stay. @@ -386,7 +422,7 @@ func (r *runner) searchMinimal(ctx context.Context, ent *entityState, rec *entit } last = res } - return adjustResult{res: last, body: body, adjusted: true, gaveUp: true}, nil + return adjustResult{res: last, body: body, adjusted: true, gaveUp: true, tried: tried}, nil } // searchCandidates orders the fields the search may add, cheapest-signal diff --git a/internal/audit/run/summary_file.go b/internal/audit/run/summary_file.go new file mode 100644 index 0000000..6850b65 --- /dev/null +++ b/internal/audit/run/summary_file.go @@ -0,0 +1,47 @@ +// The committed record of how far a run got with each entity. + +package run + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +// SummaryFile is the committed file naming, one per run, written beside the +// observations and the request bodies the same run wrote — audit/summary.json +// wherever --out leaves the observations at its default. +const SummaryFile = "summary.json" + +// WriteSummary commits sum to path, creating the directory that holds it. +// +// An observation says what a run learned about one property, and a request +// body says what a create looked like when it worked. Neither says why an +// entity produced nothing: that sentence — the step that stopped, and the +// API's own words for why — reaches only the operator's terminal, so an +// entity blocked by a refusal nobody was watching for leaves no trace to act +// on afterwards. Committed rather than kept, so a newly blocked entity is a +// line in a generation pull request instead of a line in a log nobody reads. +// +// The encoding matches the observations beside it: sorted map keys, no HTML +// escaping, two-space indent. +func WriteSummary(path string, sum Summary) error { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + if err := enc.Encode(sum); err != nil { + return fmt.Errorf("encoding the run summary: %w", err) + } + if dir := filepath.Dir(path); dir != "." { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating %s: %w", dir, err) + } + } + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + return fmt.Errorf("writing %s: %w", path, err) + } + return nil +} diff --git a/internal/audit/run/summary_file_test.go b/internal/audit/run/summary_file_test.go new file mode 100644 index 0000000..28bec73 --- /dev/null +++ b/internal/audit/run/summary_file_test.go @@ -0,0 +1,86 @@ +package run + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/deploymenttheory/terraform-plugin-framework-codegen/internal/audit/observe" +) + +func TestUnit_Run_SummaryKeepsWhyAnEntityProducedNothing(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "audit", SummaryFile) + sum := Summary{ + RunID: "abcd1234", + Blocked: 1, + Entities: []EntityResult{{ + Entity: "thing", + Status: StatusBlocked, + Reason: "the minimal create was refused with status 400, and adding name did not heal it", + Refusal: &observe.Excerpt{ + Method: "POST", PathTemplate: "/things", Status: 400, + ResponseFragment: json.RawMessage(`{"detail":"field serial is required"}`), + }, + }}, + } + + if err := WriteSummary(path, sum); err != nil { + t.Fatalf("WriteSummary: %v", err) + } + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the directory holding the summary was not created: %v", err) + } + // The API's own words are the part an operator cannot get anywhere else: + // an entity that produced no request body leaves no other trace of them. + if !strings.Contains(string(raw), "field serial is required") { + t.Errorf("the refusal the API answered was not kept:\n%s", raw) + } + var back Summary + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatalf("the summary does not read back: %v", err) + } + if len(back.Entities) != 1 || back.Entities[0].Refusal == nil { + t.Fatalf("the refusal did not round-trip: %+v", back.Entities) + } + if back.Entities[0].Refusal.Status != 400 || back.Entities[0].Reason != sum.Entities[0].Reason { + t.Errorf("entity = %+v, want the status and reason as written", back.Entities[0]) + } +} + +func TestUnit_Run_SummaryIsWrittenWhenEveryEntityBlocked(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), SummaryFile) + + // A run that learned nothing is exactly the run whose reasons are worth + // keeping, so an empty entity list is still a file. + if err := WriteSummary(path, Summary{RunID: "none"}); err != nil { + t.Fatalf("WriteSummary: %v", err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("no summary was written: %v", err) + } +} + +func TestUnit_Run_ABlockedCreateNamesWhatTheSearchAskedFor(t *testing.T) { + t.Parallel() + + // Nothing to add — the document named no field the search could try — so + // the status is the whole of what is known. + if got := minimalRefusedReason(400, nil); got != "the minimal create was refused with status 400" { + t.Errorf("reason = %q, want the status alone", got) + } + // The search widened the body and was refused anyway. Which fields it + // widened it with is the difference between a document that understates + // the create and an API refusing it for another reason entirely. + got := minimalRefusedReason(400, []string{"serial", "kind"}) + want := "the minimal create was refused with status 400, and adding serial, kind did not heal it" + if got != want { + t.Errorf("reason = %q, want %q", got, want) + } +} diff --git a/internal/cli/audit.go b/internal/cli/audit.go index ef8cb21..389bdd3 100644 --- a/internal/cli/audit.go +++ b/internal/cli/audit.go @@ -39,8 +39,8 @@ const auditRunsDir = "audit/runs" // newAuditRunCommand executes the derived plan against the live API: the // only verb that touches a network with credentials. Observations land in -// --out, one file per entity, and a summary table says how far the run -// got. +// --out, one file per entity, the accepted request bodies and the run +// summary beside them, and a summary table says how far the run got. func newAuditRunCommand() *cobra.Command { var ( dir string @@ -112,6 +112,18 @@ func newAuditRunCommand() *cobra.Command { return writeErr } } + // The summary sits beside them both: the observations say what + // was learned and the request bodies what was accepted, and this + // says why an entity produced neither. Written whatever the run + // did, because a run where every entity blocked is exactly the + // run whose reasons are worth keeping. + summaryPath := filepath.Join(filepath.Dir(out), auditrun.SummaryFile) + if writeErr := auditrun.WriteSummary(summaryPath, sum); writeErr != nil { + if runErr != nil { + return fmt.Errorf("%v; additionally %w", runErr, writeErr) + } + return writeErr + } printSummary(cmd.OutOrStdout(), out, len(obs), sum) return runErr },