feat: an acceptance test replays a request the API took

internal/audit/observe/bodies.go

@@ -0,0 +1,141 @@
+// The request bodies a run got the API to accept, and what it answered with.
+//
+// An observation says something about one property. These say what a whole
+// create looked like when it worked, which is the one thing a generated
+// acceptance test cannot derive: the document describes what should be
+// accepted, and only a run knows what was.
+
+package observe
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// encodeBodies renders one entity's record deterministically, matching the
+// observations beside it: sorted map keys, no HTML escaping, two-space indent.
+func encodeBodies(b Bodies) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(b); err != nil {
+		return nil, fmt.Errorf("encoding bodies for %s: %w", b.Entity, err)
+	}
+	return buf.Bytes(), nil
+}
+
+// BodiesSuffix is the committed file naming, one file per entity, matching
+// the observations beside it.
+const BodiesSuffix = ".bodies.json"
+
+// Bodies is what one entity's creates looked like when the API accepted them.
+type Bodies struct {
+	Entity string `json:"entity"`
+	// Minimal is the smallest create the run got accepted, and Maximal the
+	// fullest. Either may be absent when no create of that shape succeeded.
+	Minimal *AcceptedBody `json:"minimal,omitempty"`
+	Maximal *AcceptedBody `json:"maximal,omitempty"`
+}
+
+// AcceptedBody is one create the API answered 2xx to.
+type AcceptedBody struct {
+	// Status is the code the API answered, kept so a reader can see that this
+	// was an acceptance rather than an assumption.
+	Status int `json:"status"`
+	// Request is the body as sent, with the run's own placeholders already
+	// resolved — what a repeat of this create would have to carry.
+	Request map[string]any `json:"request"`
+	// Response is the object the API answered with. A property the request
+	// carried and this does not is one the API accepts and never returns,
+	// which terraform cannot hold in state without losing it on the next read.
+	Response map[string]any `json:"response,omitempty"`
+}
+
+// Echoed reports whether the response carried the named wire property.
+//
+// A field the API accepts and never echoes cannot appear in a generated
+// configuration: terraform compares what it planned against what the provider
+// answers, and a value that never comes back reads as the provider losing it.
+func (b *AcceptedBody) Echoed(wire string) bool {
+	if b == nil || b.Response == nil {
+		return false
+	}
+	_, ok := b.Response[wire]
+	return ok
+}
+
+// WriteBodies commits one <entity>.bodies.json per entity under dir.
+// Encoding matches the observations: sorted keys, stable bytes, so a re-run
+// that learned nothing new rewrites nothing.
+func WriteBodies(dir string, bodies []Bodies) error {
+	if len(bodies) == 0 {
+		return nil
+	}
+	byEntity := map[string]Bodies{}
+	for _, b := range bodies {
+		if b.Entity == "" || (b.Minimal == nil && b.Maximal == nil) {
+			continue
+		}
+		byEntity[b.Entity] = b
+	}
+	entities := make([]string, 0, len(byEntity))
+	for e := range byEntity {
+		entities = append(entities, e)
+	}
+	sort.Strings(entities)
+
+	encoded := make(map[string][]byte, len(entities))
+	for _, entity := range entities {
+		raw, err := encodeBodies(byEntity[entity])
+		if err != nil {
+			return err
+		}
+		encoded[entity] = raw
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return fmt.Errorf("creating %s: %w", dir, err)
+	}
+	for _, entity := range entities {
+		path := filepath.Join(dir, entity+BodiesSuffix)
+		if err := os.WriteFile(path, encoded[entity], 0o644); err != nil {
+			return fmt.Errorf("writing %s: %w", path, err)
+		}
+	}
+	return nil
+}
+
+// ReadBodies loads every recorded body under dir, keyed by entity. A missing
+// directory is not an error: an entity the probe never cleared has none, and
+// generation falls back to deriving values from the document.
+func ReadBodies(dir string) (map[string]Bodies, error) {
+	out := map[string]Bodies{}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return out, nil
+		}
+		return nil, fmt.Errorf("reading %s: %w", dir, err)
+	}
+	for _, e := range entries {
+		if e.IsDir() || filepath.Ext(e.Name()) != ".json" {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(dir, e.Name()))
+		if err != nil {
+			return nil, fmt.Errorf("reading %s: %w", e.Name(), err)
+		}
+		var b Bodies
+		if err := json.Unmarshal(raw, &b); err != nil {
+			return nil, fmt.Errorf("reading %s: %w", e.Name(), err)
+		}
+		if b.Entity != "" {
+			out[b.Entity] = b
+		}
+	}
+	return out, nil
+}

internal/audit/observe/bodies_test.go

@@ -0,0 +1,88 @@
+package observe
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+// TestUnit_Observe_RecordedBodiesRoundTrip proves a recorded body survives the
+// trip to disk unchanged. A generated configuration is built from these, so a
+// value that shifts in the file is a configuration that no longer matches the
+// request the API accepted.
+func TestUnit_Observe_RecordedBodiesRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	in := []Bodies{
+		{
+			Entity: "tag",
+			Minimal: &AcceptedBody{
+				Status:   201,
+				Request:  map[string]any{"key": "branch", "value": "sfo"},
+				Response: map[string]any{"key": "branch", "value": "sfo", "id": "7"},
+			},
+		},
+		{
+			Entity:  "role",
+			Maximal: &AcceptedBody{Status: 200, Request: map[string]any{"name": "n"}},
+		},
+		// Neither shape accepted: nothing to record, and nothing written.
+		{Entity: "user"},
+	}
+	if err := WriteBodies(dir, in); err != nil {
+		t.Fatalf("WriteBodies: %v", err)
+	}
+
+	out, err := ReadBodies(dir)
+	if err != nil {
+		t.Fatalf("ReadBodies: %v", err)
+	}
+	if len(out) != 2 {
+		t.Fatalf("read %d entities, want the two that had an accepted create", len(out))
+	}
+	tag := out["tag"]
+	if tag.Minimal == nil || tag.Minimal.Status != 201 {
+		t.Fatalf("tag minimal = %#v", tag.Minimal)
+	}
+	if tag.Minimal.Request["value"] != "sfo" {
+		t.Errorf("request value = %#v, want the value that was sent", tag.Minimal.Request["value"])
+	}
+	if _, recorded := out["user"]; recorded {
+		t.Error("an entity with no accepted create was written")
+	}
+	if got := filepath.Base(dir); got == "" {
+		t.Fatal("temp dir vanished")
+	}
+}
+
+// TestUnit_Observe_EchoedReadsTheResponse pins the check a configuration
+// depends on: a property the API took and never returned cannot be held in
+// terraform state.
+func TestUnit_Observe_EchoedReadsTheResponse(t *testing.T) {
+	b := &AcceptedBody{
+		Request:  map[string]any{"name": "n", "matchType": "and"},
+		Response: map[string]any{"name": "n"},
+	}
+	if !b.Echoed("name") {
+		t.Error("a property the response carried reads as not echoed")
+	}
+	if b.Echoed("matchType") {
+		t.Error("a property the response omitted reads as echoed")
+	}
+	// No response recorded says nothing about any property.
+	var none *AcceptedBody
+	if none.Echoed("name") {
+		t.Error("an absent record claimed an echo")
+	}
+}
+
+// TestUnit_Observe_ReadBodiesToleratesNoRun proves a tree the probe has never
+// run against reads as empty rather than as an error: generation falls back to
+// deriving values from the document.
+func TestUnit_Observe_ReadBodiesToleratesNoRun(t *testing.T) {
+	out, err := ReadBodies(filepath.Join(t.TempDir(), "never-written"))
+	if err != nil {
+		t.Fatalf("a missing directory is a normal state: %v", err)
+	}
+	if len(out) != 0 {
+		t.Errorf("read %d entities from nothing", len(out))
+	}
+}

internal/audit/run/adjust_unit_test.go

@@ -322,3 +322,81 @@ func TestUnit_Evidence_TheIdentifyingPropertyIsFoundByValue(t *testing.T) {
 		})
 	}
 }
+
+// TestUnit_Search_CandidatesAreOrderedCheapestSignalFirst pins the order the
+// additive search adds fields in. The order decides how many live creates a
+// blocked entity costs, and it must be the same on a re-run.
+func TestUnit_Search_CandidatesAreOrderedCheapestSignalFirst(t *testing.T) {
+	t.Parallel()
+
+	r := &runner{hints: map[string]map[string]strategy.SynthHint{
+		"widget": {
+			"alreadySent": {Field: "alreadySent", Type: "string"},
+			"zNamed":      {Field: "zNamed", Type: "string"},
+			"aPlain":      {Field: "aPlain", Type: "string"},
+			"bPlain":      {Field: "bPlain", Type: "string"},
+			"withEnum":    {Field: "withEnum", Type: "string", Enum: []any{"x"}},
+			"nested":      {Field: "nested", Type: "object"},
+		},
+	}}
+	ent := &entityState{plan: &plan.EntityPlan{Entity: "widget"}}
+	body := map[string]any{"alreadySent": "v"}
+	refusal := &httpResult{body: []byte(`{"detail":"zNamed is wrong somehow"}`)}
+
+	got := r.searchCandidates(ent, body, refusal)
+	want := []string{"zNamed", "withEnum", "aPlain", "bPlain", "nested"}
+	if len(got) != len(want) {
+		t.Fatalf("candidates = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("candidates = %v, want %v", got, want)
+		}
+	}
+	// A field the body already carries is never a candidate.
+	for _, f := range got {
+		if f == "alreadySent" {
+			t.Error("a field already in the body was offered as a candidate")
+		}
+	}
+}
+
+// TestUnit_Search_AllowanceIsBounded pins the ceiling on how many live creates
+// one entity's search may spend.
+func TestUnit_Search_AllowanceIsBounded(t *testing.T) {
+	t.Parallel()
+	if got := searchAllowance(3); got != 3 {
+		t.Errorf("searchAllowance(3) = %d, want 3", got)
+	}
+	if got := searchAllowance(500); got != 24 {
+		t.Errorf("searchAllowance(500) = %d, want the cap", got)
+	}
+}
+
+// TestUnit_Search_MaximalCulpritPrefersTheNamedField pins which field the
+// reduction drops next. A refusal that names one is believed; otherwise the
+// choice is the last in order, so a re-run reduces the same way.
+func TestUnit_Search_MaximalCulpritPrefersTheNamedField(t *testing.T) {
+	t.Parallel()
+
+	r := &runner{}
+	body := map[string]any{"name": "n", "colour": "c", "shape": "s"}
+	minimal := map[string]any{"name": "n"}
+
+	named := &httpResult{body: []byte(`{"detail":"colour is not valid here"}`)}
+	if got := r.maximalCulprit(body, minimal, named); got != "colour" {
+		t.Errorf("culprit = %q, want the field the refusal named", got)
+	}
+
+	// Nothing named: the last optional field in order, never a field the
+	// minimal create needs.
+	silent := &httpResult{body: []byte(`{"detail":"bad request"}`)}
+	if got := r.maximalCulprit(body, minimal, silent); got != "shape" {
+		t.Errorf("culprit = %q, want the last optional field", got)
+	}
+
+	// Only the minimal body left: there is nothing safe to drop.
+	if got := r.maximalCulprit(minimal, minimal, silent); got != "" {
+		t.Errorf("culprit = %q, want none", got)
+	}
+}

internal/audit/run/entity.go

@@ -72,12 +72,34 @@ func (r *runner) runEntity(ctx context.Context, ep *plan.EntityPlan) {
 		ConditionalValues:  ent.ev.conditionalValues,
 		IdentifierProperty: ent.ev.idField,
 	}
+	r.summary.Bodies = append(r.summary.Bodies, recordedBodies(ep.Entity, ent))
 	r.summary.Entities = append(r.summary.Entities, EntityResult{
 		Entity: ep.Entity, Status: ent.status, Reason: ent.reason,
 	})
 	r.log.Info().Str("entity", ep.Entity).Str("status", ent.status).Str("reason", ent.reason).Int("requests", ent.requests).Msg("entity finished")
 }
 
+// recordedBodies is what this entity's accepted creates looked like, for the
+// generated acceptance tests to be built from.
+//
+// A create the API refused is not here: the point of the record is that these
+// are requests it took, so a configuration replaying one is a configuration
+// known to apply.
+func recordedBodies(entity string, ent *entityState) observe.Bodies {
+	out := observe.Bodies{Entity: entity}
+	if ent.ev.sent != nil {
+		out.Minimal = &observe.AcceptedBody{
+			Status: ent.ev.sentStatus, Request: ent.ev.sent, Response: ent.ev.got,
+		}
+	}
+	if ent.ev.maximalSent != nil {
+		out.Maximal = &observe.AcceptedBody{
+			Status: ent.ev.maximalStatus, Request: ent.ev.maximalSent, Response: ent.ev.maximalGot,
+		}
+	}
+	return out
+}
+
 // halt classifies a step failure onto the entity.
 func (r *runner) halt(ent *entityState, err error) {
 	var blocked blockedError

internal/audit/run/evidence.go

@@ -24,6 +24,10 @@ type evidence struct {
 	// optional fields a minimal create never sends.
 	maximalSent map[string]any
 	maximalGot  map[string]any
+	// The status each accepted create answered, kept so the recorded body
+	// shows it was an acceptance rather than an assumption.
+	sentStatus    int
+	maximalStatus int
 	// volatile marks fields the consecutive read saw change.
 	volatile map[string]bool
 	// omitted collects, per field, the value each created object answered

internal/audit/run/run.go

@@ -155,6 +155,10 @@ type Summary struct {
 	Blocked  int `json:"blocked"`
 	TimedOut int `json:"timedOut"`
 	Skipped  int `json:"skipped"`
+	// Bodies is what each entity's accepted creates looked like. A generated
+	// acceptance test is built from these rather than from values derived
+	// again from the document, because only these were actually accepted.
+	Bodies []observe.Bodies `json:"-"`
 	// SkippedEntities is every entity the plan left out, with its reason.
 	// Carried because the count alone cannot be acted on: it does not
 	// distinguish a run that covered the API from one that skipped most of it.