From 0aeb5d841edeb2518013992d3144aa5ead7b8084 Mon Sep 17 00:00:00 2001 From: ShocOne <62835948+ShocOne@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:33:05 +0100 Subject: [PATCH] fix: an operator-supplied value reaches the wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit audit/inputs.json declares three keys and only two of them did anything to a live run. Derive resolves parentRefs and skip at plan time, but Run replaces every entity's steps with the program its compiled strategy describes, and those bodies are synthesised from schema hints alone: the values key never left the plan. Options carries the inputs, strategize hands each entity its own, and both paths that put a value in a body now read them first — the skeleton the translator sends, and the field the adjustment loop is forced to add live. A field the API demands mid-loop is the same field the operator supplied a value for, and synthesising a different one there would send two values for one field across a single run. An operator value outranks the variant gate as well as the example: a discriminator is one of the things an operator has to supply when the document does not say which shape is valid. It substitutes rather than adds, scoped to the body's own fields, exactly as the plan's own synthesis does. Co-Authored-By: Claude Opus 5 (1M context) --- internal/audit/plan/inputs.go | 10 +++++ internal/audit/run/adaptive_test.go | 33 ++++++++++++++++ internal/audit/run/adjust_unit_test.go | 33 ++++++++++++++++ internal/audit/run/run.go | 16 ++++++-- internal/audit/run/strategize.go | 52 +++++++++++++++++++++----- internal/cli/audit.go | 33 +++++++++------- 6 files changed, 151 insertions(+), 26 deletions(-) diff --git a/internal/audit/plan/inputs.go b/internal/audit/plan/inputs.go index 8bc5ec4..db68d9a 100644 --- a/internal/audit/plan/inputs.go +++ b/internal/audit/plan/inputs.go @@ -110,6 +110,16 @@ func parseEntityInputs(entity string, raw json.RawMessage) (EntityInputs, error) // forEntity is the lookup Derive uses: absent entities read as the zero // value, per the graceful-degradation contract. +// ValuesFor answers the operator's value overrides for one entity, keyed by +// wire property name, and nil where none were supplied. +// +// The audit cannot invent a value the API will take for every field: a +// reachable endpoint, an existing agent's id, the discriminator a polymorphic +// body is keyed on. These are what the operator supplies in their place. +func (in *Inputs) ValuesFor(entity string) map[string]any { + return in.forEntity(entity).Values +} + func (in *Inputs) forEntity(entity string) EntityInputs { if in == nil || in.Entities == nil { return EntityInputs{} diff --git a/internal/audit/run/adaptive_test.go b/internal/audit/run/adaptive_test.go index 57246a0..e4a08ce 100644 --- a/internal/audit/run/adaptive_test.go +++ b/internal/audit/run/adaptive_test.go @@ -544,3 +544,36 @@ func TestUnit_Adaptive_TheReductionKeepsWhatTheCreateNeeded(t *testing.T) { t.Errorf("the reduction kept the field the API refused: %#v", recorded.Request) } } + +// TestUnit_Adaptive_AnOperatorValueReachesTheWire: a value the operator +// supplies for a field the audit cannot guess is what the live create sends. +// Run rebuilds every body the plan derived, so a value applied at plan time +// alone never leaves the plan. +func TestUnit_Adaptive_AnOperatorValueReachesTheWire(t *testing.T) { + t.Parallel() + s := quirkserver.New(t, quirkserver.Quirks{}) + opts := strategyOptions(t, s, nil) + opts.Inputs = &plan.Inputs{Entities: map[string]plan.EntityInputs{ + "monitor": {Values: map[string]any{"interval": 42}}, + }} + + _, sum := mustRun(t, opts) + + var sent map[string]any + for _, b := range sum.RequestBodies { + if b.Entity == "monitor" && b.Minimal != nil { + sent = b.Minimal.Request + } + } + if sent == nil { + t.Fatalf("no accepted create was recorded for monitor: %+v", sum.Entities) + } + if got := sent["interval"]; got != 42 { + t.Errorf("interval = %#v, want the operator's value on the wire", got) + } + // A field the operator said nothing about is still synthesised, so the + // override replaces one value rather than the whole body. + if got := sent["kind"]; got == nil || got == 42 { + t.Errorf("kind = %#v, want the synthesised value", got) + } +} diff --git a/internal/audit/run/adjust_unit_test.go b/internal/audit/run/adjust_unit_test.go index 700d75e..00073f2 100644 --- a/internal/audit/run/adjust_unit_test.go +++ b/internal/audit/run/adjust_unit_test.go @@ -417,3 +417,36 @@ func TestUnit_Search_MaximalCulpritPrefersTheNamedField(t *testing.T) { t.Errorf("culprit = %q, want none", got) } } + +// TestUnit_Strategize_AnOperatorValueOutranksEverySynthesis: the value an +// operator supplies is what the skeleton sends, over an example, an enum and +// the variant's own gate. +func TestUnit_Strategize_AnOperatorValueOutranksEverySynthesis(t *testing.T) { + t.Parallel() + sk := strategy.Skeleton{ + Fields: []string{"endpoint", "kind", "name"}, + Hints: []strategy.SynthHint{ + {Field: "endpoint", Type: "string", Example: "https://unreachable.invalid"}, + {Field: "kind", Type: "string", Enum: []any{"ping", "web"}}, + {Field: "name", Type: "string"}, + }, + } + values := map[string]any{"endpoint": "https://reachable.example", "kind": "web"} + + body := synthSkeletonBody(sk, "monitor", "tfpfgen", "kind", "ping", values) + + // The operator supplies a value precisely for the field no synthesis can + // guess: an example the API cannot reach is still an example. + if body["endpoint"] != "https://reachable.example" { + t.Errorf("endpoint = %#v, want the operator's value", body["endpoint"]) + } + // Even the gate yields: a discriminator is one of the things an operator + // has to supply when the document does not say which shape is valid. + if body["kind"] != "web" { + t.Errorf("kind = %#v, want the operator's value over the variant gate", body["kind"]) + } + // A field the operator said nothing about is synthesised as before. + if body["name"] != "tfpfgen--monitor-name" { + t.Errorf("name = %#v, want the invented name", body["name"]) + } +} diff --git a/internal/audit/run/run.go b/internal/audit/run/run.go index eabefee..47cd464 100644 --- a/internal/audit/run/run.go +++ b/internal/audit/run/run.go @@ -76,8 +76,13 @@ type Options struct { // strategy.Strategy describes, under a complexity-scaled per-entity // budget. When Doc is nil the plan is executed as given — the path the // executor's own unit tests take. - Doc *specmodel.Document - Config *config.Config + Doc *specmodel.Document + Config *config.Config + // Inputs are the operator-supplied values the audit cannot synthesize, + // read from audit/inputs.json. The plan resolves parentRefs and skip from + // them; the values reach the wire through strategize, which rebuilds every + // body the plan derived and would otherwise not see them. + Inputs *plan.Inputs BaseURL string Auth Auth // NamePrefix marks every created object's name-bearing fields and @@ -220,13 +225,14 @@ func Run(ctx context.Context, opts Options) ([]observe.Observation, Summary, err var hints map[string]map[string]strategy.SynthHint var strategies map[string]*strategy.Strategy if opts.Plan != nil && opts.Doc != nil && opts.Config != nil { - opts.Plan, hints, strategies = strategize(opts.Plan, opts.Doc, opts.Config, opts.NamePrefix) + opts.Plan, hints, strategies = strategize(opts.Plan, opts.Doc, opts.Config, opts.NamePrefix, opts.Inputs) } r, err := newRunner(opts) if err != nil { return nil, Summary{}, err } r.hints = hints + r.inputValues = entityValues(opts.Plan, opts.Inputs) r.strategies = strategies defer r.ledger.close() @@ -298,6 +304,10 @@ type runner struct { // loop draws on when it must add a field a refusal named. Nil on a // non-strategy run. hints map[string]map[string]strategy.SynthHint + // inputValues carries, per entity, the operator's value overrides, so the + // adjustment loop adding a field live uses the same value the plan would + // have sent for it. + inputValues map[string]map[string]any // strategies carries each entity's compiled strategy, so the inference // can read the hypotheses the run was meant to confirm. Nil on a // non-strategy run, which is what makes such a run skip inference. diff --git a/internal/audit/run/strategize.go b/internal/audit/run/strategize.go index 05309fd..fc95416 100644 --- a/internal/audit/run/strategize.go +++ b/internal/audit/run/strategize.go @@ -43,7 +43,7 @@ const ( // themselves, which the triangulating inference reads for the hypotheses each // run was meant to confirm. The input plan is left untouched: a new plan is // built so the caller's copy is never mutated. -func strategize(p *plan.Plan, doc *specmodel.Document, cfg *config.Config, prefix string) (*plan.Plan, map[string]map[string]strategy.SynthHint, map[string]*strategy.Strategy) { +func strategize(p *plan.Plan, doc *specmodel.Document, cfg *config.Config, prefix string, inputs *plan.Inputs) (*plan.Plan, map[string]map[string]strategy.SynthHint, map[string]*strategy.Strategy) { cls := specmodel.Classify(doc) byKey := make(map[string]specmodel.Classification, len(cls.Entities)) for _, c := range cls.Entities { @@ -69,7 +69,7 @@ func strategize(p *plan.Plan, doc *specmodel.Document, cfg *config.Config, prefi continue } addr := addressingOf(&ep) - ep.Steps = translateProgram(compiled, addr, ep.Entity, prefix) + ep.Steps = translateProgram(compiled, addr, ep.Entity, prefix, inputs.ValuesFor(ep.Entity)) ep.Budget = plan.Budget{Requests: compiled.Budget.Requests} hints[ep.Entity] = collectHints(compiled) strategies[ep.Entity] = compiled @@ -142,13 +142,33 @@ func addressingOf(ep *plan.EntityPlan) addressing { return a } +// entityValues indexes the operator's value overrides by entity, for every +// entity the plan carries. +// +// The adjustment loop reads them as well as the translator: a field the API +// forces into a body live is the same field the operator supplied a value for, +// and synthesising a different one there would send two values for one field +// across a single run. +func entityValues(p *plan.Plan, inputs *plan.Inputs) map[string]map[string]any { + if p == nil || inputs == nil { + return nil + } + out := map[string]map[string]any{} + for i := range p.Entities { + if v := inputs.ValuesFor(p.Entities[i].Entity); len(v) > 0 { + out[p.Entities[i].Entity] = v + } + } + return out +} + // translateProgram turns a strategy's ordered, value-free program into // executable steps: addressing from addr, request bodies synthesised from the variant // skeletons and per-field hints. -func translateProgram(compiled *strategy.Strategy, addr addressing, entity, prefix string) []plan.Step { +func translateProgram(compiled *strategy.Strategy, addr addressing, entity, prefix string, values map[string]any) []plan.Step { baseMinimal := map[string]any{} if len(compiled.Variants) > 0 { - baseMinimal = synthSkeletonBody(compiled.Variants[0].Minimal, entity, prefix, "", "") + baseMinimal = synthSkeletonBody(compiled.Variants[0].Minimal, entity, prefix, "", "", values) } hints := collectHints(compiled) @@ -160,11 +180,11 @@ func translateProgram(compiled *strategy.Strategy, addr addressing, entity, pref steps = append(steps, plan.Step{ Kind: s.Kind, Method: addr.createMethod, Path: addr.collectionPath, PathValues: addr.collectionValues, - Body: synthSkeletonBody(v.Minimal, entity, prefix, s.GateField, s.GateValue), + Body: synthSkeletonBody(v.Minimal, entity, prefix, s.GateField, s.GateValue, values), }) case plan.StepCreateMaximal: v := findVariant(compiled, s.GateField, s.GateValue) - body := synthSkeletonBody(v.Maximal, entity, prefix, s.GateField, s.GateValue) + body := synthSkeletonBody(v.Maximal, entity, prefix, s.GateField, s.GateValue, values) steps = append(steps, plan.Step{ Kind: s.Kind, Method: addr.createMethod, Path: addr.collectionPath, PathValues: addr.collectionValues, Body: body, @@ -307,15 +327,26 @@ func countOptional(v strategy.Variant, body map[string]any) int { } // synthSkeletonBody synthesises a create body from a skeleton: one value per -// field, drawn from that field's hint, with the gate field pinned to the -// variant's value where one is given. -func synthSkeletonBody(sk strategy.Skeleton, entity, prefix, gateField, gateValue string) map[string]any { +// field, drawn from the operator's inputs where they name it and from that +// field's hint otherwise, with the gate field pinned to the variant's value +// where one is given. +// +// An operator value outranks everything, including the gate: it is supplied +// precisely for the fields no synthesis can guess — a reachable endpoint, an +// existing agent's id, the discriminator a polymorphic body is keyed on. +// Scoped to the body's own fields, as the plan's synthesis is, because a +// wire property name says nothing about which nested object it belongs to. +func synthSkeletonBody(sk strategy.Skeleton, entity, prefix, gateField, gateValue string, values map[string]any) map[string]any { byField := make(map[string]strategy.SynthHint, len(sk.Hints)) for _, h := range sk.Hints { byField[h.Field] = h } body := map[string]any{} for _, f := range sk.Fields { + if v, ok := values[f]; ok { + body[f] = v + continue + } if f == gateField && gateValue != "" { body[f] = typedGate(byField[f], gateValue) continue @@ -330,6 +361,9 @@ func synthSkeletonBody(sk strategy.Skeleton, entity, prefix, gateField, gateValu // synthField synthesises one field the adjustment loop must add live, from its // strategy hint when known and from its name and a string default otherwise. func (r *runner) synthField(ent *entityState, field string) any { + if v, ok := r.inputValues[ent.plan.Entity][field]; ok { + return v + } if hints := r.hints[ent.plan.Entity]; hints != nil { if h, ok := hints[field]; ok { return synthValue(h, ent.plan.Entity, r.opts.NamePrefix) diff --git a/internal/cli/audit.go b/internal/cli/audit.go index ef8cb21..2fd9347 100644 --- a/internal/cli/audit.go +++ b/internal/cli/audit.go @@ -59,7 +59,7 @@ func newAuditRunCommand() *cobra.Command { "responsibility.", Args: exactArgs("tfpfgen audit run [--dir spec] [--config tfpfgen.yaml] [--out audit/observations] [--base-url URL] [--force-api-audit]"), RunE: func(cmd *cobra.Command, args []string) error { - cfg, p, doc, lock, err := auditPlan(dir, cfgFile) + cfg, p, doc, inputs, lock, err := auditPlan(dir, cfgFile) if err != nil { return err } @@ -75,6 +75,7 @@ func newAuditRunCommand() *cobra.Command { Plan: p, Doc: doc, Config: cfg, + Inputs: inputs, BaseURL: base, Auth: auditrun.Auth{ Method: cfg.Auth.Method, @@ -140,7 +141,7 @@ func newAuditCleanupCommand() *cobra.Command { Short: "delete the live test objects a previous audit left behind", Args: exactArgs("tfpfgen audit cleanup [--dir spec] [--config tfpfgen.yaml] [--base-url URL] [--prefix NAME]"), RunE: func(cmd *cobra.Command, args []string) error { - cfg, p, _, _, err := auditPlan(dir, cfgFile) + cfg, p, _, _, _, err := auditPlan(dir, cfgFile) if err != nil { return err } @@ -185,46 +186,50 @@ func newAuditCleanupCommand() *cobra.Command { return cmd } -// auditPlan loads everything both audit verbs derive from: the config, -// the revised document, the operator inputs, and the upstream pin the +// auditPlan loads everything both audit verbs derive from: the config, the +// revised document, the operator inputs, and the upstream pin the // observations are stamped with. -func auditPlan(dir, cfgFile string) (*config.Config, *plan.Plan, *specmodel.Document, store.Lock, error) { +// +// The inputs are answered as well as applied. Derive resolves parentRefs and +// skip from them, and their values reach the wire only through the run, which +// rebuilds every body the plan derived. +func auditPlan(dir, cfgFile string) (*config.Config, *plan.Plan, *specmodel.Document, *plan.Inputs, store.Lock, error) { cfg, err := config.Load(cfgFile) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } if !cfg.Audit.Enabled { - return nil, nil, nil, store.Lock{}, fmt.Errorf("audit.enabled is false in %s; nothing to do", cfgFile) + return nil, nil, nil, nil, store.Lock{}, fmt.Errorf("audit.enabled is false in %s; nothing to do", cfgFile) } data, srcPath, err := auditSpecBytes(dir) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } doc, err := specmodel.Load(data) if err != nil { - return nil, nil, nil, store.Lock{}, fmt.Errorf("%s: %w", srcPath, err) + return nil, nil, nil, nil, store.Lock{}, fmt.Errorf("%s: %w", srcPath, err) } lock, err := store.Verify(dir) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } rawInputs, err := os.ReadFile(plan.InputsPath) if err != nil && !os.IsNotExist(err) { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } inputs, err := plan.ParseInputs(rawInputs) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } p, err := plan.Derive(doc, cfg, inputs) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } - return cfg, p, doc, lock, nil + return cfg, p, doc, inputs, lock, nil } // auditBaseURL picks the audited API's root: the flag, then the config