diff --git a/internal/audit/plan/inputs.go b/internal/audit/plan/inputs.go index 8bc5ec4..db68d9a 100644 --- a/internal/audit/plan/inputs.go +++ b/internal/audit/plan/inputs.go @@ -110,6 +110,16 @@ func parseEntityInputs(entity string, raw json.RawMessage) (EntityInputs, error) // forEntity is the lookup Derive uses: absent entities read as the zero // value, per the graceful-degradation contract. +// ValuesFor answers the operator's value overrides for one entity, keyed by +// wire property name, and nil where none were supplied. +// +// The audit cannot invent a value the API will take for every field: a +// reachable endpoint, an existing agent's id, the discriminator a polymorphic +// body is keyed on. These are what the operator supplies in their place. +func (in *Inputs) ValuesFor(entity string) map[string]any { + return in.forEntity(entity).Values +} + func (in *Inputs) forEntity(entity string) EntityInputs { if in == nil || in.Entities == nil { return EntityInputs{} diff --git a/internal/audit/run/adaptive_test.go b/internal/audit/run/adaptive_test.go index 57246a0..e4a08ce 100644 --- a/internal/audit/run/adaptive_test.go +++ b/internal/audit/run/adaptive_test.go @@ -544,3 +544,36 @@ func TestUnit_Adaptive_TheReductionKeepsWhatTheCreateNeeded(t *testing.T) { t.Errorf("the reduction kept the field the API refused: %#v", recorded.Request) } } + +// TestUnit_Adaptive_AnOperatorValueReachesTheWire: a value the operator +// supplies for a field the audit cannot guess is what the live create sends. +// Run rebuilds every body the plan derived, so a value applied at plan time +// alone never leaves the plan. +func TestUnit_Adaptive_AnOperatorValueReachesTheWire(t *testing.T) { + t.Parallel() + s := quirkserver.New(t, quirkserver.Quirks{}) + opts := strategyOptions(t, s, nil) + opts.Inputs = &plan.Inputs{Entities: map[string]plan.EntityInputs{ + "monitor": {Values: map[string]any{"interval": 42}}, + }} + + _, sum := mustRun(t, opts) + + var sent map[string]any + for _, b := range sum.RequestBodies { + if b.Entity == "monitor" && b.Minimal != nil { + sent = b.Minimal.Request + } + } + if sent == nil { + t.Fatalf("no accepted create was recorded for monitor: %+v", sum.Entities) + } + if got := sent["interval"]; got != 42 { + t.Errorf("interval = %#v, want the operator's value on the wire", got) + } + // A field the operator said nothing about is still synthesised, so the + // override replaces one value rather than the whole body. + if got := sent["kind"]; got == nil || got == 42 { + t.Errorf("kind = %#v, want the synthesised value", got) + } +} diff --git a/internal/audit/run/adjust_unit_test.go b/internal/audit/run/adjust_unit_test.go index 700d75e..00073f2 100644 --- a/internal/audit/run/adjust_unit_test.go +++ b/internal/audit/run/adjust_unit_test.go @@ -417,3 +417,36 @@ func TestUnit_Search_MaximalCulpritPrefersTheNamedField(t *testing.T) { t.Errorf("culprit = %q, want none", got) } } + +// TestUnit_Strategize_AnOperatorValueOutranksEverySynthesis: the value an +// operator supplies is what the skeleton sends, over an example, an enum and +// the variant's own gate. +func TestUnit_Strategize_AnOperatorValueOutranksEverySynthesis(t *testing.T) { + t.Parallel() + sk := strategy.Skeleton{ + Fields: []string{"endpoint", "kind", "name"}, + Hints: []strategy.SynthHint{ + {Field: "endpoint", Type: "string", Example: "https://unreachable.invalid"}, + {Field: "kind", Type: "string", Enum: []any{"ping", "web"}}, + {Field: "name", Type: "string"}, + }, + } + values := map[string]any{"endpoint": "https://reachable.example", "kind": "web"} + + body := synthSkeletonBody(sk, "monitor", "tfpfgen", "kind", "ping", values) + + // The operator supplies a value precisely for the field no synthesis can + // guess: an example the API cannot reach is still an example. + if body["endpoint"] != "https://reachable.example" { + t.Errorf("endpoint = %#v, want the operator's value", body["endpoint"]) + } + // Even the gate yields: a discriminator is one of the things an operator + // has to supply when the document does not say which shape is valid. + if body["kind"] != "web" { + t.Errorf("kind = %#v, want the operator's value over the variant gate", body["kind"]) + } + // A field the operator said nothing about is synthesised as before. + if body["name"] != "tfpfgen--monitor-name" { + t.Errorf("name = %#v, want the invented name", body["name"]) + } +} diff --git a/internal/audit/run/run.go b/internal/audit/run/run.go index eabefee..47cd464 100644 --- a/internal/audit/run/run.go +++ b/internal/audit/run/run.go @@ -76,8 +76,13 @@ type Options struct { // strategy.Strategy describes, under a complexity-scaled per-entity // budget. When Doc is nil the plan is executed as given — the path the // executor's own unit tests take. - Doc *specmodel.Document - Config *config.Config + Doc *specmodel.Document + Config *config.Config + // Inputs are the operator-supplied values the audit cannot synthesize, + // read from audit/inputs.json. The plan resolves parentRefs and skip from + // them; the values reach the wire through strategize, which rebuilds every + // body the plan derived and would otherwise not see them. + Inputs *plan.Inputs BaseURL string Auth Auth // NamePrefix marks every created object's name-bearing fields and @@ -220,13 +225,14 @@ func Run(ctx context.Context, opts Options) ([]observe.Observation, Summary, err var hints map[string]map[string]strategy.SynthHint var strategies map[string]*strategy.Strategy if opts.Plan != nil && opts.Doc != nil && opts.Config != nil { - opts.Plan, hints, strategies = strategize(opts.Plan, opts.Doc, opts.Config, opts.NamePrefix) + opts.Plan, hints, strategies = strategize(opts.Plan, opts.Doc, opts.Config, opts.NamePrefix, opts.Inputs) } r, err := newRunner(opts) if err != nil { return nil, Summary{}, err } r.hints = hints + r.inputValues = entityValues(opts.Plan, opts.Inputs) r.strategies = strategies defer r.ledger.close() @@ -298,6 +304,10 @@ type runner struct { // loop draws on when it must add a field a refusal named. Nil on a // non-strategy run. hints map[string]map[string]strategy.SynthHint + // inputValues carries, per entity, the operator's value overrides, so the + // adjustment loop adding a field live uses the same value the plan would + // have sent for it. + inputValues map[string]map[string]any // strategies carries each entity's compiled strategy, so the inference // can read the hypotheses the run was meant to confirm. Nil on a // non-strategy run, which is what makes such a run skip inference. diff --git a/internal/audit/run/strategize.go b/internal/audit/run/strategize.go index 05309fd..fc95416 100644 --- a/internal/audit/run/strategize.go +++ b/internal/audit/run/strategize.go @@ -43,7 +43,7 @@ const ( // themselves, which the triangulating inference reads for the hypotheses each // run was meant to confirm. The input plan is left untouched: a new plan is // built so the caller's copy is never mutated. -func strategize(p *plan.Plan, doc *specmodel.Document, cfg *config.Config, prefix string) (*plan.Plan, map[string]map[string]strategy.SynthHint, map[string]*strategy.Strategy) { +func strategize(p *plan.Plan, doc *specmodel.Document, cfg *config.Config, prefix string, inputs *plan.Inputs) (*plan.Plan, map[string]map[string]strategy.SynthHint, map[string]*strategy.Strategy) { cls := specmodel.Classify(doc) byKey := make(map[string]specmodel.Classification, len(cls.Entities)) for _, c := range cls.Entities { @@ -69,7 +69,7 @@ func strategize(p *plan.Plan, doc *specmodel.Document, cfg *config.Config, prefi continue } addr := addressingOf(&ep) - ep.Steps = translateProgram(compiled, addr, ep.Entity, prefix) + ep.Steps = translateProgram(compiled, addr, ep.Entity, prefix, inputs.ValuesFor(ep.Entity)) ep.Budget = plan.Budget{Requests: compiled.Budget.Requests} hints[ep.Entity] = collectHints(compiled) strategies[ep.Entity] = compiled @@ -142,13 +142,33 @@ func addressingOf(ep *plan.EntityPlan) addressing { return a } +// entityValues indexes the operator's value overrides by entity, for every +// entity the plan carries. +// +// The adjustment loop reads them as well as the translator: a field the API +// forces into a body live is the same field the operator supplied a value for, +// and synthesising a different one there would send two values for one field +// across a single run. +func entityValues(p *plan.Plan, inputs *plan.Inputs) map[string]map[string]any { + if p == nil || inputs == nil { + return nil + } + out := map[string]map[string]any{} + for i := range p.Entities { + if v := inputs.ValuesFor(p.Entities[i].Entity); len(v) > 0 { + out[p.Entities[i].Entity] = v + } + } + return out +} + // translateProgram turns a strategy's ordered, value-free program into // executable steps: addressing from addr, request bodies synthesised from the variant // skeletons and per-field hints. -func translateProgram(compiled *strategy.Strategy, addr addressing, entity, prefix string) []plan.Step { +func translateProgram(compiled *strategy.Strategy, addr addressing, entity, prefix string, values map[string]any) []plan.Step { baseMinimal := map[string]any{} if len(compiled.Variants) > 0 { - baseMinimal = synthSkeletonBody(compiled.Variants[0].Minimal, entity, prefix, "", "") + baseMinimal = synthSkeletonBody(compiled.Variants[0].Minimal, entity, prefix, "", "", values) } hints := collectHints(compiled) @@ -160,11 +180,11 @@ func translateProgram(compiled *strategy.Strategy, addr addressing, entity, pref steps = append(steps, plan.Step{ Kind: s.Kind, Method: addr.createMethod, Path: addr.collectionPath, PathValues: addr.collectionValues, - Body: synthSkeletonBody(v.Minimal, entity, prefix, s.GateField, s.GateValue), + Body: synthSkeletonBody(v.Minimal, entity, prefix, s.GateField, s.GateValue, values), }) case plan.StepCreateMaximal: v := findVariant(compiled, s.GateField, s.GateValue) - body := synthSkeletonBody(v.Maximal, entity, prefix, s.GateField, s.GateValue) + body := synthSkeletonBody(v.Maximal, entity, prefix, s.GateField, s.GateValue, values) steps = append(steps, plan.Step{ Kind: s.Kind, Method: addr.createMethod, Path: addr.collectionPath, PathValues: addr.collectionValues, Body: body, @@ -307,15 +327,26 @@ func countOptional(v strategy.Variant, body map[string]any) int { } // synthSkeletonBody synthesises a create body from a skeleton: one value per -// field, drawn from that field's hint, with the gate field pinned to the -// variant's value where one is given. -func synthSkeletonBody(sk strategy.Skeleton, entity, prefix, gateField, gateValue string) map[string]any { +// field, drawn from the operator's inputs where they name it and from that +// field's hint otherwise, with the gate field pinned to the variant's value +// where one is given. +// +// An operator value outranks everything, including the gate: it is supplied +// precisely for the fields no synthesis can guess — a reachable endpoint, an +// existing agent's id, the discriminator a polymorphic body is keyed on. +// Scoped to the body's own fields, as the plan's synthesis is, because a +// wire property name says nothing about which nested object it belongs to. +func synthSkeletonBody(sk strategy.Skeleton, entity, prefix, gateField, gateValue string, values map[string]any) map[string]any { byField := make(map[string]strategy.SynthHint, len(sk.Hints)) for _, h := range sk.Hints { byField[h.Field] = h } body := map[string]any{} for _, f := range sk.Fields { + if v, ok := values[f]; ok { + body[f] = v + continue + } if f == gateField && gateValue != "" { body[f] = typedGate(byField[f], gateValue) continue @@ -330,6 +361,9 @@ func synthSkeletonBody(sk strategy.Skeleton, entity, prefix, gateField, gateValu // synthField synthesises one field the adjustment loop must add live, from its // strategy hint when known and from its name and a string default otherwise. func (r *runner) synthField(ent *entityState, field string) any { + if v, ok := r.inputValues[ent.plan.Entity][field]; ok { + return v + } if hints := r.hints[ent.plan.Entity]; hints != nil { if h, ok := hints[field]; ok { return synthValue(h, ent.plan.Entity, r.opts.NamePrefix) diff --git a/internal/cli/audit.go b/internal/cli/audit.go index ef8cb21..2fd9347 100644 --- a/internal/cli/audit.go +++ b/internal/cli/audit.go @@ -59,7 +59,7 @@ func newAuditRunCommand() *cobra.Command { "responsibility.", Args: exactArgs("tfpfgen audit run [--dir spec] [--config tfpfgen.yaml] [--out audit/observations] [--base-url URL] [--force-api-audit]"), RunE: func(cmd *cobra.Command, args []string) error { - cfg, p, doc, lock, err := auditPlan(dir, cfgFile) + cfg, p, doc, inputs, lock, err := auditPlan(dir, cfgFile) if err != nil { return err } @@ -75,6 +75,7 @@ func newAuditRunCommand() *cobra.Command { Plan: p, Doc: doc, Config: cfg, + Inputs: inputs, BaseURL: base, Auth: auditrun.Auth{ Method: cfg.Auth.Method, @@ -140,7 +141,7 @@ func newAuditCleanupCommand() *cobra.Command { Short: "delete the live test objects a previous audit left behind", Args: exactArgs("tfpfgen audit cleanup [--dir spec] [--config tfpfgen.yaml] [--base-url URL] [--prefix NAME]"), RunE: func(cmd *cobra.Command, args []string) error { - cfg, p, _, _, err := auditPlan(dir, cfgFile) + cfg, p, _, _, _, err := auditPlan(dir, cfgFile) if err != nil { return err } @@ -185,46 +186,50 @@ func newAuditCleanupCommand() *cobra.Command { return cmd } -// auditPlan loads everything both audit verbs derive from: the config, -// the revised document, the operator inputs, and the upstream pin the +// auditPlan loads everything both audit verbs derive from: the config, the +// revised document, the operator inputs, and the upstream pin the // observations are stamped with. -func auditPlan(dir, cfgFile string) (*config.Config, *plan.Plan, *specmodel.Document, store.Lock, error) { +// +// The inputs are answered as well as applied. Derive resolves parentRefs and +// skip from them, and their values reach the wire only through the run, which +// rebuilds every body the plan derived. +func auditPlan(dir, cfgFile string) (*config.Config, *plan.Plan, *specmodel.Document, *plan.Inputs, store.Lock, error) { cfg, err := config.Load(cfgFile) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } if !cfg.Audit.Enabled { - return nil, nil, nil, store.Lock{}, fmt.Errorf("audit.enabled is false in %s; nothing to do", cfgFile) + return nil, nil, nil, nil, store.Lock{}, fmt.Errorf("audit.enabled is false in %s; nothing to do", cfgFile) } data, srcPath, err := auditSpecBytes(dir) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } doc, err := specmodel.Load(data) if err != nil { - return nil, nil, nil, store.Lock{}, fmt.Errorf("%s: %w", srcPath, err) + return nil, nil, nil, nil, store.Lock{}, fmt.Errorf("%s: %w", srcPath, err) } lock, err := store.Verify(dir) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } rawInputs, err := os.ReadFile(plan.InputsPath) if err != nil && !os.IsNotExist(err) { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } inputs, err := plan.ParseInputs(rawInputs) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } p, err := plan.Derive(doc, cfg, inputs) if err != nil { - return nil, nil, nil, store.Lock{}, err + return nil, nil, nil, nil, store.Lock{}, err } - return cfg, p, doc, lock, nil + return cfg, p, doc, inputs, lock, nil } // auditBaseURL picks the audited API's root: the flag, then the config