diff --git a/cmd/waza/cmd_get.go b/cmd/waza/cmd_get.go new file mode 100644 index 00000000..a1667600 --- /dev/null +++ b/cmd/waza/cmd_get.go @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package main + +import ( + "fmt" + + "github.com/microsoft/waza/internal/models" + "github.com/microsoft/waza/internal/registry" + "github.com/spf13/cobra" +) + +func newGetCommand() *cobra.Command { + var strictLock bool + + cmd := &cobra.Command{ + Use: "get [eval.yaml]", + Short: "Resolve remote grader refs and write waza.lock", + Long: `Resolve every 'ref:' grader entry in eval.yaml against its remote Git +source, download the pinned content into the module cache, and write +(or update) waza.lock beside eval.yaml. + +Phase 1 supports Go-module-style refs of the form: + + github.com//[/path][#export]@ + +where must be an exact semver tag (vX.Y.Z) or a full 40-character +commit SHA. Floating selectors (branches, ranges, "latest") are rejected +for reproducibility. + +Examples: + + waza get # resolves ./eval.yaml + waza get evals/factuality/eval.yaml # explicit spec path + waza get --verify # do not modify the lock; verify only +`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + specPath := "eval.yaml" + if len(args) == 1 { + specPath = args[0] + } + spec, err := models.LoadEvalSpec(specPath) + if err != nil { + return fmt.Errorf("loading %s: %w", specPath, err) + } + + refCount := 0 + for _, g := range spec.Graders { + if g.Ref != "" { + refCount++ + } + } + if refCount == 0 { + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "No remote grader refs found in %s. Nothing to do.\n", specPath) + return nil + } + + resolved, lockChanged, err := registry.ResolveSpec(cmd.Context(), spec, specPath, !strictLock) + if err != nil { + return err + } + lockPath := registry.LockfilePath(specPath) + switch { + case strictLock: + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Verified %d remote grader ref(s) against %s.\n", resolved, lockPath) + case lockChanged: + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resolved %d remote grader ref(s); wrote %s.\n", resolved, lockPath) + default: + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resolved %d remote grader ref(s); %s already up to date.\n", resolved, lockPath) + } + return nil + }, + } + + cmd.Flags().BoolVar(&strictLock, "verify", false, "Verify refs against existing waza.lock without modifying it (fails if any ref is unlocked or digest-mismatched)") + return cmd +} diff --git a/cmd/waza/cmd_run.go b/cmd/waza/cmd_run.go index e02b3664..9d5879b4 100644 --- a/cmd/waza/cmd_run.go +++ b/cmd/waza/cmd_run.go @@ -34,6 +34,7 @@ import ( "github.com/microsoft/waza/internal/orchestration" "github.com/microsoft/waza/internal/projectconfig" "github.com/microsoft/waza/internal/recommend" + "github.com/microsoft/waza/internal/registry" "github.com/microsoft/waza/internal/reporting" "github.com/microsoft/waza/internal/session" "github.com/microsoft/waza/internal/snapshot" @@ -568,6 +569,13 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str return nil, fmt.Errorf("failed to load spec: %w", err) } + // Resolve remote grader refs (Phase 1: exact-pinned github.com refs). + // If waza.lock exists we verify strictly; otherwise auto-resolve and + // write the lock (permissive first-run policy). + if err := resolveGraderRefs(cmd, spec, specPath); err != nil { + return nil, err + } + // CLI flags override spec config if parallel { spec.Config.Concurrent = true @@ -2141,3 +2149,55 @@ func runDiscoverMode(cmd *cobra.Command, args []string) error { return lastErr } + +// resolveGraderRefs expands remote grader refs referenced in spec against +// waza.lock beside specPath. +// +// Policy (Phase 1): +// - No refs → no-op. +// - waza.lock exists → verify strictly. Missing entries or digest +// mismatches fail the run. +// - waza.lock missing → auto-resolve iff every ref is exact-pinned +// (tag or 40-char commit SHA), then write the lock and log a warning +// that the lock was created. If any ref uses a floating selector +// (currently rejected up front by ParseRef), fail. +func resolveGraderRefs(cmd *cobra.Command, spec *models.EvalSpec, specPath string) error { + refCount := 0 + for _, g := range spec.Graders { + if g.Ref != "" { + refCount++ + } + } + if refCount == 0 { + return nil + } + + ctx := context.Background() + if cmd != nil { + ctx = cmd.Context() + } + + lockPath := registry.LockfilePath(specPath) + lockExists := false + if _, err := os.Stat(lockPath); err == nil { + lockExists = true + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat %s: %w", lockPath, err) + } + + updateLock := !lockExists + resolved, changed, err := registry.ResolveSpec(ctx, spec, specPath, updateLock) + if err != nil { + if errors.Is(err, registry.ErrRefNotInLock) { + return fmt.Errorf( + "%w — run `waza get %s` to resolve remote grader refs and write %s", + err, specPath, filepath.Base(lockPath), + ) + } + return err + } + if updateLock && changed { + fmt.Fprintf(os.Stderr, "waza: resolved %d remote grader ref(s); wrote %s\n", resolved, lockPath) + } + return nil +} diff --git a/cmd/waza/root.go b/cmd/waza/root.go index ba953de2..466341f1 100644 --- a/cmd/waza/root.go +++ b/cmd/waza/root.go @@ -52,6 +52,7 @@ performance against predefined test cases.`, // Add subcommands cmd.AddCommand(newRunCommand()) cmd.AddCommand(newInitCommand()) + cmd.AddCommand(newGetCommand()) cmd.AddCommand(tokens.NewCommand()) cmd.AddCommand(newCompareCommand()) cmd.AddCommand(newGateCommand()) diff --git a/internal/models/spec.go b/internal/models/spec.go index 6c4da922..573ff1cd 100644 --- a/internal/models/spec.go +++ b/internal/models/spec.go @@ -56,6 +56,7 @@ type strictEvalSpec struct { } type strictGrader struct { + Ref string `yaml:"ref,omitempty"` Kind GraderKind `yaml:"type"` Identifier string `yaml:"name"` ScriptPath string `yaml:"script,omitempty"` @@ -193,6 +194,13 @@ func (c *Config) ShouldInjectSkillBody() bool { // GraderConfig defines a validator/grader type GraderConfig struct { + // Ref is an optional remote grader reference in Go-module style, + // e.g. "github.com/waza-evals/fact#factuality@v1.0.0". When set, the + // grader definition (type, config, model, ...) is loaded from the + // remote module manifest; local fields on this GraderConfig override + // remote defaults after resolution. Requires a waza.lock entry pinning + // the commit SHA + content digest of the remote preset. + Ref string `yaml:"ref,omitempty" json:"ref,omitempty"` Kind GraderKind `yaml:"type" json:"kind"` Identifier string `yaml:"name" json:"identifier"` ScriptPath string `yaml:"script,omitempty" json:"script_path,omitempty"` @@ -204,6 +212,7 @@ type GraderConfig struct { func (g *GraderConfig) UnmarshalYAML(node *yaml.Node) error { type rawGraderConfig struct { + Ref string `yaml:"ref,omitempty"` Kind GraderKind `yaml:"type"` Identifier string `yaml:"name"` ScriptPath string `yaml:"script,omitempty"` @@ -227,17 +236,35 @@ func (g *GraderConfig) UnmarshalYAML(node *yaml.Node) error { return err } + g.Ref = raw.Ref + g.Identifier = raw.Identifier + g.ScriptPath = raw.ScriptPath + g.Rubric = raw.Rubric + g.ModelID = raw.ModelID + g.Weight = raw.Weight + + // When a remote ref is set, the grader kind and config come from the + // remote preset. Preserve the raw override config as a generic map so + // the resolver can deep-merge it later. Validation of type/params is + // deferred to after ref expansion. + if raw.Ref != "" { + if raw.Parameters.Kind != 0 { + overrides, err := decodeYAMLNode[GenericGraderParameters](&raw.Parameters) + if err != nil { + return fmt.Errorf("invalid override config for ref %q: %w", raw.Ref, err) + } + g.Parameters = overrides + } + g.Kind = raw.Kind // may be empty; resolver fills it in + return nil + } + params, err := decodeGraderParameters(raw.Kind, &raw.Parameters) if err != nil { return fmt.Errorf("invalid grader config for %q (type %q): %w", raw.Identifier, raw.Kind, err) } g.Kind = raw.Kind - g.Identifier = raw.Identifier - g.ScriptPath = raw.ScriptPath - g.Rubric = raw.Rubric - g.ModelID = raw.ModelID - g.Weight = raw.Weight g.Parameters = params // Validate grader-type-specific required fields diff --git a/internal/models/spec_test.go b/internal/models/spec_test.go index ce2cf562..c501d774 100644 --- a/internal/models/spec_test.go +++ b/internal/models/spec_test.go @@ -738,3 +738,52 @@ tasks: } }) } + +func TestEvalSpec_GraderRef(t *testing.T) { + tempDir := t.TempDir() + yamlContent := `name: ref-grader +skill: test +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: mock +graders: + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + name: my-fact-check + weight: 2.0 + config: + threshold: 0.9 +` + specPath := filepath.Join(tempDir, "ref.yaml") + if err := os.WriteFile(specPath, []byte(yamlContent), 0o644); err != nil { + t.Fatalf("Failed to write spec file: %v", err) + } + spec, err := LoadEvalSpec(specPath) + if err != nil { + t.Fatalf("LoadEvalSpec (ref grader) failed: %v", err) + } + if len(spec.Graders) != 1 { + t.Fatalf("Expected 1 grader, got %d", len(spec.Graders)) + } + g := spec.Graders[0] + if g.Ref != "github.com/waza-evals/fact#factuality@v1.0.0" { + t.Errorf("Ref = %q, want github.com/waza-evals/fact#factuality@v1.0.0", g.Ref) + } + // Type/config validation should be deferred — no error even without type. + if g.Kind != "" { + t.Errorf("Kind = %q, want empty until resolution", g.Kind) + } + if g.Identifier != "my-fact-check" { + t.Errorf("Identifier = %q, want my-fact-check", g.Identifier) + } + if g.Weight != 2.0 { + t.Errorf("Weight = %v, want 2.0", g.Weight) + } + overrides, ok := g.Parameters.(GenericGraderParameters) + if !ok { + t.Fatalf("Parameters = %T, want GenericGraderParameters", g.Parameters) + } + if got := overrides["threshold"]; got != 0.9 { + t.Errorf("threshold override = %v, want 0.9", got) + } +} diff --git a/internal/registry/expand.go b/internal/registry/expand.go new file mode 100644 index 00000000..d809b4dc --- /dev/null +++ b/internal/registry/expand.go @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "fmt" + + "github.com/microsoft/waza/internal/models" + "gopkg.in/yaml.v3" +) + +// ExpandGraderConfig takes a resolved remote grader-preset and merges it with +// the local override GraderConfig, returning a concrete GraderConfig that can +// flow through the normal grader pipeline. +// +// Merge rules (design doc §7): +// 1. Remote preset is the base; it MUST include "type". +// 2. Local scalar fields override remote scalars when non-zero: +// name, weight, model, rubric, script. +// 3. Local `config` deep-merges into remote `config`; local wins on key collisions. +// 4. Lists are replaced, not concatenated. +// 5. `ref` is preserved on the expanded config for auditability. +func ExpandGraderConfig(local models.GraderConfig, resolved *ResolvedGrader) (models.GraderConfig, error) { + if resolved == nil { + return models.GraderConfig{}, fmt.Errorf("expand: resolved grader is nil") + } + + // Parse the remote preset YAML as a generic map so we can deep-merge overrides. + var remote map[string]any + if err := yaml.Unmarshal(resolved.PresetYAML, &remote); err != nil { + return models.GraderConfig{}, fmt.Errorf("parsing remote preset for %s: %w", resolved.Ref.Raw, err) + } + if remote == nil { + remote = map[string]any{} + } + if _, ok := remote["type"]; !ok { + return models.GraderConfig{}, fmt.Errorf("remote preset %s is missing required 'type' field", resolved.Ref.Raw) + } + + // Scalar overrides. + if local.Identifier != "" { + remote["name"] = local.Identifier + } + if local.Weight > 0 { + remote["weight"] = local.Weight + } + if local.ModelID != "" { + remote["model"] = local.ModelID + } + if local.Rubric != "" { + remote["rubric"] = local.Rubric + } + if local.ScriptPath != "" { + remote["script"] = local.ScriptPath + } + // A local `type` on a ref entry is unusual but permit it (matches design + // doc note that local scalars win). + if local.Kind != "" { + remote["type"] = string(local.Kind) + } + + // Deep-merge config maps. + if overrides, ok := local.Parameters.(models.GenericGraderParameters); ok && len(overrides) > 0 { + remoteCfg, _ := remote["config"].(map[string]any) + if remoteCfg == nil { + remoteCfg = map[string]any{} + } + mergeMaps(remoteCfg, map[string]any(overrides)) + remote["config"] = remoteCfg + } + + // Round-trip through YAML so the existing GraderConfig.UnmarshalYAML does + // the strongly-typed decode + validation for us. + merged, err := yaml.Marshal(remote) + if err != nil { + return models.GraderConfig{}, fmt.Errorf("re-marshaling merged grader for %s: %w", resolved.Ref.Raw, err) + } + var node yaml.Node + if err := yaml.Unmarshal(merged, &node); err != nil { + return models.GraderConfig{}, fmt.Errorf("re-parsing merged grader for %s: %w", resolved.Ref.Raw, err) + } + // yaml.Node from Unmarshal is a document node; step into its content. + target := &node + if node.Kind == yaml.DocumentNode && len(node.Content) > 0 { + target = node.Content[0] + } + var out models.GraderConfig + if err := out.UnmarshalYAML(target); err != nil { + return models.GraderConfig{}, fmt.Errorf("expanding ref %s: %w", resolved.Ref.Raw, err) + } + // Preserve the ref for downstream auditing / dashboard display. + out.Ref = resolved.Ref.Raw + // If the remote preset omitted a name, default to the export or last path + // segment so grader outputs have a stable identifier. + if out.Identifier == "" { + out.Identifier = defaultGraderName(resolved.Ref) + } + return out, nil +} + +// mergeMaps deep-merges src into dst, mutating dst. Values in src override dst +// on key collisions. Nested maps are merged recursively; slices are replaced. +func mergeMaps(dst, src map[string]any) { + for k, v := range src { + if existing, ok := dst[k]; ok { + if em, ok := existing.(map[string]any); ok { + if sm, ok := v.(map[string]any); ok { + mergeMaps(em, sm) + continue + } + } + } + dst[k] = v + } +} + +func defaultGraderName(ref Ref) string { + if ref.Export != "" { + return ref.Export + } + if ref.Path != "" { + // last segment, stripped of extension + s := ref.Path + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '/' { + s = s[i+1:] + break + } + } + for i := len(s) - 1; i >= 0; i-- { + if s[i] == '.' { + s = s[:i] + break + } + } + return s + } + return ref.Module() +} diff --git a/internal/registry/expand_test.go b/internal/registry/expand_test.go new file mode 100644 index 00000000..99739e16 --- /dev/null +++ b/internal/registry/expand_test.go @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "testing" + + "github.com/microsoft/waza/internal/models" +) + +func TestExpandGraderConfig_Basic(t *testing.T) { + ref, err := ParseRef("github.com/o/r#g@v1.0.0") + if err != nil { + t.Fatal(err) + } + preset := []byte(`type: text +name: factuality +weight: 1 +config: + contains: + - alpha + - beta +`) + resolved := &ResolvedGrader{Ref: ref, PresetYAML: preset, Lock: LockModule{Ref: ref.Raw}} + + local := models.GraderConfig{Ref: ref.Raw} + got, err := ExpandGraderConfig(local, resolved) + if err != nil { + t.Fatalf("ExpandGraderConfig: %v", err) + } + if got.Kind != models.GraderKindText { + t.Fatalf("Kind = %q, want text", got.Kind) + } + if got.Identifier != "factuality" { + t.Fatalf("Identifier = %q, want factuality", got.Identifier) + } + if got.Ref != ref.Raw { + t.Fatalf("Ref not preserved: %q", got.Ref) + } + params, ok := got.Parameters.(models.TextGraderParameters) + if !ok { + t.Fatalf("Parameters = %T, want TextGraderParameters", got.Parameters) + } + if len(params.Contains) != 2 || params.Contains[0] != "alpha" { + t.Fatalf("Contains = %v", params.Contains) + } +} + +func TestExpandGraderConfig_LocalScalarOverride(t *testing.T) { + ref, _ := ParseRef("github.com/o/r#g@v1.0.0") + preset := []byte("type: text\nname: preset-name\nweight: 1\n") + resolved := &ResolvedGrader{Ref: ref, PresetYAML: preset, Lock: LockModule{Ref: ref.Raw}} + + local := models.GraderConfig{ + Ref: ref.Raw, + Identifier: "my-name", + Weight: 5, + } + got, err := ExpandGraderConfig(local, resolved) + if err != nil { + t.Fatal(err) + } + if got.Identifier != "my-name" { + t.Fatalf("Identifier = %q, want my-name", got.Identifier) + } + if got.Weight != 5 { + t.Fatalf("Weight = %v, want 5", got.Weight) + } +} + +func TestExpandGraderConfig_ConfigDeepMerge(t *testing.T) { + ref, _ := ParseRef("github.com/o/r#g@v1.0.0") + preset := []byte("type: text\nname: g\nconfig:\n contains: [a, b]\n not_contains: [x]\n") + resolved := &ResolvedGrader{Ref: ref, PresetYAML: preset, Lock: LockModule{Ref: ref.Raw}} + + local := models.GraderConfig{ + Ref: ref.Raw, + // Override list should REPLACE the remote list. + Parameters: models.GenericGraderParameters{ + "contains": []any{"z"}, + }, + } + got, err := ExpandGraderConfig(local, resolved) + if err != nil { + t.Fatal(err) + } + params, ok := got.Parameters.(models.TextGraderParameters) + if !ok { + t.Fatalf("Parameters = %T, want TextGraderParameters", got.Parameters) + } + if len(params.Contains) != 1 || params.Contains[0] != "z" { + t.Fatalf("Contains = %v, want [z]", params.Contains) + } + if len(params.NotContains) != 1 || params.NotContains[0] != "x" { + t.Fatalf("NotContains = %v, want [x] (preserved from remote)", params.NotContains) + } +} + +func TestExpandGraderConfig_MissingType(t *testing.T) { + ref, _ := ParseRef("github.com/o/r#g@v1.0.0") + preset := []byte("name: bad\n") // no type + resolved := &ResolvedGrader{Ref: ref, PresetYAML: preset, Lock: LockModule{Ref: ref.Raw}} + + if _, err := ExpandGraderConfig(models.GraderConfig{Ref: ref.Raw}, resolved); err == nil { + t.Fatal("expected error for preset missing type") + } +} + +func TestExpandGraderConfig_DefaultName(t *testing.T) { + ref, _ := ParseRef("github.com/o/r#factuality@v1.0.0") + preset := []byte("type: text\n") // no name + resolved := &ResolvedGrader{Ref: ref, PresetYAML: preset, Lock: LockModule{Ref: ref.Raw}} + + got, err := ExpandGraderConfig(models.GraderConfig{Ref: ref.Raw}, resolved) + if err != nil { + t.Fatal(err) + } + if got.Identifier != "factuality" { + t.Fatalf("default name = %q, want factuality", got.Identifier) + } +} diff --git a/internal/registry/lockfile.go b/internal/registry/lockfile.go new file mode 100644 index 00000000..d018ad1b --- /dev/null +++ b/internal/registry/lockfile.go @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "time" + + "gopkg.in/yaml.v3" +) + +// LockfileName is the file name for the Waza module lockfile. +const LockfileName = "waza.lock" + +// LockfileSchemaVersion is the current lockfile schema version. +const LockfileSchemaVersion = 1 + +// Lockfile is the on-disk representation of pinned module resolutions. +// See docs/research/waza-eval-registry-design.md §8. +type Lockfile struct { + SchemaVersion int `yaml:"schema_version"` + Modules []LockModule `yaml:"modules,omitempty"` +} + +// LockModule pins one resolved ref to an immutable commit SHA and content digest. +type LockModule struct { + // Ref is the canonical ref string exactly as it appears in eval.yaml + // (e.g. "github.com/waza-evals/fact#factuality@v1.0.0"). + Ref string `yaml:"ref"` + // Module is the module identity ("host/owner/repo"). + Module string `yaml:"module"` + // Version is the human-friendly selector — a semver tag or the same + // commit SHA as Commit when the ref pinned by SHA. + Version string `yaml:"version"` + // Commit is the resolved 40-char commit SHA. This is the reproducibility + // guarantee: `waza run` verifies the cached module contents against this + // SHA + Digest before loading. + Commit string `yaml:"commit"` + // Digest is the sha256 content digest of the resolved artifact (e.g. the + // grader preset YAML file), in "sha256:" form. + Digest string `yaml:"digest"` + // URL is the resolved source URL used to fetch the artifact. Recorded + // for auditability; not verified on read. + URL string `yaml:"url,omitempty"` + // ResolvedAt is when the lock entry was written. + ResolvedAt time.Time `yaml:"resolved_at,omitempty"` +} + +// LockfilePath returns the expected path of the lockfile beside the given +// eval.yaml path. +func LockfilePath(evalPath string) string { + return filepath.Join(filepath.Dir(evalPath), LockfileName) +} + +// LoadLockfile reads and parses a lockfile. It returns (nil, nil) when the +// file does not exist so callers can distinguish "no lock" from a parse error. +func LoadLockfile(path string) (*Lockfile, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("reading lockfile %s: %w", path, err) + } + var lock Lockfile + if err := yaml.Unmarshal(data, &lock); err != nil { + return nil, fmt.Errorf("parsing lockfile %s: %w", path, err) + } + if lock.SchemaVersion == 0 { + // Older/empty files: default to current schema for forward-compat. + lock.SchemaVersion = LockfileSchemaVersion + } + if lock.SchemaVersion != LockfileSchemaVersion { + return nil, fmt.Errorf("lockfile %s: unsupported schema_version %d (expected %d)", path, lock.SchemaVersion, LockfileSchemaVersion) + } + return &lock, nil +} + +// Save writes the lockfile to disk with entries sorted by ref for stable diffs. +func (l *Lockfile) Save(path string) error { + if l.SchemaVersion == 0 { + l.SchemaVersion = LockfileSchemaVersion + } + sort.Slice(l.Modules, func(i, j int) bool { + return l.Modules[i].Ref < l.Modules[j].Ref + }) + data, err := yaml.Marshal(l) + if err != nil { + return fmt.Errorf("marshaling lockfile: %w", err) + } + // Prepend a small header comment so hand-inspection is obvious. + header := []byte("# waza.lock — auto-generated by `waza get`. Do not edit by hand.\n# See docs/research/waza-eval-registry-design.md for the design.\n") + if err := os.WriteFile(path, append(header, data...), 0o644); err != nil { + return fmt.Errorf("writing lockfile %s: %w", path, err) + } + return nil +} + +// Lookup returns the lock entry for the given ref string, or nil if absent. +func (l *Lockfile) Lookup(ref string) *LockModule { + if l == nil { + return nil + } + for i := range l.Modules { + if l.Modules[i].Ref == ref { + return &l.Modules[i] + } + } + return nil +} + +// Upsert inserts or replaces the entry for the given ref. +func (l *Lockfile) Upsert(m LockModule) { + for i := range l.Modules { + if l.Modules[i].Ref == m.Ref { + l.Modules[i] = m + return + } + } + l.Modules = append(l.Modules, m) +} diff --git a/internal/registry/lockfile_test.go b/internal/registry/lockfile_test.go new file mode 100644 index 00000000..a0d1db31 --- /dev/null +++ b/internal/registry/lockfile_test.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestLockfile_RoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "waza.lock") + + lock := &Lockfile{ + SchemaVersion: LockfileSchemaVersion, + Modules: []LockModule{ + { + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", Module: "github.com/waza-evals/fact", + Version: "v1.0.0", Commit: "abcdef0123456789abcdef0123456789abcdef01", + Digest: "sha256:deadbeef", URL: "https://example/x", ResolvedAt: time.Unix(0, 0).UTC(), + }, + { + Ref: "github.com/o/r/sub@v0.1.0", Module: "github.com/o/r", + Version: "v0.1.0", Commit: "1111111111111111111111111111111111111111", + Digest: "sha256:cafebabe", + }, + }, + } + if err := lock.Save(path); err != nil { + t.Fatal(err) + } + + // Header comment should be present. + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !containsBytes(raw, []byte("waza.lock")) { + t.Fatalf("expected header comment in lockfile:\n%s", raw) + } + + got, err := LoadLockfile(path) + if err != nil { + t.Fatal(err) + } + if len(got.Modules) != 2 { + t.Fatalf("got %d modules, want 2", len(got.Modules)) + } + // Sorted output check. + if got.Modules[0].Ref > got.Modules[1].Ref { + t.Fatalf("modules not sorted by ref: %+v", got.Modules) + } +} + +func TestLockfile_LookupUpsert(t *testing.T) { + lock := &Lockfile{SchemaVersion: LockfileSchemaVersion} + if lock.Lookup("missing") != nil { + t.Fatal("expected nil for missing ref") + } + entry := LockModule{Ref: "github.com/o/r#x@v1.0.0", Commit: "aaaa", Digest: "sha256:z"} + lock.Upsert(entry) + got := lock.Lookup(entry.Ref) + if got == nil || got.Commit != "aaaa" { + t.Fatalf("Lookup after Upsert failed: %+v", got) + } + // Replace. + entry.Commit = "bbbb" + lock.Upsert(entry) + if got := lock.Lookup(entry.Ref); got.Commit != "bbbb" { + t.Fatalf("Upsert did not replace: %+v", got) + } + if len(lock.Modules) != 1 { + t.Fatalf("expected 1 module after upsert-replace, got %d", len(lock.Modules)) + } +} + +func TestLoadLockfile_Missing(t *testing.T) { + got, err := LoadLockfile(filepath.Join(t.TempDir(), "waza.lock")) + if err != nil { + t.Fatalf("expected nil error for missing file, got %v", err) + } + if got != nil { + t.Fatal("expected nil lock for missing file") + } +} + +func TestLoadLockfile_BadSchema(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "waza.lock") + if err := os.WriteFile(path, []byte("schema_version: 99\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadLockfile(path); err == nil { + t.Fatal("expected schema_version error") + } +} + +func TestLockfilePath(t *testing.T) { + got := LockfilePath("/tmp/evals/eval.yaml") + want := "/tmp/evals/waza.lock" + if got != want { + t.Fatalf("LockfilePath = %q, want %q", got, want) + } +} + +func containsBytes(hay, needle []byte) bool { + return len(needle) == 0 || bytesIndex(hay, needle) >= 0 +} + +// bytesIndex is a tiny local Index to avoid importing bytes. +func bytesIndex(s, sep []byte) int { + n := len(sep) + if n == 0 { + return 0 + } + for i := 0; i+n <= len(s); i++ { + match := true + for j := 0; j < n; j++ { + if s[i+j] != sep[j] { + match = false + break + } + } + if match { + return i + } + } + return -1 +} diff --git a/internal/registry/manifest.go b/internal/registry/manifest.go new file mode 100644 index 00000000..623128fa --- /dev/null +++ b/internal/registry/manifest.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +// ManifestFileName is the name of the Waza module manifest at the repo root. +const ManifestFileName = "waza.registry.yaml" + +// Manifest is a minimal Phase 1 view of waza.registry.yaml. Only the fields +// needed to locate an exported grader preset are decoded; unknown fields are +// ignored so that future manifest additions do not break older Waza clients. +type Manifest struct { + SchemaVersion int `yaml:"schema_version"` + Module string `yaml:"module"` + Description string `yaml:"description,omitempty"` + License string `yaml:"license,omitempty"` + Exports ManifestExports `yaml:"exports,omitempty"` +} + +// ManifestExports holds the exported artifact tables. Phase 1 only reads +// graders. +type ManifestExports struct { + Graders map[string]ManifestExport `yaml:"graders,omitempty"` +} + +// ManifestExport describes one exported artifact. Only Path is required in +// Phase 1; the resolver reads the file at that path relative to the module root. +type ManifestExport struct { + Path string `yaml:"path"` + Description string `yaml:"description,omitempty"` + Tags []string `yaml:"tags,omitempty"` +} + +// ParseManifest parses a waza.registry.yaml file. +func ParseManifest(data []byte) (*Manifest, error) { + var m Manifest + if err := yaml.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parsing waza.registry.yaml: %w", err) + } + if m.SchemaVersion == 0 { + m.SchemaVersion = 1 + } + if m.SchemaVersion != 1 { + return nil, fmt.Errorf("waza.registry.yaml: unsupported schema_version %d (expected 1)", m.SchemaVersion) + } + return &m, nil +} + +// ResolveGraderPath returns the manifest-relative path to the grader preset +// selected by the ref. When the ref uses "#export" syntax, the export is looked +// up in the manifest; when the ref uses a subpath, the path is used directly +// (with a ".yaml" suffix appended when missing). +func (m *Manifest) ResolveGraderPath(ref Ref) (string, error) { + if ref.Export != "" { + if m == nil { + return "", fmt.Errorf("ref %q uses #export syntax but no waza.registry.yaml was found in the module", ref.Raw) + } + exp, ok := m.Exports.Graders[ref.Export] + if !ok { + return "", fmt.Errorf("ref %q: export %q not found in module manifest (available: %v)", ref.Raw, ref.Export, sortedGraderNames(m)) + } + if exp.Path == "" { + return "", fmt.Errorf("ref %q: export %q has empty path in manifest", ref.Raw, ref.Export) + } + return exp.Path, nil + } + if ref.Path == "" { + return "", fmt.Errorf("ref %q: must specify either an export (#name) or a subpath", ref.Raw) + } + p := ref.Path + // Users can write either "graders/factuality" or "graders/factuality.yaml". + if !hasYAMLSuffix(p) { + p += ".yaml" + } + return p, nil +} + +func hasYAMLSuffix(p string) bool { + return len(p) >= 5 && (p[len(p)-5:] == ".yaml" || p[len(p)-4:] == ".yml") +} + +func sortedGraderNames(m *Manifest) []string { + if m == nil { + return nil + } + names := make([]string, 0, len(m.Exports.Graders)) + for n := range m.Exports.Graders { + names = append(names, n) + } + return names +} diff --git a/internal/registry/ref.go b/internal/registry/ref.go new file mode 100644 index 00000000..4cb41182 --- /dev/null +++ b/internal/registry/ref.go @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +// Package registry implements Phase 1 of the Waza eval/grader registry: Go-module-style +// references, a resolver/cache, and a lockfile for reproducible remote grader presets. +// See docs/research/waza-eval-registry-design.md for the full design. +package registry + +import ( + "fmt" + "regexp" + "strings" +) + +// Ref is a parsed Waza module reference in Go-module style: +// +// //[/path][#export]@ +// +// Examples: +// +// github.com/waza-evals/fact#factuality@v1.0.0 +// github.com/waza-evals/fact/graders/factuality@v1.0.0 +// github.com/myorg/private-evals/security#secrets@v2.1.3 +type Ref struct { + // Raw is the original ref string as it appeared in YAML. + Raw string + // Host is the source host, e.g. "github.com". Phase 1 only supports github.com. + Host string + // Owner is the org or user, e.g. "waza-evals". + Owner string + // Repo is the repository name, e.g. "fact". + Repo string + // Path is the optional subpath inside the repo, e.g. "graders/factuality". + // Empty when using the "#export" syntax. + Path string + // Export is the optional export name from waza.registry.yaml. + // Empty when using the path syntax. + Export string + // Version is the version selector — a semver tag (v1.0.0) or full commit SHA. + // Phase 1 rejects floating selectors (branches, ranges) in eval.yaml. + Version string +} + +// Module returns the canonical module identity ("host/owner/repo") without +// path/export/version qualifiers. +func (r Ref) Module() string { + return r.Host + "/" + r.Owner + "/" + r.Repo +} + +// String returns the canonical form of the ref. +func (r Ref) String() string { + var b strings.Builder + b.WriteString(r.Module()) + if r.Path != "" { + b.WriteString("/") + b.WriteString(r.Path) + } + if r.Export != "" { + b.WriteString("#") + b.WriteString(r.Export) + } + if r.Version != "" { + b.WriteString("@") + b.WriteString(r.Version) + } + return b.String() +} + +// commitSHARE matches a 40-char lowercase hex commit SHA. +var commitSHARE = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// semverTagRE matches semver-style tags: v1.2.3, v1.2.3-rc1, v1.2.3+build, etc. +var semverTagRE = regexp.MustCompile(`^v\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$`) + +// ParseRef parses a canonical Waza ref. It rejects floating version selectors +// (branches, ranges) because Phase 1 only supports reproducible refs in eval.yaml. +func ParseRef(s string) (Ref, error) { + raw := strings.TrimSpace(s) + if raw == "" { + return Ref{}, fmt.Errorf("ref is empty") + } + + // Split off @version. + at := strings.LastIndex(raw, "@") + if at < 0 { + return Ref{}, fmt.Errorf("ref %q missing @version selector", raw) + } + head, version := raw[:at], raw[at+1:] + if version == "" { + return Ref{}, fmt.Errorf("ref %q has empty version", raw) + } + + // Split off #export. + var export string + if hash := strings.Index(head, "#"); hash >= 0 { + export = head[hash+1:] + head = head[:hash] + if export == "" { + return Ref{}, fmt.Errorf("ref %q has empty export after '#'", raw) + } + } + + // Split host/owner/repo[/path]. + parts := strings.Split(head, "/") + if len(parts) < 3 { + return Ref{}, fmt.Errorf("ref %q must be host/owner/repo[/path][#export]@version", raw) + } + host, owner, repo := parts[0], parts[1], parts[2] + if host == "" || owner == "" || repo == "" { + return Ref{}, fmt.Errorf("ref %q has empty host/owner/repo segment", raw) + } + // Phase 1: only github.com is supported. + if host != "github.com" { + return Ref{}, fmt.Errorf("ref %q: only github.com is supported in Phase 1 (got %q)", raw, host) + } + + path := strings.Join(parts[3:], "/") + if path != "" && export != "" { + return Ref{}, fmt.Errorf("ref %q cannot combine subpath and #export syntax", raw) + } + + // Version must be an exact tag or full commit SHA for reproducibility. + if !semverTagRE.MatchString(version) && !commitSHARE.MatchString(version) { + return Ref{}, fmt.Errorf("ref %q: version %q must be a semver tag (vX.Y.Z) or 40-char commit SHA (floating selectors are not allowed in eval.yaml)", raw, version) + } + + return Ref{ + Raw: raw, + Host: host, + Owner: owner, + Repo: repo, + Path: path, + Export: export, + Version: version, + }, nil +} + +// IsCommitSHA reports whether the ref version is already a pinned commit SHA +// (as opposed to a semver tag that still needs resolving). +func (r Ref) IsCommitSHA() bool { + return commitSHARE.MatchString(r.Version) +} diff --git a/internal/registry/ref_test.go b/internal/registry/ref_test.go new file mode 100644 index 00000000..e171b5ec --- /dev/null +++ b/internal/registry/ref_test.go @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "strings" + "testing" +) + +func TestParseRef_Valid(t *testing.T) { + tests := []struct { + name string + in string + want Ref + }{ + { + name: "export syntax with tag", + in: "github.com/waza-evals/fact#factuality@v1.0.0", + want: Ref{ + Host: "github.com", Owner: "waza-evals", Repo: "fact", + Export: "factuality", Version: "v1.0.0", + }, + }, + { + name: "subpath syntax with tag", + in: "github.com/waza-evals/fact/graders/factuality@v1.2.3-rc1", + want: Ref{ + Host: "github.com", Owner: "waza-evals", Repo: "fact", + Path: "graders/factuality", Version: "v1.2.3-rc1", + }, + }, + { + name: "pinned commit SHA", + in: "github.com/o/r#g@" + strings.Repeat("a", 40), + want: Ref{ + Host: "github.com", Owner: "o", Repo: "r", + Export: "g", Version: strings.Repeat("a", 40), + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseRef(tc.in) + if err != nil { + t.Fatalf("ParseRef(%q): %v", tc.in, err) + } + tc.want.Raw = tc.in + if got != tc.want { + t.Fatalf("ParseRef(%q) = %+v, want %+v", tc.in, got, tc.want) + } + }) + } +} + +func TestParseRef_Errors(t *testing.T) { + cases := map[string]string{ + "empty": "", + "no version": "github.com/o/r", + "empty version": "github.com/o/r@", + "non-github host": "gitlab.com/o/r#x@v1.0.0", + "floating branch": "github.com/o/r#x@main", + "floating range": "github.com/o/r#x@^v1.0.0", + "short sha": "github.com/o/r#x@abcdef1", + "path + export mixed": "github.com/o/r/sub#x@v1.0.0", + "missing owner/repo": "github.com/o@v1.0.0", + "empty export": "github.com/o/r#@v1.0.0", + } + for name, in := range cases { + t.Run(name, func(t *testing.T) { + if _, err := ParseRef(in); err == nil { + t.Fatalf("ParseRef(%q) expected error, got nil", in) + } + }) + } +} + +func TestRef_IsCommitSHA(t *testing.T) { + sha := strings.Repeat("f", 40) + r, err := ParseRef("github.com/o/r#g@" + sha) + if err != nil { + t.Fatal(err) + } + if !r.IsCommitSHA() { + t.Fatal("expected IsCommitSHA true") + } + r2, err := ParseRef("github.com/o/r#g@v1.0.0") + if err != nil { + t.Fatal(err) + } + if r2.IsCommitSHA() { + t.Fatal("expected IsCommitSHA false for tag") + } +} + +func TestRef_StringRoundTrip(t *testing.T) { + in := "github.com/o/r#g@v1.0.0" + r, err := ParseRef(in) + if err != nil { + t.Fatal(err) + } + if got := r.String(); got != in { + t.Fatalf("String() = %q, want %q", got, in) + } +} diff --git a/internal/registry/resolver.go b/internal/registry/resolver.go new file mode 100644 index 00000000..0036493f --- /dev/null +++ b/internal/registry/resolver.go @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +// ErrRefNotInLock is returned when a ref is required by eval.yaml but has no +// entry in waza.lock and the caller is running in strict (verify-only) mode. +var ErrRefNotInLock = errors.New("ref not in waza.lock; run `waza get` to resolve") + +// ErrDigestMismatch is returned when cached or freshly-fetched content does +// not match the digest recorded in the lockfile. +var ErrDigestMismatch = errors.New("content digest does not match lockfile") + +// ResolvedGrader is the output of resolving one grader-preset ref: the raw +// YAML bytes of the remote preset plus the lock entry that pinned it. +type ResolvedGrader struct { + Ref Ref + Lock LockModule + PresetYAML []byte +} + +// Fetcher abstracts the source backend (GitHub over HTTP by default) so tests +// can inject a mock. All methods take a context so callers can enforce timeouts. +type Fetcher interface { + // ResolveCommit turns a version selector (semver tag OR commit SHA) into + // a concrete 40-char commit SHA. Implementations should short-circuit + // when the version is already a SHA. + ResolveCommit(ctx context.Context, ref Ref) (string, error) + // FetchFile downloads a single file from the module at the given commit + // SHA. Path is relative to the repo root. + FetchFile(ctx context.Context, ref Ref, commit, path string) ([]byte, error) +} + +// Resolver expands refs into concrete grader-preset YAML using a Fetcher, +// content-addressed disk cache, and lockfile for reproducibility. +type Resolver struct { + Fetcher Fetcher + CacheDir string + // Now is injected for deterministic tests. + Now func() time.Time +} + +// NewResolver returns a Resolver with the default HTTP-backed GitHub fetcher +// and cache location. +func NewResolver() (*Resolver, error) { + cache, err := DefaultCacheDir() + if err != nil { + return nil, err + } + return &Resolver{ + Fetcher: &GitHubFetcher{HTTP: http.DefaultClient}, + CacheDir: cache, + Now: time.Now, + }, nil +} + +// DefaultCacheDir returns the module cache root: ~/.waza/cache. +// (The design doc suggests $XDG_CACHE_HOME; ~/.waza/cache is chosen for Phase 1 +// per issue #15's spec so users have one predictable location.) +func DefaultCacheDir() (string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolving home dir for module cache: %w", err) + } + return filepath.Join(home, ".waza", "cache"), nil +} + +// cachePath returns the on-disk location for a module at the given commit. +func (r *Resolver) cachePath(ref Ref, commit string) string { + return filepath.Join(r.CacheDir, ref.Host, ref.Owner, ref.Repo, commit) +} + +// Resolve returns the resolved grader for the given ref, using the lockfile +// to enforce reproducibility. +// +// Behavior: +// - If lock has an entry, use its pinned commit + digest. Cache miss triggers +// a download; digest mismatch returns ErrDigestMismatch. +// - If lock has no entry and updateLock is false, returns ErrRefNotInLock. +// - If updateLock is true, resolves the version to a commit, fetches, computes +// the digest, and writes/updates the lockfile in memory (caller persists). +func (r *Resolver) Resolve(ctx context.Context, ref Ref, lock *Lockfile, updateLock bool) (*ResolvedGrader, error) { + // Fast path: locked entry. + if entry := lock.Lookup(ref.Raw); entry != nil { + return r.resolveLocked(ctx, ref, *entry) + } + + if !updateLock { + return nil, fmt.Errorf("%w: %s", ErrRefNotInLock, ref.Raw) + } + + // Update path: resolve version -> commit and fetch fresh. + commit, err := r.Fetcher.ResolveCommit(ctx, ref) + if err != nil { + return nil, fmt.Errorf("resolving commit for %s: %w", ref.Raw, err) + } + if !commitSHARE.MatchString(commit) { + return nil, fmt.Errorf("resolving commit for %s: fetcher returned invalid SHA %q", ref.Raw, commit) + } + + presetPath, err := r.fetchAndCacheModule(ctx, ref, commit) + if err != nil { + return nil, err + } + presetYAML, err := os.ReadFile(presetPath) + if err != nil { + return nil, fmt.Errorf("reading cached preset %s: %w", presetPath, err) + } + digest := computeDigest(presetYAML) + url := githubRawURL(ref, commit, "") // module-relative URL noted below + entry := LockModule{ + Ref: ref.Raw, + Module: ref.Module(), + Version: ref.Version, + Commit: commit, + Digest: digest, + URL: url, + ResolvedAt: r.Now(), + } + lock.Upsert(entry) + return &ResolvedGrader{Ref: ref, Lock: entry, PresetYAML: presetYAML}, nil +} + +// resolveLocked verifies a locked ref's cached content matches the recorded +// digest, fetching from the source if the cache is cold. +func (r *Resolver) resolveLocked(ctx context.Context, ref Ref, entry LockModule) (*ResolvedGrader, error) { + if !commitSHARE.MatchString(entry.Commit) { + return nil, fmt.Errorf("lock entry for %s: invalid commit SHA %q", ref.Raw, entry.Commit) + } + + // Determine the preset path relative to the module root. + manifest, err := r.loadCachedManifest(ctx, ref, entry.Commit) + if err != nil { + return nil, err + } + presetRel, err := manifest.ResolveGraderPath(ref) + if err != nil { + return nil, err + } + + dir := r.cachePath(ref, entry.Commit) + presetPath := filepath.Join(dir, "source", presetRel) + presetYAML, err := os.ReadFile(presetPath) + if err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("reading cached preset %s: %w", presetPath, err) + } + // Cache miss for this file: refetch just the preset. + data, ferr := r.Fetcher.FetchFile(ctx, ref, entry.Commit, presetRel) + if ferr != nil { + return nil, fmt.Errorf("fetching preset %s: %w", ref.Raw, ferr) + } + if err := writeCacheFile(presetPath, data); err != nil { + return nil, err + } + presetYAML = data + } + + got := computeDigest(presetYAML) + if got != entry.Digest { + return nil, fmt.Errorf("%w: ref %s: cached digest %s does not match lockfile %s", ErrDigestMismatch, ref.Raw, got, entry.Digest) + } + return &ResolvedGrader{Ref: ref, Lock: entry, PresetYAML: presetYAML}, nil +} + +// fetchAndCacheModule downloads waza.registry.yaml (if the ref uses #export) +// and the target preset file into the cache, returning the on-disk path of +// the preset. +func (r *Resolver) fetchAndCacheModule(ctx context.Context, ref Ref, commit string) (string, error) { + dir := r.cachePath(ref, commit) + if err := os.MkdirAll(filepath.Join(dir, "source"), 0o755); err != nil { + return "", fmt.Errorf("creating cache dir %s: %w", dir, err) + } + + // Fetch the manifest when needed to resolve export -> path. + var manifest *Manifest + if ref.Export != "" { + manifestBytes, err := r.Fetcher.FetchFile(ctx, ref, commit, ManifestFileName) + if err != nil { + return "", fmt.Errorf("fetching %s: %w", ManifestFileName, err) + } + if err := writeCacheFile(filepath.Join(dir, "source", ManifestFileName), manifestBytes); err != nil { + return "", err + } + manifest, err = ParseManifest(manifestBytes) + if err != nil { + return "", err + } + } + + presetRel, err := manifest.ResolveGraderPath(ref) + if err != nil { + return "", err + } + presetBytes, err := r.Fetcher.FetchFile(ctx, ref, commit, presetRel) + if err != nil { + return "", fmt.Errorf("fetching preset %s: %w", presetRel, err) + } + presetPath := filepath.Join(dir, "source", presetRel) + if err := writeCacheFile(presetPath, presetBytes); err != nil { + return "", err + } + return presetPath, nil +} + +// loadCachedManifest loads the manifest from cache, falling back to fetching +// if it is missing. Returns nil manifest when the ref does not use #export. +func (r *Resolver) loadCachedManifest(ctx context.Context, ref Ref, commit string) (*Manifest, error) { + if ref.Export == "" { + return nil, nil + } + manifestPath := filepath.Join(r.cachePath(ref, commit), "source", ManifestFileName) + data, err := os.ReadFile(manifestPath) + if err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("reading cached manifest %s: %w", manifestPath, err) + } + fetched, ferr := r.Fetcher.FetchFile(ctx, ref, commit, ManifestFileName) + if ferr != nil { + return nil, fmt.Errorf("fetching %s: %w", ManifestFileName, ferr) + } + if err := writeCacheFile(manifestPath, fetched); err != nil { + return nil, err + } + data = fetched + } + return ParseManifest(data) +} + +func writeCacheFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("creating cache dir for %s: %w", path, err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("writing cache file %s: %w", path, err) + } + return nil +} + +// computeDigest returns "sha256:" of the given content. +func computeDigest(data []byte) string { + sum := sha256.Sum256(data) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// GitHubFetcher implements Fetcher against github.com using the public REST +// and raw-content endpoints. Private repos work when GH_TOKEN or GITHUB_TOKEN +// is set in the environment; no other authentication is attempted. +type GitHubFetcher struct { + HTTP *http.Client + // TokenEnv is optional override for the env vars checked for auth. + // Defaults to ["GH_TOKEN", "GITHUB_TOKEN"]. + TokenEnv []string + // BaseAPI overrides the API base URL (for testing). + BaseAPI string + // BaseRaw overrides the raw content base URL (for testing). + BaseRaw string +} + +func (g *GitHubFetcher) client() *http.Client { + if g.HTTP != nil { + return g.HTTP + } + return http.DefaultClient +} + +func (g *GitHubFetcher) token() string { + envs := g.TokenEnv + if len(envs) == 0 { + envs = []string{"GH_TOKEN", "GITHUB_TOKEN"} + } + for _, e := range envs { + if v := os.Getenv(e); v != "" { + return v + } + } + return "" +} + +// ResolveCommit calls the GitHub commits API to resolve a tag or ref to a SHA. +func (g *GitHubFetcher) ResolveCommit(ctx context.Context, ref Ref) (string, error) { + if ref.IsCommitSHA() { + return ref.Version, nil + } + base := g.BaseAPI + if base == "" { + base = "https://api.github.com" + } + // Using /repos/{owner}/{repo}/commits/{ref} lets GitHub resolve tags, + // branches, and short SHAs to a full commit. + url := fmt.Sprintf("%s/repos/%s/%s/commits/%s", base, ref.Owner, ref.Repo, ref.Version) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + if tok := g.token(); tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + resp, err := g.client().Do(req) + if err != nil { + return "", fmt.Errorf("GET %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return "", fmt.Errorf("GET %s: HTTP %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body))) + } + var payload struct { + SHA string `json:"sha"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return "", fmt.Errorf("parsing commits response: %w", err) + } + if payload.SHA == "" { + return "", fmt.Errorf("commits API returned empty sha for %s", ref.Raw) + } + return payload.SHA, nil +} + +// FetchFile downloads a file from github.com's raw content endpoint. +func (g *GitHubFetcher) FetchFile(ctx context.Context, ref Ref, commit, path string) ([]byte, error) { + url := githubRawURL(ref, commit, path) + if g.BaseRaw != "" { + url = fmt.Sprintf("%s/%s/%s/%s/%s", strings.TrimRight(g.BaseRaw, "/"), ref.Owner, ref.Repo, commit, path) + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + if tok := g.token(); tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + resp, err := g.client().Do(req) + if err != nil { + return nil, fmt.Errorf("GET %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("%s: not found at commit %s (path %s)", ref.Raw, commit, path) + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return nil, fmt.Errorf("GET %s: HTTP %d: %s", url, resp.StatusCode, strings.TrimSpace(string(body))) + } + return io.ReadAll(resp.Body) +} + +// githubRawURL builds the raw.githubusercontent.com URL for a file at a commit. +func githubRawURL(ref Ref, commit, path string) string { + if path == "" { + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/", ref.Owner, ref.Repo, commit) + } + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", ref.Owner, ref.Repo, commit, path) +} diff --git a/internal/registry/resolver_test.go b/internal/registry/resolver_test.go new file mode 100644 index 00000000..0152b4fd --- /dev/null +++ b/internal/registry/resolver_test.go @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "path" + "strings" + "testing" + "time" +) + +// mockGitHub returns a resolver wired to two httptest servers that emulate the +// GitHub API and raw content endpoints. The returned map of file bodies is +// consulted by the raw server; keys are "///". +func mockGitHub(t *testing.T, commit string, files map[string]string) (*Resolver, func()) { + t.Helper() + + apiHits := map[string]int{} + rawHits := map[string]int{} + + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + apiHits[r.URL.Path]++ + // Only the commits API is needed. + if !strings.HasPrefix(r.URL.Path, "/repos/") { + http.Error(w, "not implemented", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"sha":%q}`, commit) + })) + + raw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/") + rawHits[key]++ + body, ok := files[key] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/plain") + _, _ = fmt.Fprint(w, body) + })) + + fetcher := &GitHubFetcher{ + HTTP: http.DefaultClient, + BaseAPI: api.URL, + BaseRaw: raw.URL, + } + res := &Resolver{ + Fetcher: fetcher, + CacheDir: t.TempDir(), + Now: func() time.Time { return time.Unix(0, 0).UTC() }, + } + teardown := func() { + api.Close() + raw.Close() + } + return res, teardown +} + +func TestResolve_TagFetchDigestAndCache(t *testing.T) { + commit := strings.Repeat("a", 40) + preset := "type: code\nname: factuality\nconfig:\n x: 1\n" + manifest := "schema_version: 1\nmodule: github.com/waza-evals/fact\nexports:\n graders:\n factuality:\n path: graders/factuality.yaml\n" + + files := map[string]string{ + "waza-evals/fact/" + commit + "/waza.registry.yaml": manifest, + "waza-evals/fact/" + commit + "/graders/factuality.yaml": preset, + } + res, teardown := mockGitHub(t, commit, files) + defer teardown() + + ref, err := ParseRef("github.com/waza-evals/fact#factuality@v1.0.0") + if err != nil { + t.Fatal(err) + } + lock := &Lockfile{SchemaVersion: LockfileSchemaVersion} + got, err := res.Resolve(context.Background(), ref, lock, true) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + + if got.Lock.Commit != commit { + t.Fatalf("commit = %q, want %q", got.Lock.Commit, commit) + } + sum := sha256.Sum256([]byte(preset)) + wantDigest := "sha256:" + hex.EncodeToString(sum[:]) + if got.Lock.Digest != wantDigest { + t.Fatalf("digest = %q, want %q", got.Lock.Digest, wantDigest) + } + if string(got.PresetYAML) != preset { + t.Fatalf("PresetYAML mismatch") + } + + // Second resolve uses the lock — cache hit path, no digest error. + got2, err := res.Resolve(context.Background(), ref, lock, false) + if err != nil { + t.Fatalf("second Resolve: %v", err) + } + if got2.Lock.Digest != wantDigest { + t.Fatalf("cached digest mismatch") + } +} + +func TestResolve_RefNotInLock(t *testing.T) { + res := &Resolver{ + Fetcher: &stubFetcher{}, + CacheDir: t.TempDir(), + Now: time.Now, + } + ref, _ := ParseRef("github.com/o/r#x@v1.0.0") + lock := &Lockfile{SchemaVersion: LockfileSchemaVersion} + _, err := res.Resolve(context.Background(), ref, lock, false) + if !errors.Is(err, ErrRefNotInLock) { + t.Fatalf("expected ErrRefNotInLock, got %v", err) + } +} + +func TestResolve_DigestMismatch(t *testing.T) { + commit := strings.Repeat("b", 40) + presetGood := "type: code\nname: g\n" + manifest := "schema_version: 1\nmodule: github.com/o/r\nexports:\n graders:\n g:\n path: g.yaml\n" + files := map[string]string{ + "o/r/" + commit + "/waza.registry.yaml": manifest, + "o/r/" + commit + "/g.yaml": presetGood, + } + res, teardown := mockGitHub(t, commit, files) + defer teardown() + + ref, _ := ParseRef("github.com/o/r#g@" + commit) + // Pre-seed lock with a bad digest for the same commit. + lock := &Lockfile{ + SchemaVersion: LockfileSchemaVersion, + Modules: []LockModule{{ + Ref: ref.Raw, Module: ref.Module(), Version: ref.Version, + Commit: commit, Digest: "sha256:0000", + }}, + } + _, err := res.Resolve(context.Background(), ref, lock, false) + if !errors.Is(err, ErrDigestMismatch) { + t.Fatalf("expected ErrDigestMismatch, got %v", err) + } +} + +func TestResolve_SubpathNoManifest(t *testing.T) { + commit := strings.Repeat("c", 40) + preset := "type: text\nname: sub\n" + files := map[string]string{ + "o/r/" + commit + "/graders/sub.yaml": preset, + } + res, teardown := mockGitHub(t, commit, files) + defer teardown() + + // Subpath form — no manifest lookup should be attempted. + ref, err := ParseRef("github.com/o/r/graders/sub@v1.0.0") + if err != nil { + t.Fatal(err) + } + lock := &Lockfile{SchemaVersion: LockfileSchemaVersion} + got, err := res.Resolve(context.Background(), ref, lock, true) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if string(got.PresetYAML) != preset { + t.Fatalf("PresetYAML mismatch: %q", got.PresetYAML) + } + if got.Lock.Commit != commit { + t.Fatalf("commit mismatch") + } +} + +func TestDefaultCacheDir(t *testing.T) { + got, err := DefaultCacheDir() + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, path.Join(".waza", "cache")) { + t.Fatalf("DefaultCacheDir() = %q, want to contain .waza/cache", got) + } +} + +type stubFetcher struct{} + +func (stubFetcher) ResolveCommit(_ context.Context, _ Ref) (string, error) { + return "", errors.New("not implemented") +} +func (stubFetcher) FetchFile(_ context.Context, _ Ref, _, _ string) ([]byte, error) { + return nil, errors.New("not implemented") +} diff --git a/internal/registry/spec_resolve.go b/internal/registry/spec_resolve.go new file mode 100644 index 00000000..00d708c0 --- /dev/null +++ b/internal/registry/spec_resolve.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "context" + "errors" + "fmt" + + "github.com/microsoft/waza/internal/models" +) + +// ResolveSpec expands all `ref:` grader entries in spec against the lockfile +// beside evalPath. It returns the number of refs that were resolved and +// whether the lockfile was modified (so callers can decide to persist it). +// +// Behavior: +// - If updateLock is false: every ref must already be in the lock, and the +// cached content must match the recorded digest. Unlocked refs return +// ErrRefNotInLock. +// - If updateLock is true: missing entries are resolved and added to lock. +// +// Non-ref graders are left untouched. +func ResolveSpec(ctx context.Context, spec *models.EvalSpec, evalPath string, updateLock bool) (resolved int, lockChanged bool, err error) { + if spec == nil { + return 0, false, errors.New("resolve: spec is nil") + } + refCount := 0 + for _, g := range spec.Graders { + if g.Ref != "" { + refCount++ + } + } + if refCount == 0 { + return 0, false, nil + } + + lockPath := LockfilePath(evalPath) + lock, err := LoadLockfile(lockPath) + if err != nil { + return 0, false, err + } + if lock == nil { + lock = &Lockfile{SchemaVersion: LockfileSchemaVersion} + } + + res, err := NewResolver() + if err != nil { + return 0, false, err + } + + before := snapshotLock(lock) + + for i, g := range spec.Graders { + if g.Ref == "" { + continue + } + ref, err := ParseRef(g.Ref) + if err != nil { + return resolved, false, fmt.Errorf("grader[%d]: %w", i, err) + } + got, err := res.Resolve(ctx, ref, lock, updateLock) + if err != nil { + return resolved, false, fmt.Errorf("grader[%d] %s: %w", i, ref.Raw, err) + } + expanded, err := ExpandGraderConfig(g, got) + if err != nil { + return resolved, false, fmt.Errorf("grader[%d] %s: %w", i, ref.Raw, err) + } + spec.Graders[i] = expanded + resolved++ + } + + lockChanged = !lockSnapshotEqual(before, snapshotLock(lock)) + if updateLock && lockChanged { + if err := lock.Save(lockPath); err != nil { + return resolved, lockChanged, err + } + } + return resolved, lockChanged, nil +} + +// snapshotLock captures a comparable representation of lock entries for +// change detection. +type lockSnap struct { + ref, commit, digest string +} + +func snapshotLock(l *Lockfile) []lockSnap { + if l == nil { + return nil + } + out := make([]lockSnap, 0, len(l.Modules)) + for _, m := range l.Modules { + out = append(out, lockSnap{ref: m.Ref, commit: m.Commit, digest: m.Digest}) + } + return out +} + +func lockSnapshotEqual(a, b []lockSnap) bool { + if len(a) != len(b) { + return false + } + // Build map keyed by ref for order independence. + m := make(map[string]lockSnap, len(a)) + for _, e := range a { + m[e.ref] = e + } + for _, e := range b { + if got, ok := m[e.ref]; !ok || got != e { + return false + } + } + return true +}