diff --git a/cmd/waza/cmd_registry.go b/cmd/waza/cmd_registry.go new file mode 100644 index 00000000..d275ff72 --- /dev/null +++ b/cmd/waza/cmd_registry.go @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package main + +import ( + "github.com/spf13/cobra" +) + +// newRegistryCommand builds the `waza registry` parent command tree. +// +// Phase 1 (issue #17) ships the `search` and `add` subcommands. Full +// end-to-end functionality — actual index HTTP calls and ref +// resolution — depends on issue #15's resolver, which is stubbed here +// with clear TODOs. +func newRegistryCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "registry", + Short: "Discover and add remote grader presets and eval modules", + Long: `Manage Waza registry sources for composable eval construction. + +The registry lets you discover reusable graders, evals, and datasets +published to the waza-evals GitHub org (or any additional registry you +configure) and add them to your eval.yaml with a single command. + +See docs/research/waza-eval-registry-design.md for the full design.`, + } + + cmd.AddCommand(newRegistrySearchCommand()) + cmd.AddCommand(newRegistryAddCommand()) + + return cmd +} diff --git a/cmd/waza/cmd_registry_add.go b/cmd/waza/cmd_registry_add.go new file mode 100644 index 00000000..88982a97 --- /dev/null +++ b/cmd/waza/cmd_registry_add.go @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package main + +import ( + "bufio" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/microsoft/waza/internal/registry" + "github.com/spf13/cobra" +) + +type registryAddFlags struct { + evalPath string + name string + sets []string + weight float64 + allowExec bool + dryRun bool + yes bool +} + +func newRegistryAddCommand() *cobra.Command { + f := ®istryAddFlags{} + cmd := &cobra.Command{ + Use: "add ", + Short: "Add a registry artifact to eval.yaml and update waza.lock", + Long: `Resolve a registry ref, append it to eval.yaml as a "ref:" grader +entry, and update waza.lock with the resolved commit and digest. + +Program graders (executable artifacts) are refused unless the caller +passes --allow-exec or confirms the interactive prompt. + +Examples: + waza registry add github.com/waza-evals/fact#factuality@v1.0.0 + waza registry add github.com/waza-evals/fact#factuality@v1.0.0 --eval eval.yaml + waza registry add github.com/waza-evals/fact#factuality@v1.0.0 \ + --name factuality_strict --set config.threshold=0.9`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runRegistryAdd(cmd.OutOrStdout(), cmd.InOrStdin(), args[0], *f) + }, + } + + cmd.Flags().StringVar(&f.evalPath, "eval", "eval.yaml", "Path to the eval file to modify") + cmd.Flags().StringVar(&f.name, "name", "", "Local alias for the grader (overrides remote default)") + cmd.Flags().StringSliceVar(&f.sets, "set", nil, "Config overrides, key.path=value (repeatable)") + cmd.Flags().Float64Var(&f.weight, "weight", 0, "Grader weight override (0 keeps default)") + cmd.Flags().BoolVar(&f.allowExec, "allow-exec", false, "Allow adding program-grader artifacts without prompting") + cmd.Flags().BoolVar(&f.dryRun, "dry-run", false, "Print planned changes without writing files") + cmd.Flags().BoolVarP(&f.yes, "yes", "y", false, "Assume yes to interactive prompts") + + return cmd +} + +func runRegistryAdd(out io.Writer, in io.Reader, refStr string, f registryAddFlags) error { + if !registry.IsRemote(refStr) { + return fmt.Errorf("%q is not a registry ref (expected host/owner/repo#export@version)", refStr) + } + ref, err := registry.ParseRef(refStr) + if err != nil { + return err + } + + config, err := registry.ParseSetFlag(f.sets) + if err != nil { + return err + } + + // TODO(#15): call the real resolver. For now the stub returns + // syntax-derived metadata so we can still write the ref entry. + resolver := registry.StubResolver{} + resolution, err := resolver.Resolve(ref) + if err != nil { + return fmt.Errorf("resolving %s: %w", ref, err) + } + + if resolution.Kind == registry.KindProgramGrader { + if !f.allowExec && !f.yes { + ok, err := confirmProgramGrader(out, in, ref.String()) + if err != nil { + return err + } + if !ok { + return errors.New("aborted by user; re-run with --allow-exec to skip the prompt") + } + } + resolution.Trusted = true + } + + entry := registry.GraderRefEntry{ + Ref: ref.String(), + Name: f.name, + Weight: f.weight, + Config: config, + } + + if f.dryRun { + return printAddDryRun(out, f.evalPath, entry, resolution) + } + + evalPath, err := filepath.Abs(f.evalPath) + if err != nil { + return fmt.Errorf("resolving eval path: %w", err) + } + if err := registry.AppendGraderRef(evalPath, entry); err != nil { + return err + } + + lockPath := filepath.Join(filepath.Dir(evalPath), registry.LockFileName) + lf, err := registry.LoadLockFile(lockPath) + if err != nil { + return err + } + lockEntry := registry.EntryFromResolution(resolution) + lf.Upsert(lockEntry) + if err := lf.Save(lockPath); err != nil { + return err + } + + fmt.Fprintf(out, "Added grader %s to %s\n", ref.String(), f.evalPath) //nolint:errcheck + fmt.Fprintf(out, "Updated %s\n", registry.LockFileName) //nolint:errcheck + // TODO(#15): print resolved commit + digest once the real resolver + // returns them. + return nil +} + +func confirmProgramGrader(out io.Writer, in io.Reader, ref string) (bool, error) { + fmt.Fprintf(out, "%s is a program grader (executable). Trust and add? [y/N]: ", ref) //nolint:errcheck + reader := bufio.NewReader(in) + line, err := reader.ReadString('\n') + if err != nil && err != io.EOF { + return false, fmt.Errorf("reading confirmation: %w", err) + } + line = strings.TrimSpace(strings.ToLower(line)) + return line == "y" || line == "yes", nil +} + +func printAddDryRun(out io.Writer, evalPath string, entry registry.GraderRefEntry, res registry.Resolution) error { + fmt.Fprintf(out, "DRY RUN: would add grader to %s:\n", evalPath) //nolint:errcheck + fmt.Fprintf(out, " ref: %s\n", entry.Ref) //nolint:errcheck + if entry.Name != "" { + fmt.Fprintf(out, " name: %s\n", entry.Name) //nolint:errcheck + } + if entry.Weight != 0 { + fmt.Fprintf(out, " weight: %g\n", entry.Weight) //nolint:errcheck + } + if len(entry.Config) > 0 { + fmt.Fprintln(out, " config:") //nolint:errcheck + for k, v := range entry.Config { + fmt.Fprintf(out, " %s: %v\n", k, v) //nolint:errcheck + } + } + fmt.Fprintf(out, "DRY RUN: would update %s with module %s@%s\n", registry.LockFileName, res.Module, res.Version) //nolint:errcheck + return nil +} diff --git a/cmd/waza/cmd_registry_add_test.go b/cmd/waza/cmd_registry_add_test.go new file mode 100644 index 00000000..0ebc28b1 --- /dev/null +++ b/cmd/waza/cmd_registry_add_test.go @@ -0,0 +1,132 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func writeEvalYAML(t *testing.T, dir, content string) string { + t.Helper() + path := filepath.Join(dir, "eval.yaml") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +func TestRegistryAddAppendsGraderAndWritesLock(t *testing.T) { + dir := t.TempDir() + evalPath := writeEvalYAML(t, dir, "name: my-eval\nversion: 1\n") + + cmd := newRegistryCommand() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "add", "github.com/waza-evals/fact#factuality@v1.0.0", + "--eval", evalPath, + "--name", "factuality_strict", + "--set", "config.threshold=0.9", + }) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v\n%s", err, buf.String()) + } + + data, err := os.ReadFile(evalPath) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("re-parse eval.yaml: %v\n%s", err, data) + } + graders, ok := doc["graders"].([]any) + if !ok || len(graders) != 1 { + t.Fatalf("graders sequence missing: %#v", doc["graders"]) + } + g, ok := graders[0].(map[string]any) + if !ok { + t.Fatalf("grader entry not a map: %#v", graders[0]) + } + if g["ref"] != "github.com/waza-evals/fact#factuality@v1.0.0" { + t.Errorf("ref: %v", g["ref"]) + } + if g["name"] != "factuality_strict" { + t.Errorf("name: %v", g["name"]) + } + + lockPath := filepath.Join(dir, "waza.lock") + lockData, err := os.ReadFile(lockPath) + if err != nil { + t.Fatalf("read waza.lock: %v", err) + } + if !strings.Contains(string(lockData), "github.com/waza-evals/fact#factuality@v1.0.0") { + t.Errorf("lock missing ref:\n%s", lockData) + } + if !strings.Contains(string(lockData), "schema_version: 1") { + t.Errorf("lock missing schema_version:\n%s", lockData) + } +} + +func TestRegistryAddDryRun(t *testing.T) { + dir := t.TempDir() + evalPath := writeEvalYAML(t, dir, "name: e\n") + + cmd := newRegistryCommand() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "add", "github.com/waza-evals/fact#factuality@v1.0.0", + "--eval", evalPath, + "--dry-run", + }) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + if !strings.Contains(buf.String(), "DRY RUN") { + t.Errorf("expected DRY RUN in output:\n%s", buf.String()) + } + // Eval file must be unchanged. + data, _ := os.ReadFile(evalPath) + if strings.Contains(string(data), "ref:") { + t.Errorf("dry run modified eval.yaml:\n%s", data) + } + if _, err := os.Stat(filepath.Join(dir, "waza.lock")); !os.IsNotExist(err) { + t.Errorf("dry run wrote waza.lock") + } +} + +func TestRegistryAddRejectsBadRef(t *testing.T) { + cmd := newRegistryCommand() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetArgs([]string{"add", "./local.yaml"}) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error for local path") + } +} + +func TestRegistryAddRejectsBadSetFlag(t *testing.T) { + dir := t.TempDir() + evalPath := writeEvalYAML(t, dir, "name: e\n") + cmd := newRegistryCommand() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetArgs([]string{ + "add", "github.com/waza-evals/fact#factuality@v1.0.0", + "--eval", evalPath, + "--set", "malformed", + }) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error for malformed --set") + } +} diff --git a/cmd/waza/cmd_registry_search.go b/cmd/waza/cmd_registry_search.go new file mode 100644 index 00000000..e9612dbd --- /dev/null +++ b/cmd/waza/cmd_registry_search.go @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package main + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "text/tabwriter" + + "github.com/microsoft/waza/internal/registry" + "github.com/spf13/cobra" +) + +type registrySearchFlags struct { + kind string + registry string + format string +} + +func newRegistrySearchCommand() *cobra.Command { + f := ®istrySearchFlags{} + cmd := &cobra.Command{ + Use: "search ", + Short: "Search configured registry indexes", + Long: `Search configured registry indexes for graders, evals, and datasets. + +Results are printed as a human-readable table by default, or as JSON with +--format json for automation. Multiple sources are consulted in priority +order; duplicates are collapsed by canonical ref. + +Examples: + waza registry search factual + waza registry search factual --kind grader + waza registry search factual --registry public + waza registry search factual --format json`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var query string + if len(args) == 1 { + query = args[0] + } + return runRegistrySearch(cmd.OutOrStdout(), query, *f) + }, + } + + cmd.Flags().StringVar(&f.kind, "kind", "", "Filter results by artifact kind (grader|eval|dataset)") + cmd.Flags().StringVar(&f.registry, "registry", "", "Restrict search to a single configured registry source by name") + cmd.Flags().StringVar(&f.format, "format", "table", "Output format: table|json") + + return cmd +} + +func runRegistrySearch(out io.Writer, query string, f registrySearchFlags) error { + if err := validateSearchFlags(f); err != nil { + return err + } + + // TODO(#15): load user-supplied waza.registry.yaml when present. + // For Phase 1 we use the default public registry only. + cfg := registry.DefaultConfig() + searcher := registry.NewSearcher(cfg) + + opts := registry.SearchOptions{ + Query: query, + Kind: registry.Kind(f.kind), + Registry: f.registry, + } + results, err := searcher.Search(opts) + if err != nil { + return fmt.Errorf("registry search: %w", err) + } + + switch strings.ToLower(f.format) { + case "json": + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + return enc.Encode(results) + case "table", "": + return writeSearchTable(out, results) + default: + return fmt.Errorf("unsupported --format %q (want table|json)", f.format) + } +} + +func validateSearchFlags(f registrySearchFlags) error { + if f.kind != "" { + switch registry.Kind(f.kind) { + case registry.KindGrader, registry.KindEval, registry.KindDataset, registry.KindProgramGrader: + default: + return fmt.Errorf("unsupported --kind %q (want grader|eval|dataset)", f.kind) + } + } + if f.format != "" { + switch strings.ToLower(f.format) { + case "table", "json": + default: + return fmt.Errorf("unsupported --format %q (want table|json)", f.format) + } + } + return nil +} + +func writeSearchTable(out io.Writer, results []registry.SearchResult) error { + if len(results) == 0 { + fmt.Fprintln(out, "No matching results.") //nolint:errcheck + return nil + } + tw := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "REF\tKIND\tSTARS\tDESCRIPTION") //nolint:errcheck + fmt.Fprintln(tw, "---\t----\t-----\t-----------") //nolint:errcheck + for _, r := range results { + fmt.Fprintf(tw, "%s\t%s\t%d\t%s\n", r.Ref, r.Kind, r.Stars, r.Description) //nolint:errcheck + } + return tw.Flush() +} diff --git a/cmd/waza/cmd_registry_search_test.go b/cmd/waza/cmd_registry_search_test.go new file mode 100644 index 00000000..f28d2ff6 --- /dev/null +++ b/cmd/waza/cmd_registry_search_test.go @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package main + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestRegistrySearchTable(t *testing.T) { + cmd := newRegistryCommand() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"search", "factuality"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + out := buf.String() + if !strings.Contains(out, "REF") || !strings.Contains(out, "KIND") { + t.Errorf("table header missing:\n%s", out) + } + if !strings.Contains(out, "factuality") { + t.Errorf("expected factuality result:\n%s", out) + } +} + +func TestRegistrySearchJSON(t *testing.T) { + cmd := newRegistryCommand() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"search", "factuality", "--format", "json"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + var results []map[string]any + if err := json.Unmarshal(buf.Bytes(), &results); err != nil { + t.Fatalf("json decode: %v\n%s", err, buf.String()) + } + if len(results) == 0 { + t.Fatal("expected results") + } + if _, ok := results[0]["ref"]; !ok { + t.Errorf("missing 'ref' field: %#v", results[0]) + } +} + +func TestRegistrySearchKindValidation(t *testing.T) { + cmd := newRegistryCommand() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetArgs([]string{"search", "x", "--kind", "bogus"}) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error for bogus --kind") + } +} + +func TestRegistrySearchFormatValidation(t *testing.T) { + cmd := newRegistryCommand() + cmd.SetOut(&bytes.Buffer{}) + cmd.SetErr(&bytes.Buffer{}) + cmd.SetArgs([]string{"search", "x", "--format", "yaml"}) + if err := cmd.Execute(); err == nil { + t.Fatal("expected error for unsupported --format") + } +} + +func TestRegistrySearchNoQueryOK(t *testing.T) { + cmd := newRegistryCommand() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"search"}) + if err := cmd.Execute(); err != nil { + t.Fatalf("execute: %v", err) + } + if !strings.Contains(buf.String(), "REF") { + t.Errorf("expected table output for empty query, got:\n%s", buf.String()) + } +} diff --git a/cmd/waza/root.go b/cmd/waza/root.go index ba953de2..50a7a373 100644 --- a/cmd/waza/root.go +++ b/cmd/waza/root.go @@ -74,6 +74,7 @@ performance against predefined test cases.`, cmd.AddCommand(newMCPMockCommand()) cmd.AddCommand(newReplayCommand()) cmd.AddCommand(newAdversarialCommand()) + cmd.AddCommand(newRegistryCommand()) return cmd } diff --git a/internal/registry/config.go b/internal/registry/config.go new file mode 100644 index 00000000..3fb85bad --- /dev/null +++ b/internal/registry/config.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +// Source describes a configured registry index that the CLI can query for +// discovery. Multiple sources are consulted in priority order by +// `waza registry search` (see design §12). +type Source struct { + // Name is a short human identifier (e.g. "public", "company"). + Name string `yaml:"name" json:"name"` + // URL is the base URL of the index. For the public default + // this is the waza-evals GitHub org page; index format is + // documented in docs/research/waza-eval-registry-design.md §12. + URL string `yaml:"url" json:"url"` + // Priority ranks sources during federated search. Lower numbers + // are consulted first. Zero is treated as the default (100). + Priority int `yaml:"priority,omitempty" json:"priority,omitempty"` +} + +// Config holds the set of registry sources known to the waza CLI. The +// list is loaded from `waza.registry.yaml` (design §7) but the file +// itself is not required — a default source is always injected so that +// `waza registry search` works out of the box. +type Config struct { + Sources []Source `yaml:"registries" json:"registries"` +} + +// DefaultPublicSource is the built-in public registry pointing at the +// waza-evals GitHub org. It is always available unless the caller +// explicitly overrides the source list with a config file that omits it. +var DefaultPublicSource = Source{ + Name: "public", + URL: "https://github.com/waza-evals", + Priority: 100, +} + +// DefaultConfig returns a Config seeded with the public waza-evals org. +// This is what the CLI uses when no user-supplied config is found. +func DefaultConfig() Config { + return Config{Sources: []Source{DefaultPublicSource}} +} + +// FindSource returns the configured source with the given name, or the +// zero value and false if not found. +func (c Config) FindSource(name string) (Source, bool) { + for _, s := range c.Sources { + if s.Name == name { + return s, true + } + } + return Source{}, false +} diff --git a/internal/registry/evalfile.go b/internal/registry/evalfile.go new file mode 100644 index 00000000..2c4d4274 --- /dev/null +++ b/internal/registry/evalfile.go @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +// GraderRefEntry describes the minimal information needed to append a +// remote grader reference to an existing eval.yaml (design §7). +type GraderRefEntry struct { + Ref string + Name string + Weight float64 + // Config holds arbitrary key/value overrides supplied via + // `--set key=value`. Nested keys use dot notation. + Config map[string]any +} + +// ErrEvalFileMissing is returned when the target eval.yaml does not exist. +var ErrEvalFileMissing = errors.New("eval file not found") + +// AppendGraderRef parses evalPath, appends a new grader entry using the +// `ref:` short-form, and writes the file back. It preserves the caller's +// formatting for the surrounding YAML by round-tripping through +// yaml.Node. +func AppendGraderRef(evalPath string, entry GraderRefEntry) error { + data, err := os.ReadFile(evalPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("%w: %s", ErrEvalFileMissing, evalPath) + } + return fmt.Errorf("reading %s: %w", evalPath, err) + } + + var root yaml.Node + if err := yaml.Unmarshal(data, &root); err != nil { + return fmt.Errorf("parsing %s: %w", evalPath, err) + } + if root.Kind != yaml.DocumentNode || len(root.Content) == 0 { + return fmt.Errorf("%s: unexpected YAML shape", evalPath) + } + doc := root.Content[0] + if doc.Kind != yaml.MappingNode { + return fmt.Errorf("%s: top-level must be a mapping", evalPath) + } + + graders := findOrCreateSequence(doc, "graders") + graderNode, err := buildGraderRefNode(entry) + if err != nil { + return err + } + graders.Content = append(graders.Content, graderNode) + + out, err := yaml.Marshal(&root) + if err != nil { + return fmt.Errorf("marshaling %s: %w", evalPath, err) + } + if err := os.WriteFile(evalPath, out, 0o644); err != nil { + return fmt.Errorf("writing %s: %w", evalPath, err) + } + return nil +} + +// findOrCreateSequence returns the child sequence under key `name`, +// creating one if it doesn't yet exist. +func findOrCreateSequence(mapping *yaml.Node, name string) *yaml.Node { + for i := 0; i+1 < len(mapping.Content); i += 2 { + k := mapping.Content[i] + v := mapping.Content[i+1] + if k.Value == name { + if v.Kind != yaml.SequenceNode { + // Overwrite non-sequence with a fresh sequence. + v.Kind = yaml.SequenceNode + v.Tag = "!!seq" + v.Value = "" + v.Content = nil + } + return v + } + } + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: name} + seq := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + mapping.Content = append(mapping.Content, keyNode, seq) + return seq +} + +func buildGraderRefNode(entry GraderRefEntry) (*yaml.Node, error) { + m := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + setMapString(m, "ref", entry.Ref) + if entry.Name != "" { + setMapString(m, "name", entry.Name) + } + if entry.Weight != 0 { + setMapFloat(m, "weight", entry.Weight) + } + if len(entry.Config) > 0 { + cfg, err := mapToYAMLNode(entry.Config) + if err != nil { + return nil, err + } + m.Content = append(m.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "config"}, + cfg, + ) + } + return m, nil +} + +func setMapString(m *yaml.Node, key, val string) { + m.Content = append(m.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: val}, + ) +} + +func setMapFloat(m *yaml.Node, key string, val float64) { + m.Content = append(m.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!float", Value: strconv.FormatFloat(val, 'g', -1, 64)}, + ) +} + +func mapToYAMLNode(m map[string]any) (*yaml.Node, error) { + node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + for k, v := range m { + valNode, err := valueToYAMLNode(v) + if err != nil { + return nil, err + } + node.Content = append(node.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: k}, + valNode, + ) + } + return node, nil +} + +func valueToYAMLNode(v any) (*yaml.Node, error) { + switch val := v.(type) { + case string: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: val}, nil + case bool: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!bool", Value: strconv.FormatBool(val)}, nil + case int: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: strconv.Itoa(val)}, nil + case int64: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: strconv.FormatInt(val, 10)}, nil + case float64: + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!float", Value: strconv.FormatFloat(val, 'g', -1, 64)}, nil + case map[string]any: + return mapToYAMLNode(val) + default: + return nil, fmt.Errorf("unsupported --set value type %T", v) + } +} + +// ParseSetFlag parses --set key.path=value inputs into a nested map. It +// mirrors kubectl/helm conventions: the last `=` separates key and value, +// dots in the key segment describe map nesting. +func ParseSetFlag(inputs []string) (map[string]any, error) { + out := map[string]any{} + for _, in := range inputs { + eq := strings.Index(in, "=") + if eq < 0 { + return nil, fmt.Errorf("--set %q: expected key=value", in) + } + key := strings.TrimSpace(in[:eq]) + val := strings.TrimSpace(in[eq+1:]) + if key == "" { + return nil, fmt.Errorf("--set %q: empty key", in) + } + parts := strings.Split(key, ".") + insertNested(out, parts, coerceScalar(val)) + } + return out, nil +} + +func insertNested(m map[string]any, keys []string, val any) { + for i := 0; i < len(keys)-1; i++ { + k := keys[i] + next, ok := m[k].(map[string]any) + if !ok { + next = map[string]any{} + m[k] = next + } + m = next + } + m[keys[len(keys)-1]] = val +} + +// coerceScalar tries a small set of common scalar conversions before +// falling back to string. +func coerceScalar(s string) any { + if s == "true" { + return true + } + if s == "false" { + return false + } + if i, err := strconv.ParseInt(s, 10, 64); err == nil { + return i + } + if f, err := strconv.ParseFloat(s, 64); err == nil { + return f + } + return s +} diff --git a/internal/registry/evalfile_test.go b/internal/registry/evalfile_test.go new file mode 100644 index 00000000..208b4f52 --- /dev/null +++ b/internal/registry/evalfile_test.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestAppendGraderRefCreatesGraders(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "eval.yaml") + seed := "name: my-eval\nversion: 1\n" + if err := os.WriteFile(path, []byte(seed), 0o644); err != nil { + t.Fatal(err) + } + + err := AppendGraderRef(path, GraderRefEntry{ + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", + Name: "factuality_strict", + Weight: 2.0, + Config: map[string]any{"threshold": 0.9}, + }) + if err != nil { + t.Fatalf("AppendGraderRef: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + // Verify structural correctness with a second parse. + var doc map[string]any + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("re-parse: %v\n---\n%s", err, data) + } + graders, ok := doc["graders"].([]any) + if !ok || len(graders) != 1 { + t.Fatalf("expected 1 grader, got %#v", doc["graders"]) + } + g, ok := graders[0].(map[string]any) + if !ok { + t.Fatalf("grader entry not a map: %#v", graders[0]) + } + if g["ref"] != "github.com/waza-evals/fact#factuality@v1.0.0" { + t.Errorf("ref not written: %v", g["ref"]) + } + if g["name"] != "factuality_strict" { + t.Errorf("name not written: %v", g["name"]) + } + if cfg, ok := g["config"].(map[string]any); !ok || cfg["threshold"] != 0.9 { + t.Errorf("config not written: %v", g["config"]) + } + // Sanity: seed content preserved. + if !strings.Contains(string(data), "name: my-eval") { + t.Errorf("original content lost:\n%s", data) + } +} + +func TestAppendGraderRefMissingFile(t *testing.T) { + err := AppendGraderRef(filepath.Join(t.TempDir(), "missing.yaml"), GraderRefEntry{Ref: "x"}) + if err == nil { + t.Fatal("expected error for missing file") + } +} + +func TestParseSetFlag(t *testing.T) { + got, err := ParseSetFlag([]string{ + "config.threshold=0.9", + "config.mode=rubric", + "weight=2", + "enabled=true", + }) + if err != nil { + t.Fatalf("ParseSetFlag: %v", err) + } + cfg, ok := got["config"].(map[string]any) + if !ok { + t.Fatalf("config not nested: %#v", got) + } + if cfg["threshold"] != 0.9 { + t.Errorf("threshold: %v", cfg["threshold"]) + } + if cfg["mode"] != "rubric" { + t.Errorf("mode: %v", cfg["mode"]) + } + if got["weight"] != int64(2) { + t.Errorf("weight: %v", got["weight"]) + } + if got["enabled"] != true { + t.Errorf("enabled: %v", got["enabled"]) + } +} + +func TestParseSetFlagInvalid(t *testing.T) { + if _, err := ParseSetFlag([]string{"no-equals"}); err == nil { + t.Error("expected error for missing '='") + } + if _, err := ParseSetFlag([]string{"=value"}); err == nil { + t.Error("expected error for empty key") + } +} + +func TestLockFileRoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, LockFileName) + + lf, err := LoadLockFile(path) + if err != nil { + t.Fatal(err) + } + if lf.SchemaVersion != LockSchemaVersion { + t.Errorf("schema: %d", lf.SchemaVersion) + } + + added := lf.Upsert(LockEntry{ + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", + Module: "github.com/waza-evals/fact", + Version: "v1.0.0", + }) + if !added { + t.Error("expected new entry to be added") + } + if err := lf.Save(path); err != nil { + t.Fatal(err) + } + + // Re-load and upsert same ref → should replace, not append. + lf2, err := LoadLockFile(path) + if err != nil { + t.Fatal(err) + } + if len(lf2.Modules) != 1 { + t.Fatalf("expected 1 module, got %d", len(lf2.Modules)) + } + added = lf2.Upsert(LockEntry{ + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", + Module: "github.com/waza-evals/fact", + Version: "v1.0.1", + }) + if added { + t.Error("expected replace, got new insert") + } + if lf2.Modules[0].Version != "v1.0.1" { + t.Errorf("version: %s", lf2.Modules[0].Version) + } +} diff --git a/internal/registry/lock.go b/internal/registry/lock.go new file mode 100644 index 00000000..e77547d6 --- /dev/null +++ b/internal/registry/lock.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "fmt" + "os" + "time" + + "gopkg.in/yaml.v3" +) + +// LockFileName is the conventional file name of the reproducibility +// lockfile (design §8). +const LockFileName = "waza.lock" + +// LockSchemaVersion is the current lockfile schema version. +const LockSchemaVersion = 1 + +// LockEntry is a single resolved module recorded in waza.lock. +type LockEntry struct { + Ref string `yaml:"ref" json:"ref"` + Module string `yaml:"module" json:"module"` + Version string `yaml:"version,omitempty" json:"version,omitempty"` + Commit string `yaml:"commit,omitempty" json:"commit,omitempty"` + Digest string `yaml:"digest,omitempty" json:"digest,omitempty"` + Trusted bool `yaml:"trusted,omitempty" json:"trusted,omitempty"` + ResolvedAt string `yaml:"resolved_at,omitempty" json:"resolved_at,omitempty"` + Dependencies []string `yaml:"dependencies,omitempty" json:"dependencies,omitempty"` +} + +// LockFile is the on-disk shape of waza.lock. +type LockFile struct { + SchemaVersion int `yaml:"schema_version" json:"schema_version"` + Modules []LockEntry `yaml:"modules" json:"modules"` +} + +// LoadLockFile reads a lockfile from path. A missing file is reported as +// an empty LockFile with the current schema version so callers can +// unconditionally Upsert then Save. +func LoadLockFile(path string) (*LockFile, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return &LockFile{SchemaVersion: LockSchemaVersion}, nil + } + return nil, fmt.Errorf("reading lock file %s: %w", path, err) + } + var lf LockFile + if err := yaml.Unmarshal(data, &lf); err != nil { + return nil, fmt.Errorf("parsing lock file %s: %w", path, err) + } + if lf.SchemaVersion == 0 { + lf.SchemaVersion = LockSchemaVersion + } + return &lf, nil +} + +// Save writes the lockfile back to disk with stable field ordering. +func (lf *LockFile) Save(path string) error { + if lf.SchemaVersion == 0 { + lf.SchemaVersion = LockSchemaVersion + } + data, err := yaml.Marshal(lf) + if err != nil { + return fmt.Errorf("marshaling lock file: %w", err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("writing lock file %s: %w", path, err) + } + return nil +} + +// Upsert inserts or replaces a lock entry keyed by Ref. It returns true +// if the entry was newly added. +func (lf *LockFile) Upsert(entry LockEntry) bool { + if entry.ResolvedAt == "" { + entry.ResolvedAt = time.Now().UTC().Format(time.RFC3339) + } + for i, e := range lf.Modules { + if e.Ref == entry.Ref { + lf.Modules[i] = entry + return false + } + } + lf.Modules = append(lf.Modules, entry) + return true +} + +// EntryFromResolution converts a resolver output into a LockEntry. +func EntryFromResolution(r Resolution) LockEntry { + return LockEntry{ + Ref: r.Ref.String(), + Module: r.Module, + Version: r.Version, + Commit: r.Commit, + Digest: r.Digest, + Trusted: r.Trusted, + ResolvedAt: time.Now().UTC().Format(time.RFC3339), + } +} diff --git a/internal/registry/ref.go b/internal/registry/ref.go new file mode 100644 index 00000000..cb0e8950 --- /dev/null +++ b/internal/registry/ref.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +// Package registry provides ref parsing, configuration, and (in future +// issues) resolution logic for the Waza eval registry described in +// docs/research/waza-eval-registry-design.md. +// +// This file implements the ref syntax used by CLI subcommands introduced +// in issue #17 (`waza registry search`, `waza registry add`). Actual +// resolution of a Ref to a cached module tree is provided by the resolver +// added in issue #15; see resolver.go for the integration TODO. +package registry + +import ( + "errors" + "fmt" + "strings" +) + +// Ref is the canonical form of a registry reference: +// +// //[/path][#export]@ +// +// Examples: +// +// github.com/waza-evals/fact#factuality@v1.0.0 +// github.com/waza-evals/fact/graders/factuality@v1.0.0 +// github.com/myorg/private-evals/security#secrets@v2.1.3 +type Ref struct { + // Raw is the original string the caller supplied. + Raw string + // Host is the source host (e.g. "github.com"). + Host string + // Owner is the org or user under the host (e.g. "waza-evals"). + Owner string + // Repo is the repository name (e.g. "fact"). + Repo string + // Path is the optional sub-path inside the repo (e.g. "graders/factuality"). + // Empty when omitted. + Path string + // Export is the optional artifact name after the "#" separator + // (e.g. "factuality"). Empty when omitted. + Export string + // Version is the tag, semver, branch, or commit selector after "@" + // (e.g. "v1.0.0", "main", "4f8c2d6a"). Empty when omitted. + Version string +} + +// Module returns the "//" portion of the ref, i.e. the +// module identity without any sub-path, export, or version suffix. +func (r Ref) Module() string { + if r.Host == "" || r.Owner == "" || r.Repo == "" { + return "" + } + return r.Host + "/" + r.Owner + "/" + r.Repo +} + +// String reassembles a canonical string form of the ref. It is idempotent +// with ParseRef for valid inputs. +func (r Ref) String() string { + var b strings.Builder + b.WriteString(r.Module()) + if r.Path != "" { + b.WriteByte('/') + b.WriteString(r.Path) + } + if r.Export != "" { + b.WriteByte('#') + b.WriteString(r.Export) + } + if r.Version != "" { + b.WriteByte('@') + b.WriteString(r.Version) + } + return b.String() +} + +// ErrInvalidRef is returned when a ref cannot be parsed. +var ErrInvalidRef = errors.New("invalid registry ref") + +// ParseRef parses the canonical registry ref syntax. +// +// The grammar is intentionally forgiving: an empty version is allowed at +// parse time so that CLI callers can accept floating refs and later +// enforce their own version policy (see design §6). Callers that require +// a version should check Ref.Version explicitly. +func ParseRef(s string) (Ref, error) { + trimmed := strings.TrimSpace(s) + if trimmed == "" { + return Ref{}, fmt.Errorf("%w: empty ref", ErrInvalidRef) + } + + ref := Ref{Raw: trimmed} + rest := trimmed + + if at := strings.LastIndex(rest, "@"); at >= 0 { + ref.Version = rest[at+1:] + rest = rest[:at] + if ref.Version == "" { + return Ref{}, fmt.Errorf("%w: version selector after '@' is empty", ErrInvalidRef) + } + } + + if hash := strings.LastIndex(rest, "#"); hash >= 0 { + ref.Export = rest[hash+1:] + rest = rest[:hash] + if ref.Export == "" { + return Ref{}, fmt.Errorf("%w: export after '#' is empty", ErrInvalidRef) + } + } + + parts := strings.Split(rest, "/") + if len(parts) < 3 { + return Ref{}, fmt.Errorf("%w: expected // got %q", ErrInvalidRef, rest) + } + ref.Host = parts[0] + ref.Owner = parts[1] + ref.Repo = parts[2] + if ref.Host == "" || ref.Owner == "" || ref.Repo == "" { + return Ref{}, fmt.Errorf("%w: host, owner, and repo are required", ErrInvalidRef) + } + if len(parts) > 3 { + ref.Path = strings.Join(parts[3:], "/") + } + + return ref, nil +} + +// IsRemote reports whether s looks like a remote ref (host/owner/repo) +// rather than a local path or bare grader name. +func IsRemote(s string) bool { + s = strings.TrimSpace(s) + if s == "" || strings.HasPrefix(s, ".") || strings.HasPrefix(s, "/") { + return false + } + head := s + if at := strings.Index(head, "@"); at >= 0 { + head = head[:at] + } + if hash := strings.Index(head, "#"); hash >= 0 { + head = head[:hash] + } + return strings.Count(head, "/") >= 2 +} diff --git a/internal/registry/ref_test.go b/internal/registry/ref_test.go new file mode 100644 index 00000000..b5555bb4 --- /dev/null +++ b/internal/registry/ref_test.go @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "errors" + "testing" +) + +func TestParseRef(t *testing.T) { + tests := []struct { + name string + in string + want Ref + wantErr bool + }{ + { + name: "canonical with export and version", + in: "github.com/waza-evals/fact#factuality@v1.0.0", + want: Ref{ + Raw: "github.com/waza-evals/fact#factuality@v1.0.0", + Host: "github.com", + Owner: "waza-evals", + Repo: "fact", + Export: "factuality", + Version: "v1.0.0", + }, + }, + { + name: "path form without export", + in: "github.com/waza-evals/fact/graders/factuality@v1.0.0", + want: Ref{ + Raw: "github.com/waza-evals/fact/graders/factuality@v1.0.0", + Host: "github.com", + Owner: "waza-evals", + Repo: "fact", + Path: "graders/factuality", + Version: "v1.0.0", + }, + }, + { + name: "no version", + in: "github.com/waza-evals/fact#factuality", + want: Ref{ + Raw: "github.com/waza-evals/fact#factuality", + Host: "github.com", + Owner: "waza-evals", + Repo: "fact", + Export: "factuality", + }, + }, + {name: "empty", in: "", wantErr: true}, + {name: "too few parts", in: "github.com/foo", wantErr: true}, + {name: "empty version", in: "github.com/waza-evals/fact@", wantErr: true}, + {name: "empty export", in: "github.com/waza-evals/fact#@v1", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseRef(tt.in) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error, got %+v", got) + } + if !errors.Is(err, ErrInvalidRef) { + t.Fatalf("expected ErrInvalidRef, got %v", err) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("got %+v want %+v", got, tt.want) + } + if got.String() != tt.in { + t.Fatalf("round-trip: got %q want %q", got.String(), tt.in) + } + if got.Module() != tt.want.Host+"/"+tt.want.Owner+"/"+tt.want.Repo { + t.Fatalf("module mismatch: %q", got.Module()) + } + }) + } +} + +func TestIsRemote(t *testing.T) { + tests := []struct { + in string + want bool + }{ + {"github.com/waza-evals/fact#factuality@v1.0.0", true}, + {"github.com/waza-evals/fact", true}, + {"./local.yaml", false}, + {"/abs/path", false}, + {"factuality", false}, + {"", false}, + } + for _, tt := range tests { + if got := IsRemote(tt.in); got != tt.want { + t.Errorf("IsRemote(%q) = %v want %v", tt.in, got, tt.want) + } + } +} diff --git a/internal/registry/resolver.go b/internal/registry/resolver.go new file mode 100644 index 00000000..3c1fce57 --- /dev/null +++ b/internal/registry/resolver.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import ( + "errors" + "fmt" +) + +// Resolution is the result of resolving a Ref to a concrete, cached +// module version. It is a lightweight stand-in for the richer resolver +// output that issue #15 will produce; it lets `waza registry add` write +// a plausible waza.lock entry today. +type Resolution struct { + // Ref is the canonical ref as supplied by the user. + Ref Ref + // Module is "//". + Module string + // Version is the resolved version selector (may still be floating + // for the stub). + Version string + // Commit is the resolved commit SHA. Empty from the stub. + Commit string + // Digest is the content digest. Empty from the stub. + Digest string + // Kind is the resolved artifact kind, when known. + Kind Kind + // Trusted indicates whether the caller granted trust to execute + // program-graders from this ref. See design §14. + Trusted bool +} + +// Resolver resolves a Ref to a Resolution. The real implementation lives +// in issue #15's resolver package. +type Resolver interface { + Resolve(ref Ref) (Resolution, error) +} + +// ErrResolverNotImplemented is returned by the stub resolver in place of +// a real network round-trip. CLI commands surface this to the user with +// a clear message pointing at issue #15. +var ErrResolverNotImplemented = errors.New("registry resolver not yet implemented (see issue #15)") + +// StubResolver returns partial resolution metadata sufficient for the +// Phase 1 CLI to update eval.yaml and produce a placeholder waza.lock +// entry without performing any network I/O. +type StubResolver struct{} + +// Resolve returns a Resolution derived purely from the ref's own +// syntax. It never fails on well-formed input; callers that need real +// commit/digest values must wait for issue #15. +// +// TODO(#15): replace with a real resolver that reads waza.registry.yaml, +// authenticates against the source backend, downloads the module, and +// computes commit + digest. +func (StubResolver) Resolve(ref Ref) (Resolution, error) { + if ref.Module() == "" { + return Resolution{}, fmt.Errorf("cannot resolve empty ref") + } + return Resolution{ + Ref: ref, + Module: ref.Module(), + Version: ref.Version, + }, nil +} diff --git a/internal/registry/search.go b/internal/registry/search.go new file mode 100644 index 00000000..aaa1c777 --- /dev/null +++ b/internal/registry/search.go @@ -0,0 +1,161 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +// Kind identifies a registry artifact type as described in design §4. +// Values are stringly-typed so that unknown kinds surfaced by remote +// indexes flow through unchanged. +type Kind string + +const ( + KindGrader Kind = "grader" + KindEval Kind = "eval" + KindDataset Kind = "dataset" + // KindProgramGrader is the executable-grader flavor of KindGrader. + // Consumers use it to gate on --allow-exec (design §14). + KindProgramGrader Kind = "program-grader" +) + +// SearchResult is a single row returned from an index search. The shape +// mirrors the table columns documented in design §13. +type SearchResult struct { + Ref string `json:"ref"` + Kind Kind `json:"kind"` + Description string `json:"description,omitempty"` + Stars int `json:"stars,omitempty"` + Source string `json:"source,omitempty"` +} + +// SearchOptions carries the flag-level input for a search request. +type SearchOptions struct { + // Query is the free-form user query. + Query string + // Kind, when non-empty, restricts results to that artifact kind. + Kind Kind + // Registry, when non-empty, restricts the search to a single + // configured Source by Name. + Registry string +} + +// Searcher searches configured registry sources. Real implementations +// will fan out to each Source and merge/dedupe results (design §12). +type Searcher interface { + Search(opts SearchOptions) ([]SearchResult, error) +} + +// stubSearcher returns a hard-coded set of well-known refs so the CLI +// has meaningful output before the real index API in issue #15 lands. +type stubSearcher struct { + cfg Config +} + +// NewSearcher returns a Searcher that uses the given config. Today it is +// a stub; once issue #15 lands, this will build a federated HTTP client. +// +// TODO(#15): swap this stub for a real index-backed implementation. +func NewSearcher(cfg Config) Searcher { + return &stubSearcher{cfg: cfg} +} + +// canned known-ref catalog. Keep this list small — it is only meant to +// prove the CLI plumbing works. Real content will come from a proper +// index API. +var stubCatalog = []SearchResult{ + { + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", + Kind: KindGrader, + Description: "Prompt grader for factual grounding", + Stars: 12, + }, + { + Ref: "github.com/waza-evals/fact#closedqa@v1.0.0", + Kind: KindGrader, + Description: "Closed-question answer evaluator", + Stars: 9, + }, + { + Ref: "github.com/waza-evals/agent-basics#repo-maintainer@v1.0.0", + Kind: KindEval, + Description: "Baseline eval for repository maintenance agents", + Stars: 7, + }, + { + Ref: "github.com/waza-evals/datasets#humaneval@v0.1.0", + Kind: KindDataset, + Description: "HumanEval-style dataset packaged for waza", + Stars: 4, + }, +} + +func (s *stubSearcher) Search(opts SearchOptions) ([]SearchResult, error) { + // TODO(#15/#67): call registry index API. For now, filter the + // in-memory stub catalog so CLI wiring, table formatting, and + // JSON output can be exercised end-to-end. + q := opts.Query + var out []SearchResult + for _, r := range stubCatalog { + if opts.Kind != "" && r.Kind != opts.Kind { + continue + } + if q != "" && !containsFold(r.Ref, q) && !containsFold(r.Description, q) { + continue + } + // Tag every result with the primary configured source so users + // can see which registry it came from once federation lands. + if len(s.cfg.Sources) > 0 { + r.Source = s.cfg.Sources[0].Name + } + if opts.Registry != "" && r.Source != opts.Registry { + continue + } + out = append(out, r) + } + return out, nil +} + +// containsFold is a small case-insensitive substring helper. Kept +// private so we don't pull in strings.EqualFold semantics for every +// caller that only needs substring matching. +func containsFold(haystack, needle string) bool { + if needle == "" { + return true + } + // Manual case fold to avoid an extra strings import elsewhere. + h := make([]byte, len(haystack)) + for i := 0; i < len(haystack); i++ { + c := haystack[i] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + h[i] = c + } + n := make([]byte, len(needle)) + for i := 0; i < len(needle); i++ { + c := needle[i] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + n[i] = c + } + return bytesContains(h, n) +} + +func bytesContains(h, n []byte) bool { + if len(n) == 0 { + return true + } + if len(n) > len(h) { + return false + } +outer: + for i := 0; i <= len(h)-len(n); i++ { + for j := 0; j < len(n); j++ { + if h[i+j] != n[j] { + continue outer + } + } + return true + } + return false +} diff --git a/internal/registry/search_test.go b/internal/registry/search_test.go new file mode 100644 index 00000000..46dd0cd6 --- /dev/null +++ b/internal/registry/search_test.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +package registry + +import "testing" + +func TestSearcherFiltersByKind(t *testing.T) { + s := NewSearcher(DefaultConfig()) + got, err := s.Search(SearchOptions{Kind: KindGrader}) + if err != nil { + t.Fatal(err) + } + if len(got) == 0 { + t.Fatal("expected results") + } + for _, r := range got { + if r.Kind != KindGrader { + t.Errorf("got kind %q, want grader", r.Kind) + } + } +} + +func TestSearcherFiltersByQuery(t *testing.T) { + s := NewSearcher(DefaultConfig()) + got, err := s.Search(SearchOptions{Query: "factuality"}) + if err != nil { + t.Fatal(err) + } + if len(got) == 0 { + t.Fatal("expected results") + } + for _, r := range got { + if !containsFold(r.Ref, "factuality") && !containsFold(r.Description, "factuality") { + t.Errorf("result %+v does not match query", r) + } + } +} + +func TestSearcherRegistryFilter(t *testing.T) { + s := NewSearcher(DefaultConfig()) + got, err := s.Search(SearchOptions{Registry: "public"}) + if err != nil { + t.Fatal(err) + } + if len(got) == 0 { + t.Fatal("expected some results from default public source") + } + empty, err := s.Search(SearchOptions{Registry: "nonexistent"}) + if err != nil { + t.Fatal(err) + } + if len(empty) != 0 { + t.Errorf("expected 0 results for unknown registry, got %d", len(empty)) + } +} + +func TestDefaultConfigIncludesPublic(t *testing.T) { + cfg := DefaultConfig() + if _, ok := cfg.FindSource("public"); !ok { + t.Error("default config missing public source") + } +}