feat: Composable eval construction from registry graders (Phase 1 CLI) #17

cmd/waza/cmd_registry.go

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// newRegistryCommand builds the `waza registry` parent command tree.
+//
+// Phase 1 (issue #17) ships the `search` and `add` subcommands. Full
+// end-to-end functionality — actual index HTTP calls and ref
+// resolution — depends on issue #15's resolver, which is stubbed here
+// with clear TODOs.
+func newRegistryCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "registry",
+		Short: "Discover and add remote grader presets and eval modules",
+		Long: `Manage Waza registry sources for composable eval construction.
+
+The registry lets you discover reusable graders, evals, and datasets
+published to the waza-evals GitHub org (or any additional registry you
+configure) and add them to your eval.yaml with a single command.
+
+See docs/research/waza-eval-registry-design.md for the full design.`,
+	}
+
+	cmd.AddCommand(newRegistrySearchCommand())
+	cmd.AddCommand(newRegistryAddCommand())
+
+	return cmd
+}

cmd/waza/cmd_registry_add.go

@@ -0,0 +1,161 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"github.com/microsoft/waza/internal/registry"
+	"github.com/spf13/cobra"
+)
+
+type registryAddFlags struct {
+	evalPath  string
+	name      string
+	sets      []string
+	weight    float64
+	allowExec bool
+	dryRun    bool
+	yes       bool
+}
+
+func newRegistryAddCommand() *cobra.Command {
+	f := &registryAddFlags{}
+	cmd := &cobra.Command{
+		Use:   "add <ref>",
+		Short: "Add a registry artifact to eval.yaml and update waza.lock",
+		Long: `Resolve a registry ref, append it to eval.yaml as a "ref:" grader
+entry, and update waza.lock with the resolved commit and digest.
+
+Program graders (executable artifacts) are refused unless the caller
+passes --allow-exec or confirms the interactive prompt.
+
+Examples:
+  waza registry add github.com/waza-evals/fact#factuality@v1.0.0
+  waza registry add github.com/waza-evals/fact#factuality@v1.0.0 --eval eval.yaml
+  waza registry add github.com/waza-evals/fact#factuality@v1.0.0 \
+      --name factuality_strict --set config.threshold=0.9`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runRegistryAdd(cmd.OutOrStdout(), cmd.InOrStdin(), args[0], *f)
+		},
+	}
+
+	cmd.Flags().StringVar(&f.evalPath, "eval", "eval.yaml", "Path to the eval file to modify")
+	cmd.Flags().StringVar(&f.name, "name", "", "Local alias for the grader (overrides remote default)")
+	cmd.Flags().StringSliceVar(&f.sets, "set", nil, "Config overrides, key.path=value (repeatable)")
+	cmd.Flags().Float64Var(&f.weight, "weight", 0, "Grader weight override (0 keeps default)")
+	cmd.Flags().BoolVar(&f.allowExec, "allow-exec", false, "Allow adding program-grader artifacts without prompting")
+	cmd.Flags().BoolVar(&f.dryRun, "dry-run", false, "Print planned changes without writing files")
+	cmd.Flags().BoolVarP(&f.yes, "yes", "y", false, "Assume yes to interactive prompts")
+
+	return cmd
+}
+
+func runRegistryAdd(out io.Writer, in io.Reader, refStr string, f registryAddFlags) error {
+	if !registry.IsRemote(refStr) {
+		return fmt.Errorf("%q is not a registry ref (expected host/owner/repo#export@version)", refStr)
+	}
+	ref, err := registry.ParseRef(refStr)
+	if err != nil {
+		return err
+	}
+
+	config, err := registry.ParseSetFlag(f.sets)
+	if err != nil {
+		return err
+	}
+
+	// TODO(#15): call the real resolver. For now the stub returns
+	// syntax-derived metadata so we can still write the ref entry.
+	resolver := registry.StubResolver{}
+	resolution, err := resolver.Resolve(ref)
+	if err != nil {
+		return fmt.Errorf("resolving %s: %w", ref, err)
+	}
+
+	if resolution.Kind == registry.KindProgramGrader {
+		if !f.allowExec && !f.yes {
+			ok, err := confirmProgramGrader(out, in, ref.String())
+			if err != nil {
+				return err
+			}
+			if !ok {
+				return errors.New("aborted by user; re-run with --allow-exec to skip the prompt")
+			}
+		}
+		resolution.Trusted = true
+	}
+
+	entry := registry.GraderRefEntry{
+		Ref:    ref.String(),
+		Name:   f.name,
+		Weight: f.weight,
+		Config: config,
+	}
+
+	if f.dryRun {
+		return printAddDryRun(out, f.evalPath, entry, resolution)
+	}
+
+	evalPath, err := filepath.Abs(f.evalPath)
+	if err != nil {
+		return fmt.Errorf("resolving eval path: %w", err)
+	}
+	if err := registry.AppendGraderRef(evalPath, entry); err != nil {
+		return err
+	}
+
+	lockPath := filepath.Join(filepath.Dir(evalPath), registry.LockFileName)
+	lf, err := registry.LoadLockFile(lockPath)
+	if err != nil {
+		return err
+	}
+	lockEntry := registry.EntryFromResolution(resolution)
+	lf.Upsert(lockEntry)
+	if err := lf.Save(lockPath); err != nil {
+		return err
+	}
+
+	fmt.Fprintf(out, "Added grader %s to %s\n", ref.String(), f.evalPath) //nolint:errcheck
+	fmt.Fprintf(out, "Updated %s\n", registry.LockFileName)               //nolint:errcheck
+	// TODO(#15): print resolved commit + digest once the real resolver
+	// returns them.
+	return nil
+}
+
+func confirmProgramGrader(out io.Writer, in io.Reader, ref string) (bool, error) {
+	fmt.Fprintf(out, "%s is a program grader (executable). Trust and add? [y/N]: ", ref) //nolint:errcheck
+	reader := bufio.NewReader(in)
+	line, err := reader.ReadString('\n')
+	if err != nil && err != io.EOF {
+		return false, fmt.Errorf("reading confirmation: %w", err)
+	}
+	line = strings.TrimSpace(strings.ToLower(line))
+	return line == "y" || line == "yes", nil
+}
+
+func printAddDryRun(out io.Writer, evalPath string, entry registry.GraderRefEntry, res registry.Resolution) error {
+	fmt.Fprintf(out, "DRY RUN: would add grader to %s:\n", evalPath) //nolint:errcheck
+	fmt.Fprintf(out, "  ref: %s\n", entry.Ref)                       //nolint:errcheck
+	if entry.Name != "" {
+		fmt.Fprintf(out, "  name: %s\n", entry.Name) //nolint:errcheck
+	}
+	if entry.Weight != 0 {
+		fmt.Fprintf(out, "  weight: %g\n", entry.Weight) //nolint:errcheck
+	}
+	if len(entry.Config) > 0 {
+		fmt.Fprintln(out, "  config:") //nolint:errcheck
+		for k, v := range entry.Config {
+			fmt.Fprintf(out, "    %s: %v\n", k, v) //nolint:errcheck
+		}
+	}
+	fmt.Fprintf(out, "DRY RUN: would update %s with module %s@%s\n", registry.LockFileName, res.Module, res.Version) //nolint:errcheck
+	return nil
+}

cmd/waza/cmd_registry_add_test.go

@@ -0,0 +1,132 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func writeEvalYAML(t *testing.T, dir, content string) string {
+	t.Helper()
+	path := filepath.Join(dir, "eval.yaml")
+	if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+func TestRegistryAddAppendsGraderAndWritesLock(t *testing.T) {
+	dir := t.TempDir()
+	evalPath := writeEvalYAML(t, dir, "name: my-eval\nversion: 1\n")
+
+	cmd := newRegistryCommand()
+	buf := &bytes.Buffer{}
+	cmd.SetOut(buf)
+	cmd.SetErr(buf)
+	cmd.SetArgs([]string{
+		"add", "github.com/waza-evals/fact#factuality@v1.0.0",
+		"--eval", evalPath,
+		"--name", "factuality_strict",
+		"--set", "config.threshold=0.9",
+	})
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("execute: %v\n%s", err, buf.String())
+	}
+
+	data, err := os.ReadFile(evalPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var doc map[string]any
+	if err := yaml.Unmarshal(data, &doc); err != nil {
+		t.Fatalf("re-parse eval.yaml: %v\n%s", err, data)
+	}
+	graders, ok := doc["graders"].([]any)
+	if !ok || len(graders) != 1 {
+		t.Fatalf("graders sequence missing: %#v", doc["graders"])
+	}
+	g, ok := graders[0].(map[string]any)
+	if !ok {
+		t.Fatalf("grader entry not a map: %#v", graders[0])
+	}
+	if g["ref"] != "github.com/waza-evals/fact#factuality@v1.0.0" {
+		t.Errorf("ref: %v", g["ref"])
+	}
+	if g["name"] != "factuality_strict" {
+		t.Errorf("name: %v", g["name"])
+	}
+
+	lockPath := filepath.Join(dir, "waza.lock")
+	lockData, err := os.ReadFile(lockPath)
+	if err != nil {
+		t.Fatalf("read waza.lock: %v", err)
+	}
+	if !strings.Contains(string(lockData), "github.com/waza-evals/fact#factuality@v1.0.0") {
+		t.Errorf("lock missing ref:\n%s", lockData)
+	}
+	if !strings.Contains(string(lockData), "schema_version: 1") {
+		t.Errorf("lock missing schema_version:\n%s", lockData)
+	}
+}
+
+func TestRegistryAddDryRun(t *testing.T) {
+	dir := t.TempDir()
+	evalPath := writeEvalYAML(t, dir, "name: e\n")
+
+	cmd := newRegistryCommand()
+	buf := &bytes.Buffer{}
+	cmd.SetOut(buf)
+	cmd.SetErr(buf)
+	cmd.SetArgs([]string{
+		"add", "github.com/waza-evals/fact#factuality@v1.0.0",
+		"--eval", evalPath,
+		"--dry-run",
+	})
+	if err := cmd.Execute(); err != nil {
+		t.Fatalf("execute: %v", err)
+	}
+	if !strings.Contains(buf.String(), "DRY RUN") {
+		t.Errorf("expected DRY RUN in output:\n%s", buf.String())
+	}
+	// Eval file must be unchanged.
+	data, _ := os.ReadFile(evalPath)
+	if strings.Contains(string(data), "ref:") {
+		t.Errorf("dry run modified eval.yaml:\n%s", data)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "waza.lock")); !os.IsNotExist(err) {
+		t.Errorf("dry run wrote waza.lock")
+	}
+}
+
+func TestRegistryAddRejectsBadRef(t *testing.T) {
+	cmd := newRegistryCommand()
+	cmd.SetOut(&bytes.Buffer{})
+	cmd.SetErr(&bytes.Buffer{})
+	cmd.SetArgs([]string{"add", "./local.yaml"})
+	if err := cmd.Execute(); err == nil {
+		t.Fatal("expected error for local path")
+	}
+}
+
+func TestRegistryAddRejectsBadSetFlag(t *testing.T) {
+	dir := t.TempDir()
+	evalPath := writeEvalYAML(t, dir, "name: e\n")
+	cmd := newRegistryCommand()
+	cmd.SetOut(&bytes.Buffer{})
+	cmd.SetErr(&bytes.Buffer{})
+	cmd.SetArgs([]string{
+		"add", "github.com/waza-evals/fact#factuality@v1.0.0",
+		"--eval", evalPath,
+		"--set", "malformed",
+	})
+	if err := cmd.Execute(); err == nil {
+		t.Fatal("expected error for malformed --set")
+	}
+}