feat: Phase 1 Go-module-style grader refs

cmd/waza/cmd_get.go

@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/microsoft/waza/internal/models"
+	"github.com/microsoft/waza/internal/registry"
+	"github.com/spf13/cobra"
+)
+
+func newGetCommand() *cobra.Command {
+	var strictLock bool
+
+	cmd := &cobra.Command{
+		Use:   "get [eval.yaml]",
+		Short: "Resolve remote grader refs and write waza.lock",
+		Long: `Resolve every 'ref:' grader entry in eval.yaml against its remote Git
+source, download the pinned content into the module cache, and write
+(or update) waza.lock beside eval.yaml.
+
+Phase 1 supports Go-module-style refs of the form:
+
+  github.com/<owner>/<repo>[/path][#export]@<version>
+
+where <version> must be an exact semver tag (vX.Y.Z) or a full 40-character
+commit SHA. Floating selectors (branches, ranges, "latest") are rejected
+for reproducibility.
+
+Examples:
+
+  waza get                              # resolves ./eval.yaml
+  waza get evals/factuality/eval.yaml   # explicit spec path
+  waza get --verify                     # do not modify the lock; verify only
+`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			specPath := "eval.yaml"
+			if len(args) == 1 {
+				specPath = args[0]
+			}
+			spec, err := models.LoadEvalSpec(specPath)
+			if err != nil {
+				return fmt.Errorf("loading %s: %w", specPath, err)
+			}
+
+			refCount := 0
+			for _, g := range spec.Graders {
+				if g.Ref != "" {
+					refCount++
+				}
+			}
+			if refCount == 0 {
+				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "No remote grader refs found in %s. Nothing to do.\n", specPath)
+				return nil
+			}
+
+			resolved, lockChanged, err := registry.ResolveSpec(cmd.Context(), spec, specPath, !strictLock)
+			if err != nil {
+				return err
+			}
+			lockPath := registry.LockfilePath(specPath)
+			switch {
+			case strictLock:
+				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Verified %d remote grader ref(s) against %s.\n", resolved, lockPath)
+			case lockChanged:
+				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resolved %d remote grader ref(s); wrote %s.\n", resolved, lockPath)
+			default:
+				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Resolved %d remote grader ref(s); %s already up to date.\n", resolved, lockPath)
+			}
+			return nil
+		},
+	}
+
+	cmd.Flags().BoolVar(&strictLock, "verify", false, "Verify refs against existing waza.lock without modifying it (fails if any ref is unlocked or digest-mismatched)")
+	return cmd
+}

cmd/waza/cmd_run.go

@@ -34,6 +34,7 @@ import (
 	"github.com/microsoft/waza/internal/orchestration"
 	"github.com/microsoft/waza/internal/projectconfig"
 	"github.com/microsoft/waza/internal/recommend"
+	"github.com/microsoft/waza/internal/registry"
 	"github.com/microsoft/waza/internal/reporting"
 	"github.com/microsoft/waza/internal/session"
 	"github.com/microsoft/waza/internal/snapshot"
@@ -568,6 +569,13 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str
 		return nil, fmt.Errorf("failed to load spec: %w", err)
 	}
 
+	// Resolve remote grader refs (Phase 1: exact-pinned github.com refs).
+	// If waza.lock exists we verify strictly; otherwise auto-resolve and
+	// write the lock (permissive first-run policy).
+	if err := resolveGraderRefs(cmd, spec, specPath); err != nil {
+		return nil, err
+	}
+
 	// CLI flags override spec config
 	if parallel {
 		spec.Config.Concurrent = true
@@ -2141,3 +2149,55 @@ func runDiscoverMode(cmd *cobra.Command, args []string) error {
 
 	return lastErr
 }
+
+// resolveGraderRefs expands remote grader refs referenced in spec against
+// waza.lock beside specPath.
+//
+// Policy (Phase 1):
+//   - No refs → no-op.
+//   - waza.lock exists → verify strictly. Missing entries or digest
+//     mismatches fail the run.
+//   - waza.lock missing → auto-resolve iff every ref is exact-pinned
+//     (tag or 40-char commit SHA), then write the lock and log a warning
+//     that the lock was created. If any ref uses a floating selector
+//     (currently rejected up front by ParseRef), fail.
+func resolveGraderRefs(cmd *cobra.Command, spec *models.EvalSpec, specPath string) error {
+	refCount := 0
+	for _, g := range spec.Graders {
+		if g.Ref != "" {
+			refCount++
+		}
+	}
+	if refCount == 0 {
+		return nil
+	}
+
+	ctx := context.Background()
+	if cmd != nil {
+		ctx = cmd.Context()
+	}
+
+	lockPath := registry.LockfilePath(specPath)
+	lockExists := false
+	if _, err := os.Stat(lockPath); err == nil {
+		lockExists = true
+	} else if !os.IsNotExist(err) {
+		return fmt.Errorf("stat %s: %w", lockPath, err)
+	}
+
+	updateLock := !lockExists
+	resolved, changed, err := registry.ResolveSpec(ctx, spec, specPath, updateLock)
+	if err != nil {
+		if errors.Is(err, registry.ErrRefNotInLock) {
+			return fmt.Errorf(
+				"%w — run `waza get %s` to resolve remote grader refs and write %s",
+				err, specPath, filepath.Base(lockPath),
+			)
+		}
+		return err
+	}
+	if updateLock && changed {
+		fmt.Fprintf(os.Stderr, "waza: resolved %d remote grader ref(s); wrote %s\n", resolved, lockPath)
+	}
+	return nil
+}

cmd/waza/root.go

@@ -52,6 +52,7 @@ performance against predefined test cases.`,
 	// Add subcommands
 	cmd.AddCommand(newRunCommand())
 	cmd.AddCommand(newInitCommand())
+	cmd.AddCommand(newGetCommand())
 	cmd.AddCommand(tokens.NewCommand())
 	cmd.AddCommand(newCompareCommand())
 	cmd.AddCommand(newGateCommand())

internal/models/spec.go

@@ -56,6 +56,7 @@ type strictEvalSpec struct {
 }
 
 type strictGrader struct {
+	Ref        string     `yaml:"ref,omitempty"`
 	Kind       GraderKind `yaml:"type"`
 	Identifier string     `yaml:"name"`
 	ScriptPath string     `yaml:"script,omitempty"`
@@ -193,6 +194,13 @@ func (c *Config) ShouldInjectSkillBody() bool {
 
 // GraderConfig defines a validator/grader
 type GraderConfig struct {
+	// Ref is an optional remote grader reference in Go-module style,
+	// e.g. "github.com/waza-evals/fact#factuality@v1.0.0". When set, the
+	// grader definition (type, config, model, ...) is loaded from the
+	// remote module manifest; local fields on this GraderConfig override
+	// remote defaults after resolution. Requires a waza.lock entry pinning
+	// the commit SHA + content digest of the remote preset.
+	Ref        string           `yaml:"ref,omitempty" json:"ref,omitempty"`
 	Kind       GraderKind       `yaml:"type" json:"kind"`
 	Identifier string           `yaml:"name" json:"identifier"`
 	ScriptPath string           `yaml:"script,omitempty" json:"script_path,omitempty"`
@@ -204,6 +212,7 @@ type GraderConfig struct {
 
 func (g *GraderConfig) UnmarshalYAML(node *yaml.Node) error {
 	type rawGraderConfig struct {
+		Ref        string     `yaml:"ref,omitempty"`
 		Kind       GraderKind `yaml:"type"`
 		Identifier string     `yaml:"name"`
 		ScriptPath string     `yaml:"script,omitempty"`
@@ -227,17 +236,35 @@ func (g *GraderConfig) UnmarshalYAML(node *yaml.Node) error {
 		return err
 	}
 
+	g.Ref = raw.Ref
+	g.Identifier = raw.Identifier
+	g.ScriptPath = raw.ScriptPath
+	g.Rubric = raw.Rubric
+	g.ModelID = raw.ModelID
+	g.Weight = raw.Weight
+
+	// When a remote ref is set, the grader kind and config come from the
+	// remote preset. Preserve the raw override config as a generic map so
+	// the resolver can deep-merge it later. Validation of type/params is
+	// deferred to after ref expansion.
+	if raw.Ref != "" {
+		if raw.Parameters.Kind != 0 {
+			overrides, err := decodeYAMLNode[GenericGraderParameters](&raw.Parameters)
+			if err != nil {
+				return fmt.Errorf("invalid override config for ref %q: %w", raw.Ref, err)
+			}
+			g.Parameters = overrides
+		}
+		g.Kind = raw.Kind // may be empty; resolver fills it in
+		return nil
+	}
+
 	params, err := decodeGraderParameters(raw.Kind, &raw.Parameters)
 	if err != nil {
 		return fmt.Errorf("invalid grader config for %q (type %q): %w", raw.Identifier, raw.Kind, err)
 	}
 
 	g.Kind = raw.Kind
-	g.Identifier = raw.Identifier
-	g.ScriptPath = raw.ScriptPath
-	g.Rubric = raw.Rubric
-	g.ModelID = raw.ModelID
-	g.Weight = raw.Weight
 	g.Parameters = params
 
 	// Validate grader-type-specific required fields

internal/models/spec_test.go

@@ -738,3 +738,52 @@ tasks:
 		}
 	})
 }
+
+func TestEvalSpec_GraderRef(t *testing.T) {
+	tempDir := t.TempDir()
+	yamlContent := `name: ref-grader
+skill: test
+config:
+  trials_per_task: 1
+  timeout_seconds: 60
+  executor: mock
+graders:
+  - ref: github.com/waza-evals/fact#factuality@v1.0.0
+    name: my-fact-check
+    weight: 2.0
+    config:
+      threshold: 0.9
+`
+	specPath := filepath.Join(tempDir, "ref.yaml")
+	if err := os.WriteFile(specPath, []byte(yamlContent), 0o644); err != nil {
+		t.Fatalf("Failed to write spec file: %v", err)
+	}
+	spec, err := LoadEvalSpec(specPath)
+	if err != nil {
+		t.Fatalf("LoadEvalSpec (ref grader) failed: %v", err)
+	}
+	if len(spec.Graders) != 1 {
+		t.Fatalf("Expected 1 grader, got %d", len(spec.Graders))
+	}
+	g := spec.Graders[0]
+	if g.Ref != "github.com/waza-evals/fact#factuality@v1.0.0" {
+		t.Errorf("Ref = %q, want github.com/waza-evals/fact#factuality@v1.0.0", g.Ref)
+	}
+	// Type/config validation should be deferred — no error even without type.
+	if g.Kind != "" {
+		t.Errorf("Kind = %q, want empty until resolution", g.Kind)
+	}
+	if g.Identifier != "my-fact-check" {
+		t.Errorf("Identifier = %q, want my-fact-check", g.Identifier)
+	}
+	if g.Weight != 2.0 {
+		t.Errorf("Weight = %v, want 2.0", g.Weight)
+	}
+	overrides, ok := g.Parameters.(GenericGraderParameters)
+	if !ok {
+		t.Fatalf("Parameters = %T, want GenericGraderParameters", g.Parameters)
+	}
+	if got := overrides["threshold"]; got != 0.9 {
+		t.Errorf("threshold override = %v, want 0.9", got)
+	}
+}