microsoft · Shayne Boyer (spboyer) · Jul 28, 2026
@@ -134,6 +134,9 @@ waza suggest skills/my-skill --apply
 waza spec verify skills/my-skill evals/my-skill/eval.yaml
 waza spec verify skills/my-skill evals/my-skill/eval.yaml --fail --format github-actions
 
+# Resolve remote grader refs and write waza.lock
+waza get evals/my-skill/eval.yaml
+
 # Note: 'generate' is available as an alias for 'new' (see below for new command)
 # Note: Custom agents (.agent.md) are supported — see https://microsoft.github.io/waza/guides/custom-agents/
 
@@ -389,6 +392,15 @@ Cached results are automatically invalidated when:
 
 **Note:** Caching is automatically disabled for evaluations using non-deterministic graders (`behavior`, `prompt`).
 
+### `waza get [eval.yaml | ref]`
+
+Resolve remote grader refs and write `waza.lock`. When passed an eval file, `waza get` resolves every `graders[].ref`, downloads module contents into `~/.waza/cache/{host}/{org}/{repo}/{sha}/`, and pins each ref to a commit SHA and `sha256:` content digest. `waza run` requires a valid lock and cache entry for remote refs; it does not silently resolve unlocked refs during a run.
+
+```bash
+waza get eval.yaml
+waza get github.com/waza-evals/fact#factuality@v1.0.0
+```
+
 **Exit Codes**
 
 The `run` command uses exit codes to enable CI/CD integration:
@@ -1086,6 +1098,12 @@ mcp_mocks:
               issues: []
 
 graders:
+  - ref: github.com/waza-evals/fact#factuality@v1.0.0
+    name: factuality_strict
+    weight: 2.0
+    config:
+      threshold: 0.9
+
   - type: text
     name: pattern_check
     config:
@@ -1114,6 +1132,8 @@ tasks:
 
 `schemaVersion` uses `MAJOR.MINOR` format. Missing values are interpreted as the current schema version (currently `1.2`). Readers allow same-major minor additions with warnings for unknown fields, but reject different majors with a hint to run `waza migrate <file>`.
 
+Remote grader refs use Go-module-style paths: `<host>/<owner>/<repo>[/path][#export]@<version>`. The remote module must provide a `waza.registry.yaml` manifest and export a config-only grader preset that expands to a built-in grader type. Run `waza get eval.yaml` after adding or changing refs so `waza.lock` records the resolved commit and digest.
+
 `results.json` is currently emitted at `schemaVersion` `1.2`. Version `1.1` added per-turn checkpoints (`runs[].checkpoints[]`, see #358) and the normalized `runs[].tool_events[]` array (`turn`, `sequence`, `tool_call_id`, `tool_name`, `args`, `result`, `success`, `error`, `duration_ms`; see #366). Version `1.2` adds `runs[].snapshot_path` for `waza run --snapshot` artifacts (#367) and the eval-level `adversarial:` block consumed by `waza adversarial --spec` (#365). See [docs/PRD](docs/PRD.md) and [schema-changes](site/src/content/docs/reference/schema-changes.md) for details.
 
 ### MCP Mock Servers

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/microsoft/waza/internal/models"
+	"github.com/microsoft/waza/internal/registry"
+	"github.com/spf13/cobra"
+)
+
+func newGetCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "get [eval.yaml | ref]",
+		Short: "Resolve remote grader refs and update waza.lock",
+		Long: `Resolve remote grader refs and update waza.lock.
+
+When given an eval YAML file, resolves every graders[].ref entry, downloads the
+module source into the Waza module cache, and writes a lockfile next to the eval.
+When given a single ref, resolves that ref and writes waza.lock in the current
+directory.`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			target := "eval.yaml"
+			if len(args) > 0 {
+				target = args[0]
+			}
+			resolver, err := registry.NewResolver()
+			if err != nil {
+				return err
+			}
+			return runGet(cmd.OutOrStdout(), cmd.Context(), resolver, target)
+		},
+	}
+	return cmd
+}
+
+type getResolver interface {
+	ResolveEvalLock(ctx context.Context, evalPath string) (*models.Lockfile, []models.LockfileGrader, error)
+	ResolveRefs(ctx context.Context, refs []string) ([]models.LockfileGrader, error)
+}
+
+func runGet(out io.Writer, ctx context.Context, resolver getResolver, target string) error {
+	target = strings.TrimSpace(target)
+	if target == "" {
+		target = "eval.yaml"
+	}
+	if isEvalYAMLTarget(target) {
+		lock, entries, err := resolver.ResolveEvalLock(ctx, target)
+		if err != nil {
+			return err
+		}
+		lockPath := filepath.Join(filepath.Dir(target), models.LockfileName)
+		if err := models.WriteLockfile(lockPath, lock); err != nil {
+			return fmt.Errorf("writing %s: %w", lockPath, err)
+		}
+		_, err = fmt.Fprintf(out, "Resolved %d remote grader ref(s); wrote %s\n", len(entries), lockPath)
+		return err
+	}
+
+	entries, err := resolver.ResolveRefs(ctx, []string{target})
+	if err != nil {
+		return err
+	}
+	lock := models.NewLockfile()
+	for _, entry := range entries {
+		lock.UpsertGrader(entry)
+	}
+	lockPath := models.LockfileName
+	if err := models.WriteLockfile(lockPath, lock); err != nil {
+		return fmt.Errorf("writing %s: %w", lockPath, err)
+	}
+	_, err = fmt.Fprintf(out, "Resolved %d remote grader ref(s); wrote %s\n", len(entries), lockPath)
+	return err
+}
+
+func isEvalYAMLTarget(target string) bool {
+	if info, err := os.Stat(target); err == nil && !info.IsDir() {
+		return true
+	}
+	ext := strings.ToLower(filepath.Ext(target))
+	return ext == ".yaml" || ext == ".yml"
+}
@@ -0,0 +1,61 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/microsoft/waza/internal/models"
+)
+
+type fakeGetResolver struct {
+	lock    *models.Lockfile
+	entries []models.LockfileGrader
+}
+
+func (f fakeGetResolver) ResolveEvalLock(context.Context, string) (*models.Lockfile, []models.LockfileGrader, error) {
+	return f.lock, f.entries, nil
+}
+
+func (f fakeGetResolver) ResolveRefs(context.Context, []string) ([]models.LockfileGrader, error) {
+	return f.entries, nil
+}
+
+func TestRootCommandHasGetSubcommand(t *testing.T) {
+	root := newRootCommand()
+	for _, cmd := range root.Commands() {
+		if cmd.Name() == "get" {
+			return
+		}
+	}
+	t.Fatalf("root command should have get subcommand")
+}
+
+func TestRunGetWritesLockForSingleRef(t *testing.T) {
+	dir := t.TempDir()
+	t.Chdir(dir)
+	lock := models.NewLockfile()
+	entry := models.LockfileGrader{
+		Ref:    "github.com/waza-evals/fact#factuality@v1.0.0",
+		Commit: "0123456789abcdef0123456789abcdef01234567",
+		Digest: "sha256:abc123",
+		URL:    "https://github.com/waza-evals/fact.git",
+	}
+	lock.UpsertGrader(entry)
+	var out bytes.Buffer
+
+	err := runGet(&out, context.Background(), fakeGetResolver{lock: lock, entries: []models.LockfileGrader{entry}}, entry.Ref)
+	if err != nil {
+		t.Fatalf("runGet() error = %v", err)
+	}
+	if out.String() != "Resolved 1 remote grader ref(s); wrote waza.lock\n" {
+		t.Fatalf("output = %q", out.String())
+	}
+	loaded, err := models.LoadLockfile(models.LockfileName)
+	if err != nil {
+		t.Fatalf("LoadLockfile() error = %v", err)
+	}
+	if _, ok := loaded.Grader(entry.Ref); !ok {
+		t.Fatalf("expected lock entry for %s", entry.Ref)
+	}
+}
@@ -34,6 +34,7 @@ import (
 	"github.com/microsoft/waza/internal/orchestration"
 	"github.com/microsoft/waza/internal/projectconfig"
 	"github.com/microsoft/waza/internal/recommend"
+	"github.com/microsoft/waza/internal/registry"
 	"github.com/microsoft/waza/internal/reporting"
 	"github.com/microsoft/waza/internal/session"
 	"github.com/microsoft/waza/internal/snapshot"
@@ -567,6 +568,17 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str
 	if err != nil {
 		return nil, fmt.Errorf("failed to load spec: %w", err)
 	}
+	resolver, err := registry.NewResolver()
+	if err != nil {
+		return nil, err
+	}
+	resolveCtx := context.Background()
+	if cmd != nil {
+		resolveCtx = cmd.Context()
+	}
+	if err := resolver.ExpandLockedGraders(resolveCtx, spec, specPath); err != nil {
+		return nil, err
+	}
 
 	// CLI flags override spec config
 	if parallel {

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/microsoft/waza/internal/models"
+	"github.com/microsoft/waza/internal/registry"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunCommandForSpecExpandsLockedRemoteGraders(t *testing.T) {
+	resetRunGlobals()
+	dir := t.TempDir()
+	cacheRoot := filepath.Join(dir, "cache")
+	t.Setenv("WAZA_MODULE_CACHE", cacheRoot)
+
+	ref := "example.com/acme/graders#factuality@v1.0.0"
+	commit := "0123456789abcdef0123456789abcdef01234567"
+	moduleDir := filepath.Join(cacheRoot, "example.com", "acme", "graders", commit)
+	require.NoError(t, os.MkdirAll(filepath.Join(moduleDir, "graders"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(moduleDir, "waza.registry.yaml"), []byte(`schema_version: 1
+module: example.com/acme/graders
+exports:
+  graders:
+    factuality:
+      path: graders/factuality.yaml
+`), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(moduleDir, "graders", "factuality.yaml"), []byte(`type: text
+name: factuality
+config:
+  contains: ["mock response"]
+`), 0o644))
+	digest, err := registry.DigestDirectory(moduleDir)
+	require.NoError(t, err)
+	lock := models.NewLockfile()
+	lock.UpsertGrader(models.LockfileGrader{
+		Ref:    ref,
+		Commit: commit,
+		Digest: digest,
+		URL:    "https://example.com/acme/graders.git",
+	})
+	require.NoError(t, models.WriteLockfile(filepath.Join(dir, models.LockfileName), lock))
+
+	taskPath := filepath.Join(dir, "task.yaml")
+	require.NoError(t, os.WriteFile(taskPath, []byte(`id: remote-ref-task
+name: Remote Ref Task
+prompt: Say mock response
+`), 0o644))
+	specPath := filepath.Join(dir, "eval.yaml")
+	require.NoError(t, os.WriteFile(specPath, []byte(`name: remote-ref-eval
+skill: test-skill
+config:
+  trials_per_task: 1
+  timeout_seconds: 10
+  executor: mock
+  model: mock-model
+tasks:
+  - task.yaml
+graders:
+  - ref: `+ref+`
+    name: remote_text
+metrics: []
+`), 0o644))
+
+	contextDir = dir
+	results, err := runCommandForSpec(nil, skillSpecPath{evalSpecPath: specPath}, nil)
+	require.NoError(t, err)
+	require.Len(t, results, 1)
+	require.NotNil(t, results[0].outcome)
+	require.Len(t, results[0].outcome.TestOutcomes, 1)
+	require.Contains(t, results[0].outcome.TestOutcomes[0].Runs[0].Validations, "remote_text")
+}
@@ -52,6 +52,7 @@ performance against predefined test cases.`,
 	// Add subcommands
 	cmd.AddCommand(newRunCommand())
 	cmd.AddCommand(newInitCommand())
+	cmd.AddCommand(newGetCommand())
 	cmd.AddCommand(tokens.NewCommand())
 	cmd.AddCommand(newCompareCommand())
 	cmd.AddCommand(newGateCommand())

@@ -10,6 +10,7 @@ Waza helps you:
 - **Evaluate custom agents** (`.agent.md` files) with automatic tool constraint validation
 - **Create test suites** with realistic test cases and validation rules  
 - **Run evaluations** against different AI models to measure skill effectiveness
+- **Reuse remote grader presets** with Go-module-style `ref` entries pinned by `waza.lock`
 - **Compare results** across models and versions to track improvement
 - **View metrics** in an interactive dashboard with live results, trends, and detailed analysis
 
@@ -201,6 +202,15 @@ Execute the benchmark:
 waza run evals/code-explainer/eval.yaml --context-dir evals/code-explainer/fixtures -v
 ```
 
+If your eval uses remote grader presets, resolve them first:
+
+```bash
+waza get evals/code-explainer/eval.yaml
+waza run evals/code-explainer/eval.yaml --context-dir evals/code-explainer/fixtures -v
+```
+
+Remote refs use `<host>/<owner>/<repo>[/path][#export]@<version>` in `graders[].ref`. `waza get` downloads the module into `~/.waza/cache/{host}/{org}/{repo}/{sha}/` and writes `waza.lock`; `waza run` uses the lock and refuses missing or digest-mismatched cache entries.
+
 **Output:**
 - `✓ Passed` — Task passed all validators
 - `✗ Failed` — Task failed one or more validators