From 11d110c9d1d88b6af5d6165fadd422f61a8e4684 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:39:41 -0400 Subject: [PATCH] feat: add remote grader refs #15 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c823d89f-d737-4983-b78a-5d4757a3b42e --- README.md | 20 + cmd/waza/cmd_get.go | 87 ++++ cmd/waza/cmd_get_test.go | 61 +++ cmd/waza/cmd_run.go | 12 + cmd/waza/cmd_run_remote_refs_test.go | 74 +++ cmd/waza/root.go | 1 + docs/GUIDE.md | 10 + internal/models/lockfile.go | 128 +++++ internal/models/lockfile_test.go | 92 ++++ internal/models/spec.go | 13 +- internal/models/spec_test.go | 51 ++ internal/registry/ref.go | 62 +++ internal/registry/ref_test.go | 28 + internal/registry/resolver.go | 568 +++++++++++++++++++++ internal/registry/resolver_test.go | 167 ++++++ site/src/content/docs/guides/eval-yaml.mdx | 15 + site/src/content/docs/guides/graders.mdx | 12 + site/src/content/docs/reference/cli.mdx | 17 + 18 files changed, 1417 insertions(+), 1 deletion(-) create mode 100644 cmd/waza/cmd_get.go create mode 100644 cmd/waza/cmd_get_test.go create mode 100644 cmd/waza/cmd_run_remote_refs_test.go create mode 100644 internal/models/lockfile.go create mode 100644 internal/models/lockfile_test.go create mode 100644 internal/registry/ref.go create mode 100644 internal/registry/ref_test.go create mode 100644 internal/registry/resolver.go create mode 100644 internal/registry/resolver_test.go diff --git a/README.md b/README.md index 004137ed..ed71add7 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,9 @@ waza suggest skills/my-skill --apply waza spec verify skills/my-skill evals/my-skill/eval.yaml waza spec verify skills/my-skill evals/my-skill/eval.yaml --fail --format github-actions +# Resolve remote grader refs and write waza.lock +waza get evals/my-skill/eval.yaml + # Note: 'generate' is available as an alias for 'new' (see below for new command) # Note: Custom agents (.agent.md) are supported — see https://microsoft.github.io/waza/guides/custom-agents/ @@ -389,6 +392,15 @@ Cached results are automatically invalidated when: **Note:** Caching is automatically disabled for evaluations using non-deterministic graders (`behavior`, `prompt`). +### `waza get [eval.yaml | ref]` + +Resolve remote grader refs and write `waza.lock`. When passed an eval file, `waza get` resolves every `graders[].ref`, downloads module contents into `~/.waza/cache/{host}/{org}/{repo}/{sha}/`, and pins each ref to a commit SHA and `sha256:` content digest. `waza run` requires a valid lock and cache entry for remote refs; it does not silently resolve unlocked refs during a run. + +```bash +waza get eval.yaml +waza get github.com/waza-evals/fact#factuality@v1.0.0 +``` + **Exit Codes** The `run` command uses exit codes to enable CI/CD integration: @@ -1086,6 +1098,12 @@ mcp_mocks: issues: [] graders: + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + name: factuality_strict + weight: 2.0 + config: + threshold: 0.9 + - type: text name: pattern_check config: @@ -1114,6 +1132,8 @@ tasks: `schemaVersion` uses `MAJOR.MINOR` format. Missing values are interpreted as the current schema version (currently `1.2`). Readers allow same-major minor additions with warnings for unknown fields, but reject different majors with a hint to run `waza migrate `. +Remote grader refs use Go-module-style paths: `//[/path][#export]@`. The remote module must provide a `waza.registry.yaml` manifest and export a config-only grader preset that expands to a built-in grader type. Run `waza get eval.yaml` after adding or changing refs so `waza.lock` records the resolved commit and digest. + `results.json` is currently emitted at `schemaVersion` `1.2`. Version `1.1` added per-turn checkpoints (`runs[].checkpoints[]`, see #358) and the normalized `runs[].tool_events[]` array (`turn`, `sequence`, `tool_call_id`, `tool_name`, `args`, `result`, `success`, `error`, `duration_ms`; see #366). Version `1.2` adds `runs[].snapshot_path` for `waza run --snapshot` artifacts (#367) and the eval-level `adversarial:` block consumed by `waza adversarial --spec` (#365). See [docs/PRD](docs/PRD.md) and [schema-changes](site/src/content/docs/reference/schema-changes.md) for details. ### MCP Mock Servers diff --git a/cmd/waza/cmd_get.go b/cmd/waza/cmd_get.go new file mode 100644 index 00000000..736a3a0d --- /dev/null +++ b/cmd/waza/cmd_get.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/microsoft/waza/internal/models" + "github.com/microsoft/waza/internal/registry" + "github.com/spf13/cobra" +) + +func newGetCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "get [eval.yaml | ref]", + Short: "Resolve remote grader refs and update waza.lock", + Long: `Resolve remote grader refs and update waza.lock. + +When given an eval YAML file, resolves every graders[].ref entry, downloads the +module source into the Waza module cache, and writes a lockfile next to the eval. +When given a single ref, resolves that ref and writes waza.lock in the current +directory.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + target := "eval.yaml" + if len(args) > 0 { + target = args[0] + } + resolver, err := registry.NewResolver() + if err != nil { + return err + } + return runGet(cmd.OutOrStdout(), cmd.Context(), resolver, target) + }, + } + return cmd +} + +type getResolver interface { + ResolveEvalLock(ctx context.Context, evalPath string) (*models.Lockfile, []models.LockfileGrader, error) + ResolveRefs(ctx context.Context, refs []string) ([]models.LockfileGrader, error) +} + +func runGet(out io.Writer, ctx context.Context, resolver getResolver, target string) error { + target = strings.TrimSpace(target) + if target == "" { + target = "eval.yaml" + } + if isEvalYAMLTarget(target) { + lock, entries, err := resolver.ResolveEvalLock(ctx, target) + if err != nil { + return err + } + lockPath := filepath.Join(filepath.Dir(target), models.LockfileName) + if err := models.WriteLockfile(lockPath, lock); err != nil { + return fmt.Errorf("writing %s: %w", lockPath, err) + } + _, err = fmt.Fprintf(out, "Resolved %d remote grader ref(s); wrote %s\n", len(entries), lockPath) + return err + } + + entries, err := resolver.ResolveRefs(ctx, []string{target}) + if err != nil { + return err + } + lock := models.NewLockfile() + for _, entry := range entries { + lock.UpsertGrader(entry) + } + lockPath := models.LockfileName + if err := models.WriteLockfile(lockPath, lock); err != nil { + return fmt.Errorf("writing %s: %w", lockPath, err) + } + _, err = fmt.Fprintf(out, "Resolved %d remote grader ref(s); wrote %s\n", len(entries), lockPath) + return err +} + +func isEvalYAMLTarget(target string) bool { + if info, err := os.Stat(target); err == nil && !info.IsDir() { + return true + } + ext := strings.ToLower(filepath.Ext(target)) + return ext == ".yaml" || ext == ".yml" +} diff --git a/cmd/waza/cmd_get_test.go b/cmd/waza/cmd_get_test.go new file mode 100644 index 00000000..da9ce696 --- /dev/null +++ b/cmd/waza/cmd_get_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "bytes" + "context" + "testing" + + "github.com/microsoft/waza/internal/models" +) + +type fakeGetResolver struct { + lock *models.Lockfile + entries []models.LockfileGrader +} + +func (f fakeGetResolver) ResolveEvalLock(context.Context, string) (*models.Lockfile, []models.LockfileGrader, error) { + return f.lock, f.entries, nil +} + +func (f fakeGetResolver) ResolveRefs(context.Context, []string) ([]models.LockfileGrader, error) { + return f.entries, nil +} + +func TestRootCommandHasGetSubcommand(t *testing.T) { + root := newRootCommand() + for _, cmd := range root.Commands() { + if cmd.Name() == "get" { + return + } + } + t.Fatalf("root command should have get subcommand") +} + +func TestRunGetWritesLockForSingleRef(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + lock := models.NewLockfile() + entry := models.LockfileGrader{ + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", + Commit: "0123456789abcdef0123456789abcdef01234567", + Digest: "sha256:abc123", + URL: "https://github.com/waza-evals/fact.git", + } + lock.UpsertGrader(entry) + var out bytes.Buffer + + err := runGet(&out, context.Background(), fakeGetResolver{lock: lock, entries: []models.LockfileGrader{entry}}, entry.Ref) + if err != nil { + t.Fatalf("runGet() error = %v", err) + } + if out.String() != "Resolved 1 remote grader ref(s); wrote waza.lock\n" { + t.Fatalf("output = %q", out.String()) + } + loaded, err := models.LoadLockfile(models.LockfileName) + if err != nil { + t.Fatalf("LoadLockfile() error = %v", err) + } + if _, ok := loaded.Grader(entry.Ref); !ok { + t.Fatalf("expected lock entry for %s", entry.Ref) + } +} diff --git a/cmd/waza/cmd_run.go b/cmd/waza/cmd_run.go index e02b3664..42199ee2 100644 --- a/cmd/waza/cmd_run.go +++ b/cmd/waza/cmd_run.go @@ -34,6 +34,7 @@ import ( "github.com/microsoft/waza/internal/orchestration" "github.com/microsoft/waza/internal/projectconfig" "github.com/microsoft/waza/internal/recommend" + "github.com/microsoft/waza/internal/registry" "github.com/microsoft/waza/internal/reporting" "github.com/microsoft/waza/internal/session" "github.com/microsoft/waza/internal/snapshot" @@ -567,6 +568,17 @@ func runCommandForSpec(cmd *cobra.Command, sp skillSpecPath, defaultSkills []str if err != nil { return nil, fmt.Errorf("failed to load spec: %w", err) } + resolver, err := registry.NewResolver() + if err != nil { + return nil, err + } + resolveCtx := context.Background() + if cmd != nil { + resolveCtx = cmd.Context() + } + if err := resolver.ExpandLockedGraders(resolveCtx, spec, specPath); err != nil { + return nil, err + } // CLI flags override spec config if parallel { diff --git a/cmd/waza/cmd_run_remote_refs_test.go b/cmd/waza/cmd_run_remote_refs_test.go new file mode 100644 index 00000000..195853e9 --- /dev/null +++ b/cmd/waza/cmd_run_remote_refs_test.go @@ -0,0 +1,74 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/microsoft/waza/internal/models" + "github.com/microsoft/waza/internal/registry" + "github.com/stretchr/testify/require" +) + +func TestRunCommandForSpecExpandsLockedRemoteGraders(t *testing.T) { + resetRunGlobals() + dir := t.TempDir() + cacheRoot := filepath.Join(dir, "cache") + t.Setenv("WAZA_MODULE_CACHE", cacheRoot) + + ref := "example.com/acme/graders#factuality@v1.0.0" + commit := "0123456789abcdef0123456789abcdef01234567" + moduleDir := filepath.Join(cacheRoot, "example.com", "acme", "graders", commit) + require.NoError(t, os.MkdirAll(filepath.Join(moduleDir, "graders"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(moduleDir, "waza.registry.yaml"), []byte(`schema_version: 1 +module: example.com/acme/graders +exports: + graders: + factuality: + path: graders/factuality.yaml +`), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(moduleDir, "graders", "factuality.yaml"), []byte(`type: text +name: factuality +config: + contains: ["mock response"] +`), 0o644)) + digest, err := registry.DigestDirectory(moduleDir) + require.NoError(t, err) + lock := models.NewLockfile() + lock.UpsertGrader(models.LockfileGrader{ + Ref: ref, + Commit: commit, + Digest: digest, + URL: "https://example.com/acme/graders.git", + }) + require.NoError(t, models.WriteLockfile(filepath.Join(dir, models.LockfileName), lock)) + + taskPath := filepath.Join(dir, "task.yaml") + require.NoError(t, os.WriteFile(taskPath, []byte(`id: remote-ref-task +name: Remote Ref Task +prompt: Say mock response +`), 0o644)) + specPath := filepath.Join(dir, "eval.yaml") + require.NoError(t, os.WriteFile(specPath, []byte(`name: remote-ref-eval +skill: test-skill +config: + trials_per_task: 1 + timeout_seconds: 10 + executor: mock + model: mock-model +tasks: + - task.yaml +graders: + - ref: `+ref+` + name: remote_text +metrics: [] +`), 0o644)) + + contextDir = dir + results, err := runCommandForSpec(nil, skillSpecPath{evalSpecPath: specPath}, nil) + require.NoError(t, err) + require.Len(t, results, 1) + require.NotNil(t, results[0].outcome) + require.Len(t, results[0].outcome.TestOutcomes, 1) + require.Contains(t, results[0].outcome.TestOutcomes[0].Runs[0].Validations, "remote_text") +} diff --git a/cmd/waza/root.go b/cmd/waza/root.go index ba953de2..466341f1 100644 --- a/cmd/waza/root.go +++ b/cmd/waza/root.go @@ -52,6 +52,7 @@ performance against predefined test cases.`, // Add subcommands cmd.AddCommand(newRunCommand()) cmd.AddCommand(newInitCommand()) + cmd.AddCommand(newGetCommand()) cmd.AddCommand(tokens.NewCommand()) cmd.AddCommand(newCompareCommand()) cmd.AddCommand(newGateCommand()) diff --git a/docs/GUIDE.md b/docs/GUIDE.md index cb2e3223..fa3c5a0f 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -10,6 +10,7 @@ Waza helps you: - **Evaluate custom agents** (`.agent.md` files) with automatic tool constraint validation - **Create test suites** with realistic test cases and validation rules - **Run evaluations** against different AI models to measure skill effectiveness +- **Reuse remote grader presets** with Go-module-style `ref` entries pinned by `waza.lock` - **Compare results** across models and versions to track improvement - **View metrics** in an interactive dashboard with live results, trends, and detailed analysis @@ -201,6 +202,15 @@ Execute the benchmark: waza run evals/code-explainer/eval.yaml --context-dir evals/code-explainer/fixtures -v ``` +If your eval uses remote grader presets, resolve them first: + +```bash +waza get evals/code-explainer/eval.yaml +waza run evals/code-explainer/eval.yaml --context-dir evals/code-explainer/fixtures -v +``` + +Remote refs use `//[/path][#export]@` in `graders[].ref`. `waza get` downloads the module into `~/.waza/cache/{host}/{org}/{repo}/{sha}/` and writes `waza.lock`; `waza run` uses the lock and refuses missing or digest-mismatched cache entries. + **Output:** - `✓ Passed` — Task passed all validators - `✗ Failed` — Task failed one or more validators diff --git a/internal/models/lockfile.go b/internal/models/lockfile.go new file mode 100644 index 00000000..2dc59874 --- /dev/null +++ b/internal/models/lockfile.go @@ -0,0 +1,128 @@ +package models + +import ( + "bytes" + "fmt" + "os" + "sort" + + "gopkg.in/yaml.v3" +) + +const LockfileName = "waza.lock" + +// Lockfile pins remote grader refs to immutable source and content digests. +type Lockfile struct { + SchemaVersion int `yaml:"schema_version"` + Graders []LockfileGrader `yaml:"graders"` + byRef map[string]int `yaml:"-"` +} + +type LockfileGrader struct { + Ref string `yaml:"ref"` + Commit string `yaml:"commit"` + Digest string `yaml:"digest"` + URL string `yaml:"url"` +} + +func NewLockfile() *Lockfile { + return &Lockfile{SchemaVersion: 1} +} + +func LoadLockfile(path string) (*Lockfile, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var lock Lockfile + decoder := yaml.NewDecoder(bytes.NewReader(data)) + if err := decoder.Decode(&lock); err != nil { + return nil, fmt.Errorf("parsing lockfile %s: %w", path, err) + } + if lock.SchemaVersion == 0 { + lock.SchemaVersion = 1 + } + if err := lock.Validate(); err != nil { + return nil, fmt.Errorf("validating lockfile %s: %w", path, err) + } + return &lock, nil +} + +func WriteLockfile(path string, lock *Lockfile) error { + if lock == nil { + lock = NewLockfile() + } + if lock.SchemaVersion == 0 { + lock.SchemaVersion = 1 + } + if err := lock.Validate(); err != nil { + return err + } + sort.SliceStable(lock.Graders, func(i, j int) bool { + return lock.Graders[i].Ref < lock.Graders[j].Ref + }) + data, err := yaml.Marshal(lock) + if err != nil { + return fmt.Errorf("encoding lockfile: %w", err) + } + return os.WriteFile(path, data, 0o644) +} + +func (l *Lockfile) Validate() error { + if l == nil { + return fmt.Errorf("lockfile is nil") + } + if l.SchemaVersion != 1 { + return fmt.Errorf("unsupported lockfile schema_version %d", l.SchemaVersion) + } + seen := make(map[string]bool, len(l.Graders)) + for i, g := range l.Graders { + if g.Ref == "" { + return fmt.Errorf("graders[%d].ref is required", i) + } + if g.Commit == "" { + return fmt.Errorf("graders[%d].commit is required", i) + } + if g.Digest == "" { + return fmt.Errorf("graders[%d].digest is required", i) + } + if g.URL == "" { + return fmt.Errorf("graders[%d].url is required", i) + } + if seen[g.Ref] { + return fmt.Errorf("duplicate lock entry for ref %q", g.Ref) + } + seen[g.Ref] = true + } + l.rebuildIndex() + return nil +} + +func (l *Lockfile) UpsertGrader(entry LockfileGrader) { + if l.SchemaVersion == 0 { + l.SchemaVersion = 1 + } + l.rebuildIndex() + if i, ok := l.byRef[entry.Ref]; ok { + l.Graders[i] = entry + return + } + l.Graders = append(l.Graders, entry) + l.byRef[entry.Ref] = len(l.Graders) - 1 +} + +func (l *Lockfile) Grader(ref string) (LockfileGrader, bool) { + l.rebuildIndex() + i, ok := l.byRef[ref] + if !ok { + return LockfileGrader{}, false + } + return l.Graders[i], true +} + +func (l *Lockfile) rebuildIndex() { + l.byRef = make(map[string]int, len(l.Graders)) + for i, g := range l.Graders { + l.byRef[g.Ref] = i + } +} diff --git a/internal/models/lockfile_test.go b/internal/models/lockfile_test.go new file mode 100644 index 00000000..2c7c50a3 --- /dev/null +++ b/internal/models/lockfile_test.go @@ -0,0 +1,92 @@ +package models + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLockfileReadWriteRoundTrip(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, LockfileName) + lock := NewLockfile() + lock.UpsertGrader(LockfileGrader{ + Ref: "github.com/waza-evals/fact#factuality@v1.0.0", + Commit: "0123456789abcdef0123456789abcdef01234567", + Digest: "sha256:abc123", + URL: "https://github.com/waza-evals/fact.git", + }) + + if err := WriteLockfile(path, lock); err != nil { + t.Fatalf("WriteLockfile() error = %v", err) + } + + loaded, err := LoadLockfile(path) + if err != nil { + t.Fatalf("LoadLockfile() error = %v", err) + } + entry, ok := loaded.Grader("github.com/waza-evals/fact#factuality@v1.0.0") + if !ok { + t.Fatalf("expected lock entry") + } + if entry.Commit != "0123456789abcdef0123456789abcdef01234567" { + t.Fatalf("Commit = %q", entry.Commit) + } + if entry.Digest != "sha256:abc123" { + t.Fatalf("Digest = %q", entry.Digest) + } + if entry.URL != "https://github.com/waza-evals/fact.git" { + t.Fatalf("URL = %q", entry.URL) + } +} + +func TestLoadLockfileRejectsDuplicateRefs(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, LockfileName) + data := []byte(`schema_version: 1 +graders: + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + commit: abc + digest: sha256:one + url: https://github.com/waza-evals/fact.git + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + commit: def + digest: sha256:two + url: https://github.com/waza-evals/fact.git +`) + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write lockfile: %v", err) + } + if _, err := LoadLockfile(path); err == nil { + t.Fatalf("expected duplicate ref error") + } +} + +func TestWriteLockfileKeepsLookupIndexInSortedOrder(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, LockfileName) + lock := NewLockfile() + lock.UpsertGrader(LockfileGrader{ + Ref: "github.com/waza-evals/z#grader@v1.0.0", + Commit: "z", + Digest: "sha256:z", + URL: "https://github.com/waza-evals/z.git", + }) + lock.UpsertGrader(LockfileGrader{ + Ref: "github.com/waza-evals/a#grader@v1.0.0", + Commit: "a", + Digest: "sha256:a", + URL: "https://github.com/waza-evals/a.git", + }) + + if err := WriteLockfile(path, lock); err != nil { + t.Fatalf("WriteLockfile() error = %v", err) + } + entry, ok := lock.Grader("github.com/waza-evals/a#grader@v1.0.0") + if !ok { + t.Fatalf("expected sorted lock entry lookup") + } + if entry.Commit != "a" { + t.Fatalf("Commit = %q, want a", entry.Commit) + } +} diff --git a/internal/models/spec.go b/internal/models/spec.go index 6c4da922..847fcc72 100644 --- a/internal/models/spec.go +++ b/internal/models/spec.go @@ -56,6 +56,7 @@ type strictEvalSpec struct { } type strictGrader struct { + Ref string `yaml:"ref,omitempty"` Kind GraderKind `yaml:"type"` Identifier string `yaml:"name"` ScriptPath string `yaml:"script,omitempty"` @@ -193,6 +194,7 @@ func (c *Config) ShouldInjectSkillBody() bool { // GraderConfig defines a validator/grader type GraderConfig struct { + Ref string `yaml:"ref,omitempty" json:"ref,omitempty"` Kind GraderKind `yaml:"type" json:"kind"` Identifier string `yaml:"name" json:"identifier"` ScriptPath string `yaml:"script,omitempty" json:"script_path,omitempty"` @@ -204,6 +206,7 @@ type GraderConfig struct { func (g *GraderConfig) UnmarshalYAML(node *yaml.Node) error { type rawGraderConfig struct { + Ref string `yaml:"ref,omitempty"` Kind GraderKind `yaml:"type"` Identifier string `yaml:"name"` ScriptPath string `yaml:"script,omitempty"` @@ -227,11 +230,16 @@ func (g *GraderConfig) UnmarshalYAML(node *yaml.Node) error { return err } - params, err := decodeGraderParameters(raw.Kind, &raw.Parameters) + paramsKind := raw.Kind + if raw.Ref != "" && raw.Kind == "" { + paramsKind = "" + } + params, err := decodeGraderParameters(paramsKind, &raw.Parameters) if err != nil { return fmt.Errorf("invalid grader config for %q (type %q): %w", raw.Identifier, raw.Kind, err) } + g.Ref = raw.Ref g.Kind = raw.Kind g.Identifier = raw.Identifier g.ScriptPath = raw.ScriptPath @@ -258,6 +266,9 @@ func (g *GraderConfig) EffectiveWeight() float64 { // Validate checks that the grader config has required fields for its type. func (g *GraderConfig) Validate() error { + if g.Ref != "" && g.Kind == "" { + return nil + } switch g.Kind { case GraderKindInlineScript: params, ok := g.Parameters.(InlineScriptGraderParameters) diff --git a/internal/models/spec_test.go b/internal/models/spec_test.go index ce2cf562..39079c10 100644 --- a/internal/models/spec_test.go +++ b/internal/models/spec_test.go @@ -621,6 +621,57 @@ config: }) } +func TestLoadEvalSpec_AllowsRemoteGraderRefWithoutType(t *testing.T) { + tempDir := t.TempDir() + specPath := filepath.Join(tempDir, "remote.yaml") + yamlContent := `name: remote-graders +skill: test +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: mock +graders: + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + name: factuality_strict + weight: 2 + config: + threshold: 0.9 +metrics: [] +tasks: [] +` + if err := os.WriteFile(specPath, []byte(yamlContent), 0o644); err != nil { + t.Fatalf("Failed to write spec file: %v", err) + } + + spec, err := LoadEvalSpec(specPath) + if err != nil { + t.Fatalf("LoadEvalSpec() error = %v", err) + } + if len(spec.Graders) != 1 { + t.Fatalf("Expected 1 grader, got %d", len(spec.Graders)) + } + grader := spec.Graders[0] + if grader.Ref != "github.com/waza-evals/fact#factuality@v1.0.0" { + t.Fatalf("Ref = %q", grader.Ref) + } + if grader.Kind != "" { + t.Fatalf("Kind = %q, want empty before resolver expansion", grader.Kind) + } + if grader.Identifier != "factuality_strict" { + t.Fatalf("Identifier = %q", grader.Identifier) + } + if grader.Weight != 2 { + t.Fatalf("Weight = %v", grader.Weight) + } + params, ok := grader.Parameters.(GenericGraderParameters) + if !ok { + t.Fatalf("Parameters = %T, want GenericGraderParameters", grader.Parameters) + } + if got := params["threshold"]; got != 0.9 { + t.Fatalf("threshold = %#v", got) + } +} + func TestConfig_AllSkillsDisabled(t *testing.T) { tests := []struct { name string diff --git a/internal/registry/ref.go b/internal/registry/ref.go new file mode 100644 index 00000000..dab2c3c2 --- /dev/null +++ b/internal/registry/ref.go @@ -0,0 +1,62 @@ +package registry + +import ( + "fmt" + "path" + "strings" +) + +// Ref is a Go-module-style Waza registry reference. +type Ref struct { + Raw string + Host string + Owner string + Repo string + Path string + Export string + Version string +} + +func ParseRef(raw string) (Ref, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return Ref{}, fmt.Errorf("ref is empty") + } + base, version, ok := strings.Cut(raw, "@") + if !ok || strings.TrimSpace(version) == "" { + return Ref{}, fmt.Errorf("ref %q must include @", raw) + } + base, export, _ := strings.Cut(base, "#") + parts := strings.Split(base, "/") + if len(parts) < 3 { + return Ref{}, fmt.Errorf("ref %q must use //[/path]@", raw) + } + for i, part := range parts[:3] { + if part == "" { + return Ref{}, fmt.Errorf("ref %q has empty path segment %d", raw, i) + } + } + refPath := "" + if len(parts) > 3 { + refPath = path.Clean(strings.Join(parts[3:], "/")) + if refPath == "." { + refPath = "" + } + if strings.HasPrefix(refPath, "../") || refPath == ".." || strings.HasPrefix(refPath, "/") { + return Ref{}, fmt.Errorf("ref %q has invalid module path %q", raw, refPath) + } + } + return Ref{ + Raw: raw, + Host: parts[0], + Owner: parts[1], + Repo: parts[2], + Path: refPath, + Export: strings.TrimSpace(export), + Version: strings.TrimSpace(version), + }, nil +} + +func (r Ref) ModulePath() string { + return r.Host + "/" + r.Owner + "/" + r.Repo +} diff --git a/internal/registry/ref_test.go b/internal/registry/ref_test.go new file mode 100644 index 00000000..cc7daab1 --- /dev/null +++ b/internal/registry/ref_test.go @@ -0,0 +1,28 @@ +package registry + +import "testing" + +func TestParseRef(t *testing.T) { + ref, err := ParseRef("github.com/waza-evals/fact/graders/factuality#strict@v1.2.3") + if err != nil { + t.Fatalf("ParseRef() error = %v", err) + } + if ref.Host != "github.com" || ref.Owner != "waza-evals" || ref.Repo != "fact" { + t.Fatalf("unexpected module parts: %#v", ref) + } + if ref.Path != "graders/factuality" { + t.Fatalf("Path = %q", ref.Path) + } + if ref.Export != "strict" { + t.Fatalf("Export = %q", ref.Export) + } + if ref.Version != "v1.2.3" { + t.Fatalf("Version = %q", ref.Version) + } +} + +func TestParseRefRequiresVersion(t *testing.T) { + if _, err := ParseRef("github.com/waza-evals/fact#factuality"); err == nil { + t.Fatalf("expected missing version error") + } +} diff --git a/internal/registry/resolver.go b/internal/registry/resolver.go new file mode 100644 index 00000000..33b08c02 --- /dev/null +++ b/internal/registry/resolver.go @@ -0,0 +1,568 @@ +package registry + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + + "github.com/microsoft/waza/internal/models" + "gopkg.in/yaml.v3" +) + +type GitURLFunc func(Ref) string + +type Resolver struct { + cacheRoot string + gitURL GitURLFunc +} + +type ResolverOption func(*Resolver) + +func WithCacheRoot(path string) ResolverOption { + return func(r *Resolver) { + r.cacheRoot = path + } +} + +func WithGitURLFunc(fn GitURLFunc) ResolverOption { + return func(r *Resolver) { + r.gitURL = fn + } +} + +func NewResolver(opts ...ResolverOption) (*Resolver, error) { + cacheRoot, err := DefaultCacheRoot() + if err != nil { + return nil, err + } + r := &Resolver{ + cacheRoot: cacheRoot, + gitURL: func(ref Ref) string { + return "https://" + ref.ModulePath() + ".git" + }, + } + for _, opt := range opts { + opt(r) + } + if r.cacheRoot == "" { + return nil, fmt.Errorf("module cache root is empty") + } + return r, nil +} + +func DefaultCacheRoot() (string, error) { + if dir := os.Getenv("WAZA_MODULE_CACHE"); dir != "" { + return dir, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", fmt.Errorf("resolving home directory: %w", err) + } + return filepath.Join(home, ".waza", "cache"), nil +} + +func CollectGraderRefs(spec *models.EvalSpec) []string { + seen := map[string]bool{} + var refs []string + for _, grader := range spec.Graders { + if grader.Ref == "" || seen[grader.Ref] { + continue + } + seen[grader.Ref] = true + refs = append(refs, grader.Ref) + } + return refs +} + +func (r *Resolver) ResolveRefs(ctx context.Context, refs []string) ([]models.LockfileGrader, error) { + entries := make([]models.LockfileGrader, 0, len(refs)) + for _, raw := range refs { + entry, err := r.ResolveRef(ctx, raw) + if err != nil { + return nil, err + } + entries = append(entries, entry) + } + return entries, nil +} + +func (r *Resolver) ResolveRef(ctx context.Context, raw string) (models.LockfileGrader, error) { + ref, err := ParseRef(raw) + if err != nil { + return models.LockfileGrader{}, err + } + url := r.gitURL(ref) + commit, cacheDir, err := r.ensureCached(ctx, ref, url) + if err != nil { + return models.LockfileGrader{}, fmt.Errorf("resolving %s: %w", raw, err) + } + digest, err := DigestDirectory(cacheDir) + if err != nil { + return models.LockfileGrader{}, fmt.Errorf("digesting %s: %w", cacheDir, err) + } + entry := models.LockfileGrader{ + Ref: raw, + Commit: commit, + Digest: digest, + URL: url, + } + if _, err := r.loadGraderPreset(ref, cacheDir); err != nil { + return models.LockfileGrader{}, fmt.Errorf("loading %s: %w", raw, err) + } + return entry, nil +} + +func (r *Resolver) ResolveEvalLock(ctx context.Context, evalPath string) (*models.Lockfile, []models.LockfileGrader, error) { + spec, err := models.LoadEvalSpec(evalPath) + if err != nil { + return nil, nil, err + } + refs := CollectGraderRefs(spec) + if len(refs) == 0 { + return nil, nil, fmt.Errorf("no remote grader refs found in %s", evalPath) + } + entries, err := r.ResolveRefs(ctx, refs) + if err != nil { + return nil, nil, err + } + lock := models.NewLockfile() + for _, entry := range entries { + lock.UpsertGrader(entry) + } + return lock, entries, nil +} + +func (r *Resolver) ExpandLockedGraders(ctx context.Context, spec *models.EvalSpec, evalPath string) error { + refs := CollectGraderRefs(spec) + if len(refs) == 0 { + return nil + } + lockPath := filepath.Join(filepath.Dir(evalPath), models.LockfileName) + lock, err := models.LoadLockfile(lockPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("eval contains remote grader refs but %s is missing; run `waza get %s` first", lockPath, evalPath) + } + return err + } + for i, grader := range spec.Graders { + if grader.Ref == "" { + continue + } + entry, ok := lock.Grader(grader.Ref) + if !ok { + return fmt.Errorf("remote grader ref %q is not present in %s; run `waza get %s`", grader.Ref, lockPath, evalPath) + } + ref, err := ParseRef(grader.Ref) + if err != nil { + return err + } + preset, err := r.LoadLockedGrader(ctx, ref, entry) + if err != nil { + return err + } + merged, err := MergeGraderConfig(preset, grader) + if err != nil { + return fmt.Errorf("merging remote grader %q: %w", grader.Ref, err) + } + spec.Graders[i] = merged + } + return nil +} + +func (r *Resolver) LoadLockedGrader(ctx context.Context, ref Ref, entry models.LockfileGrader) (models.GraderConfig, error) { + _ = ctx + cacheDir := r.cacheDir(ref, entry.Commit) + if _, err := os.Stat(cacheDir); err != nil { + if os.IsNotExist(err) { + return models.GraderConfig{}, fmt.Errorf("module not available offline for ref %q at %s; run `waza get` while online", entry.Ref, cacheDir) + } + return models.GraderConfig{}, err + } + digest, err := DigestDirectory(cacheDir) + if err != nil { + return models.GraderConfig{}, err + } + if digest != entry.Digest { + return models.GraderConfig{}, fmt.Errorf("digest mismatch for ref %q: lock has %s, cache has %s", entry.Ref, entry.Digest, digest) + } + return r.loadGraderPreset(ref, cacheDir) +} + +func (r *Resolver) ensureCached(ctx context.Context, ref Ref, url string) (string, string, error) { + tmp, err := os.MkdirTemp("", "waza-module-*") + if err != nil { + return "", "", err + } + defer func() { + if err := os.RemoveAll(tmp); err != nil { + slog.Warn("failed to remove temporary module mirror", "path", tmp, "error", err) + } + }() + + mirror := filepath.Join(tmp, "repo.git") + if err := runGit(ctx, "", "clone", "--quiet", "--mirror", url, mirror); err != nil { + return "", "", err + } + commitBytes, err := gitOutput(ctx, "", "--git-dir", mirror, "rev-parse", "--verify", ref.Version+"^{commit}") + if err != nil { + return "", "", fmt.Errorf("resolving version %q: %w", ref.Version, err) + } + commit := strings.TrimSpace(string(commitBytes)) + cacheDir := r.cacheDir(ref, commit) + if _, err := os.Stat(cacheDir); err == nil { + return commit, cacheDir, nil + } else if !os.IsNotExist(err) { + return "", "", err + } + parent := filepath.Dir(cacheDir) + if err := os.MkdirAll(parent, 0o755); err != nil { + return "", "", err + } + extractDir, err := os.MkdirTemp(parent, ".extract-*") + if err != nil { + return "", "", err + } + defer func() { + if err := os.RemoveAll(extractDir); err != nil { + slog.Warn("failed to remove temporary module extract", "path", extractDir, "error", err) + } + }() + + archive, err := gitOutput(ctx, "", "--git-dir", mirror, "archive", "--format=tar", commit) + if err != nil { + return "", "", fmt.Errorf("archiving commit %s: %w", commit, err) + } + if err := extractTar(bytes.NewReader(archive), extractDir); err != nil { + return "", "", err + } + if err := os.Rename(extractDir, cacheDir); err != nil { + return "", "", err + } + return commit, cacheDir, nil +} + +func (r *Resolver) cacheDir(ref Ref, commit string) string { + return filepath.Join(r.cacheRoot, ref.Host, ref.Owner, ref.Repo, commit) +} + +type manifest struct { + SchemaVersion int `yaml:"schema_version"` + Exports struct { + Graders map[string]manifestExport `yaml:"graders"` + } `yaml:"exports"` +} + +type manifestExport struct { + Path string `yaml:"path"` + Description string `yaml:"description,omitempty"` +} + +func (r *Resolver) loadGraderPreset(ref Ref, moduleDir string) (models.GraderConfig, error) { + if ref.Export != "" { + manifestDir := moduleDir + if ref.Path != "" { + manifestDir = filepath.Join(moduleDir, filepath.FromSlash(ref.Path)) + } + m, err := loadManifest(manifestDir) + if err != nil { + return models.GraderConfig{}, err + } + exp, ok := m.Exports.Graders[ref.Export] + if !ok { + return models.GraderConfig{}, fmt.Errorf("manifest does not export grader %q", ref.Export) + } + graderPath, err := safeJoinModulePath(manifestDir, exp.Path) + if err != nil { + return models.GraderConfig{}, err + } + return loadGraderFile(graderPath, ref.Export) + } + if ref.Path != "" { + return loadGraderFile(filepath.Join(moduleDir, filepath.FromSlash(ref.Path)), filepath.Base(ref.Path)) + } + m, err := loadManifest(moduleDir) + if err != nil { + return models.GraderConfig{}, err + } + if len(m.Exports.Graders) != 1 { + return models.GraderConfig{}, fmt.Errorf("ref %q must select one grader with #export or /path", ref.Raw) + } + for name, exp := range m.Exports.Graders { + graderPath, err := safeJoinModulePath(moduleDir, exp.Path) + if err != nil { + return models.GraderConfig{}, err + } + return loadGraderFile(graderPath, name) + } + return models.GraderConfig{}, fmt.Errorf("manifest has no grader exports") +} + +func safeJoinModulePath(base string, slashPath string) (string, error) { + clean := path.Clean(slashPath) + if clean == "." || clean == "" || strings.HasPrefix(clean, "../") || clean == ".." || strings.HasPrefix(clean, "/") { + return "", fmt.Errorf("invalid module path %q", slashPath) + } + return filepath.Join(base, filepath.FromSlash(clean)), nil +} + +func loadManifest(dir string) (*manifest, error) { + data, err := os.ReadFile(filepath.Join(dir, "waza.registry.yaml")) + if err != nil { + return nil, fmt.Errorf("reading manifest: %w", err) + } + var m manifest + if err := yaml.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parsing manifest: %w", err) + } + if len(m.Exports.Graders) == 0 { + return nil, fmt.Errorf("manifest has no grader exports") + } + return &m, nil +} + +func loadGraderFile(path string, fallbackName string) (models.GraderConfig, error) { + resolved, err := resolveYAMLPath(path) + if err != nil { + return models.GraderConfig{}, err + } + data, err := os.ReadFile(resolved) + if err != nil { + return models.GraderConfig{}, err + } + var grader models.GraderConfig + if err := yaml.Unmarshal(data, &grader); err != nil { + return models.GraderConfig{}, err + } + if grader.Kind == "" { + return models.GraderConfig{}, fmt.Errorf("remote grader %s must declare type", resolved) + } + if grader.Kind == models.GraderKindProgram { + return models.GraderConfig{}, fmt.Errorf("remote program graders are not supported without explicit trust") + } + if grader.Identifier == "" { + grader.Identifier = strings.TrimSuffix(fallbackName, filepath.Ext(fallbackName)) + } + return grader, nil +} + +func resolveYAMLPath(path string) (string, error) { + candidates := []string{path} + if filepath.Ext(path) == "" { + candidates = append(candidates, path+".yaml", path+".yml") + } + for _, candidate := range candidates { + info, err := os.Stat(candidate) + if err == nil && !info.IsDir() { + return candidate, nil + } + if err != nil && !os.IsNotExist(err) { + return "", err + } + } + return "", fmt.Errorf("grader file not found: %s", path) +} + +func MergeGraderConfig(preset models.GraderConfig, override models.GraderConfig) (models.GraderConfig, error) { + if override.Kind != "" && override.Kind != preset.Kind { + return models.GraderConfig{}, fmt.Errorf("local type %q does not match remote type %q", override.Kind, preset.Kind) + } + mergedConfig, err := configMap(preset.Parameters) + if err != nil { + return models.GraderConfig{}, err + } + localConfig, err := configMap(override.Parameters) + if err != nil { + return models.GraderConfig{}, err + } + deepMerge(mergedConfig, localConfig) + + raw := map[string]any{ + "type": string(preset.Kind), + "name": preset.Identifier, + "config": mergedConfig, + } + if preset.ScriptPath != "" { + raw["script"] = preset.ScriptPath + } + if preset.Rubric != "" { + raw["rubric"] = preset.Rubric + } + if preset.ModelID != "" { + raw["model"] = preset.ModelID + } + if preset.Weight != 0 { + raw["weight"] = preset.Weight + } + if override.Identifier != "" { + raw["name"] = override.Identifier + } + if override.ScriptPath != "" { + raw["script"] = override.ScriptPath + } + if override.Rubric != "" { + raw["rubric"] = override.Rubric + } + if override.ModelID != "" { + raw["model"] = override.ModelID + } + if override.Weight != 0 { + raw["weight"] = override.Weight + } + + data, err := yaml.Marshal(raw) + if err != nil { + return models.GraderConfig{}, err + } + var merged models.GraderConfig + if err := yaml.Unmarshal(data, &merged); err != nil { + return models.GraderConfig{}, err + } + merged.Ref = override.Ref + return merged, nil +} + +func configMap(params models.GraderParameters) (map[string]any, error) { + if params == nil { + return map[string]any{}, nil + } + data, err := yaml.Marshal(params) + if err != nil { + return nil, err + } + var out map[string]any + if err := yaml.Unmarshal(data, &out); err != nil { + return nil, err + } + if out == nil { + out = map[string]any{} + } + return out, nil +} + +func deepMerge(dst, src map[string]any) { + for key, srcValue := range src { + srcMap, srcOK := srcValue.(map[string]any) + dstMap, dstOK := dst[key].(map[string]any) + if srcOK && dstOK { + deepMerge(dstMap, srcMap) + continue + } + dst[key] = srcValue + } +} + +func DigestDirectory(dir string) (string, error) { + var files []string + if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("cannot digest non-regular file %s", path) + } + files = append(files, path) + return nil + }); err != nil { + return "", err + } + sort.Strings(files) + h := sha256.New() + for _, file := range files { + rel, err := filepath.Rel(dir, file) + if err != nil { + return "", err + } + h.Write([]byte(filepath.ToSlash(rel))) + h.Write([]byte{0}) + data, err := os.ReadFile(file) + if err != nil { + return "", err + } + h.Write(data) + h.Write([]byte{0}) + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +func extractTar(r io.Reader, dest string) error { + tr := tar.NewReader(r) + for { + header, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + cleanName := filepath.Clean(header.Name) + if cleanName == "." { + continue + } + if filepath.IsAbs(cleanName) || cleanName == ".." || strings.HasPrefix(cleanName, ".."+string(filepath.Separator)) { + return fmt.Errorf("archive contains unsafe path %q", header.Name) + } + target := filepath.Join(dest, cleanName) + switch header.Typeflag { + case tar.TypeXGlobalHeader, tar.TypeXHeader: + continue + case tar.TypeDir: + if err := os.MkdirAll(target, 0o755); err != nil { + return err + } + case tar.TypeReg: + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode)&0o777) + if err != nil { + return err + } + _, copyErr := io.Copy(file, tr) + closeErr := file.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("archive contains unsupported entry %q", header.Name) + } + } +} + +func runGit(ctx context.Context, dir string, args ...string) error { + _, err := gitOutput(ctx, dir, args...) + return err +} + +func gitOutput(ctx context.Context, dir string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "git", args...) + if dir != "" { + cmd.Dir = dir + } + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("git %s failed: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(string(out))) + } + return out, nil +} diff --git a/internal/registry/resolver_test.go b/internal/registry/resolver_test.go new file mode 100644 index 00000000..15199096 --- /dev/null +++ b/internal/registry/resolver_test.go @@ -0,0 +1,167 @@ +package registry + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/waza/internal/models" +) + +func TestResolverResolveAndExpandLockedGrader(t *testing.T) { + repo := createModuleRepo(t) + cacheRoot := filepath.Join(t.TempDir(), "cache") + resolver, err := NewResolver( + WithCacheRoot(cacheRoot), + WithGitURLFunc(func(Ref) string { return repo }), + ) + if err != nil { + t.Fatalf("NewResolver() error = %v", err) + } + + evalDir := t.TempDir() + evalPath := filepath.Join(evalDir, "eval.yaml") + specYAML := `name: remote-eval +skill: test +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: mock +graders: + - ref: example.com/acme/graders#factuality@v1.0.0 + name: strict_fact + weight: 2 + config: + not_contains: ["forbidden"] +metrics: [] +tasks: [] +` + if err := os.WriteFile(evalPath, []byte(specYAML), 0o644); err != nil { + t.Fatalf("write eval: %v", err) + } + + lock, entries, err := resolver.ResolveEvalLock(context.Background(), evalPath) + if err != nil { + t.Fatalf("ResolveEvalLock() error = %v", err) + } + if len(entries) != 1 { + t.Fatalf("entries = %d, want 1", len(entries)) + } + lockPath := filepath.Join(evalDir, models.LockfileName) + if err := models.WriteLockfile(lockPath, lock); err != nil { + t.Fatalf("WriteLockfile() error = %v", err) + } + + spec, err := models.LoadEvalSpec(evalPath) + if err != nil { + t.Fatalf("LoadEvalSpec() error = %v", err) + } + if err := resolver.ExpandLockedGraders(context.Background(), spec, evalPath); err != nil { + t.Fatalf("ExpandLockedGraders() error = %v", err) + } + if spec.Graders[0].Kind != models.GraderKindText { + t.Fatalf("Kind = %q", spec.Graders[0].Kind) + } + if spec.Graders[0].Identifier != "strict_fact" { + t.Fatalf("Identifier = %q", spec.Graders[0].Identifier) + } + if spec.Graders[0].Weight != 2 { + t.Fatalf("Weight = %v", spec.Graders[0].Weight) + } + params, ok := spec.Graders[0].Parameters.(models.TextGraderParameters) + if !ok { + t.Fatalf("Parameters = %T", spec.Graders[0].Parameters) + } + if len(params.Contains) != 1 || params.Contains[0] != "supported" { + t.Fatalf("remote contains not preserved: %#v", params) + } + if len(params.NotContains) != 1 || params.NotContains[0] != "forbidden" { + t.Fatalf("local not_contains not merged: %#v", params) + } +} + +func TestExpandLockedGradersFailsWithoutLock(t *testing.T) { + resolver, err := NewResolver(WithCacheRoot(t.TempDir())) + if err != nil { + t.Fatalf("NewResolver() error = %v", err) + } + evalDir := t.TempDir() + evalPath := filepath.Join(evalDir, "eval.yaml") + specYAML := `name: remote-eval +skill: test +config: + trials_per_task: 1 + timeout_seconds: 60 + executor: mock +graders: + - ref: example.com/acme/graders#factuality@v1.0.0 +metrics: [] +tasks: [] +` + if err := os.WriteFile(evalPath, []byte(specYAML), 0o644); err != nil { + t.Fatalf("write eval: %v", err) + } + spec, err := models.LoadEvalSpec(evalPath) + if err != nil { + t.Fatalf("LoadEvalSpec() error = %v", err) + } + err = resolver.ExpandLockedGraders(context.Background(), spec, evalPath) + if err == nil || !strings.Contains(err.Error(), "waza.lock is missing") { + t.Fatalf("expected missing lock error, got %v", err) + } +} + +func TestSafeJoinModulePathRejectsTraversal(t *testing.T) { + if _, err := safeJoinModulePath(t.TempDir(), "../outside.yaml"); err == nil { + t.Fatalf("expected traversal path to be rejected") + } +} + +func createModuleRepo(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + dir := t.TempDir() + run(t, dir, "git", "init", "--quiet") + run(t, dir, "git", "config", "user.email", "waza@example.com") + run(t, dir, "git", "config", "user.name", "Waza Test") + if err := os.MkdirAll(filepath.Join(dir, "graders"), 0o755); err != nil { + t.Fatalf("mkdir graders: %v", err) + } + manifest := `schema_version: 1 +module: example.com/acme/graders +exports: + graders: + factuality: + path: graders/factuality.yaml +` + grader := `type: text +name: factuality +config: + contains: ["supported"] +` + if err := os.WriteFile(filepath.Join(dir, "waza.registry.yaml"), []byte(manifest), 0o644); err != nil { + t.Fatalf("write manifest: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "graders", "factuality.yaml"), []byte(grader), 0o644); err != nil { + t.Fatalf("write grader: %v", err) + } + run(t, dir, "git", "add", ".") + run(t, dir, "git", "commit", "--quiet", "-m", "initial") + run(t, dir, "git", "tag", "v1.0.0") + return dir +} + +func run(t *testing.T, dir string, name string, args ...string) { + t.Helper() + cmd := exec.Command(name, args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s %s failed: %v: %s", name, strings.Join(args, " "), err, out) + } +} diff --git a/site/src/content/docs/guides/eval-yaml.mdx b/site/src/content/docs/guides/eval-yaml.mdx index 5005ead0..6534c3e3 100644 --- a/site/src/content/docs/guides/eval-yaml.mdx +++ b/site/src/content/docs/guides/eval-yaml.mdx @@ -245,6 +245,12 @@ Graders validate task outputs. Define once, reuse across tasks: ```yaml graders: + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + name: factuality_strict + weight: 2.0 + config: + threshold: 0.9 + - type: text name: checks_logic weight: 2.0 @@ -269,6 +275,15 @@ graders: Each grader accepts an optional `weight` (default `1.0`) that controls its influence on the composite score. See **[Validators & Graders](../graders/#weighted-scoring)** for details. +Remote grader presets can be referenced with `ref` instead of `type`. Refs use `//[/path][#export]@` and resolve through `waza get`, which writes `waza.lock` with the pinned commit SHA and content digest. Local `name`, `weight`, top-level grader fields, and `config` values override the remote preset; nested `config` maps are deep-merged, while lists are replaced. + +```bash +waza get eval.yaml +waza run eval.yaml +``` + +`waza run` requires the lockfile and cached module contents to be present for remote refs. It fails closed on missing locks, missing cache entries, or digest mismatches. + All graders return: - `score`: 0.0 to 1.0 diff --git a/site/src/content/docs/guides/graders.mdx b/site/src/content/docs/guides/graders.mdx index dbe074f8..e5e9615f 100644 --- a/site/src/content/docs/guides/graders.mdx +++ b/site/src/content/docs/guides/graders.mdx @@ -24,6 +24,18 @@ Graders are the scoring engine behind every waza evaluation. After an agent exec You can attach graders **globally** (applied to every task) or **per-task** in your eval YAML. Each grader also accepts an optional **`weight`** field that controls its influence on the composite score (see [Weighted Scoring](#weighted-scoring) below). +Global graders can also reference remote config-only presets: + +```yaml +graders: + - ref: github.com/waza-evals/fact#factuality@v1.0.0 + name: factuality_strict + config: + threshold: 0.9 +``` + +Run `waza get eval.yaml` after adding a `ref`; it writes `waza.lock` with the commit SHA and content digest. `waza run` expands locked refs from the local module cache and fails if the lock or digest is missing. + --- ## At a glance diff --git a/site/src/content/docs/reference/cli.mdx b/site/src/content/docs/reference/cli.mdx index eea43369..f30318b2 100644 --- a/site/src/content/docs/reference/cli.mdx +++ b/site/src/content/docs/reference/cli.mdx @@ -205,6 +205,23 @@ waza init my-project waza init my-project --no-skill ``` +## waza get + +Resolve remote grader refs and update `waza.lock`. + +```bash +waza get [eval.yaml | ref] +``` + +When given an eval file, `waza get` resolves each `graders[].ref`, downloads the module into `~/.waza/cache/{host}/{org}/{repo}/{sha}/`, and writes a lockfile next to the eval. The lock pins the original ref to a commit SHA, source URL, and `sha256:` content digest. + +```bash +waza get eval.yaml +waza get github.com/waza-evals/fact#factuality@v1.0.0 +``` + +`waza run` is strict: evals with remote grader refs require a valid `waza.lock` and local cache entry. If the lock is missing, cache is unavailable, or the digest mismatches, the run fails instead of resolving mutable refs implicitly. + ## waza new skill Create a new skill. In interactive mode, the wizard collects spec-aligned metadata: name, description, trigger phrases, and anti-trigger phrases.