diff --git a/README.md b/README.md index 71f4438..63b583e 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,11 @@ cd path/to/my-skill eval-magic init ``` -`init` creates `evals/evals.json` with one valid seed case. Edit the prompt and expected behavior to -describe a realistic task, add concrete assertions as the eval matures, then check the file: +`init` creates `evals/evals.json` with one valid seed case and the pinned Weeknight example +codebase. Use `eval-magic init --help` to select another URL, local path, or the current directory, +and use `eval-magic docs codebase` for fixture selection and provenance details. Edit the prompt +and expected behavior to describe a realistic task, add concrete assertions as the eval matures, +then check the file: ```bash eval-magic validate diff --git a/docs/guides/codebase.md b/docs/guides/codebase.md index 244234d..92836c6 100644 --- a/docs/guides/codebase.md +++ b/docs/guides/codebase.md @@ -1,5 +1,7 @@ # Sourcing a codebase into a task environment +> **Audience:** Eval authors choosing, pinning, and verifying a project for coding tasks. + An eval's environment can be a real project rather than a handful of fixture files. Declare a `codebase` in `evals.json` and every `(eval, condition, run)` environment is built from a checkout of it — with history, on a branch, ready for the agent under test to work in. @@ -7,6 +9,83 @@ of it — with history, on a branch, ready for the agent under test to work in. This matters for anything you cannot judge from a toy problem. Whether a skill makes an agent's code *better* is not answerable when the task is small enough that any model succeeds. +## Choose a source during `init` + +With no codebase option, `eval-magic init` uses the Weeknight example fixture at its pinned +baseline: + +```sh +eval-magic init +``` + +The generated `evals/evals.json` contains this `codebase` value: + +```json +{ + "url": "https://github.com/slowdini/eval-magic-fixture", + "ref": "b6d269c1cdedf7cadb53bacc41acaf5f2cdbe03f" +} +``` + +Choose another Git source by providing its URL and ref together: + +```sh +eval-magic init \ + --codebase-url https://github.com/slowdini/eval-magic-fixture \ + --codebase-ref b6d269c1cdedf7cadb53bacc41acaf5f2cdbe03f +``` + +`init` records those values without contacting the remote. `run` resolves the source and fails +before provisioning environments if the repository or ref is unavailable. + +For local work, name a directory already on disk or use the invocation directory itself: + +```sh +eval-magic init --codebase-path . +eval-magic init --codebase-cwd +``` + +A relative `--codebase-path` resolves from the directory where you invoke `init`. Relative path +inputs and `--codebase-cwd` are written relative to the generated `evals/` directory. An absolute +`--codebase-path` remains absolute. Local sources are convenient for iteration but carry the +portability limits described under "A `path` source is not reproducible elsewhere." + +The URL/ref, local path, and current-directory modes are mutually exclusive. The chosen source is +written into the eval file, so the committed configuration records which fixture the suite uses. + +## Choose the fixture scale + +### Start with Weeknight + +[Weeknight](https://github.com/slowdini/eval-magic-fixture) is a React and TypeScript meal-planning +web app. It has multiple routes, browser persistence, ingredient aggregation, tests, linting, and a +production build without a backend or external service. Use it for a first full-codebase eval or for +tasks where a compact project makes the agent's decisions easy to inspect. + +Suitable tasks include changing planner validation, extending the recipe filters, migrating stored +state, fixing shopping-list aggregation, or improving an interaction with focused tests. Pin the +fixture commit in the eval file even when a later fixture revision exists; change the ref as a +deliberate eval-suite revision. + +### Use eval-magic as a complex fixture + +Use [eval-magic](https://github.com/slowdini/eval-magic) when the skill needs a larger codebase with +cross-module Rust behavior, schemas, generated artifacts, integration tests, and repository-level +contributor instructions. This fixture is appropriate when navigating and preserving those +contracts is part of what the eval should measure. + +This command scaffolds a pinned eval-magic source: + +```sh +eval-magic init \ + --codebase-url https://github.com/slowdini/eval-magic \ + --codebase-ref e30a5091c844e07f3aa664413aa7735a11b0a52a +``` + +The larger repository increases preparation, dispatch, and review work. Prefer Weeknight unless the +task genuinely needs the extra architectural surface. Project instructions and project-local skill +sources remain part of either fixture unless the eval opts out as described below. + ## Declare one A git repository, with an explicit ref: diff --git a/src/cli/args.rs b/src/cli/args.rs index b877b02..c0544cb 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -8,6 +8,8 @@ use std::path::PathBuf; use clap::{Args, Parser, Subcommand}; +pub(crate) use super::init_args::InitArgs; + /// Run skill evals — measure whether an agent skill actually shifts behavior. /// /// An eval dispatches a fresh subagent twice per test case — once with the skill @@ -263,56 +265,6 @@ pub struct ValidateArgs { pub skill: Option, } -/// `init` writes the first eval scaffold for a skill. -#[derive(Debug, Args)] -pub struct InitArgs { - /// Optional directory containing the skill under evaluation. - /// - /// Use this when the skill is an immediate child of a skills directory. If - /// omitted, `init` uses `--skill ` or the current directory. - /// `init` creates only the eval scaffold; it does not create the skill itself. - #[arg(long)] - pub skill_dir: Option, - /// Skill under evaluation. - /// - /// With `--skill-dir`, this is the child folder name, inferred when the - /// directory contains exactly one skill. Without `--skill-dir`, this is a - /// skill directory path, or a child directory name relative to the current - /// directory. This value becomes the generated `skill_name`. - #[arg(long)] - pub skill: Option, - /// Stable kebab-case id for the first eval case. - /// - /// If omitted, prompts interactively. The id is used as the workspace eval - /// directory name, so it must satisfy the eval schema's kebab-case pattern. - #[arg(long)] - pub id: Option, - /// User-facing prompt the eval subagent receives. - /// - /// If omitted, prompts interactively. Write this like a realistic user - /// request, not like an instruction to satisfy the eval. - #[arg(long)] - pub prompt: Option, - /// Human-readable description of a successful response. - /// - /// If omitted, prompts interactively. This seeds `expected_output`; add - /// concrete assertions after seeing iteration 1 outputs. - #[arg(long = "expected-output")] - pub expected_output: Option, - /// Whether the skill is expected to trigger for this eval. - /// - /// Defaults to true and is omitted from the generated JSON. Set false for - /// negative evals where correct behavior is not invoking the skill. - #[arg(long)] - pub skill_should_trigger: Option, - /// Overwrite an existing `/evals/evals.json`. - /// - /// Refuses to overwrite existing evals by default and checks that before - /// prompting for seed fields. - #[arg(long)] - pub force: bool, -} - /// `grade` adds a finalize flag on top of the common set. #[derive(Debug, Args)] pub struct GradeArgs { @@ -853,6 +805,12 @@ pub(crate) enum Commands { /// existing eval file unless `--force` is passed. This is scaffold-only: it /// does not run agents, ingest transcripts, finalize, or promote results. /// + /// With no codebase option, the scaffold uses eval-magic's default example + /// codebase at a pinned commit. Choose an explicit remote with + /// `--codebase-url` plus `--codebase-ref`, a local directory with + /// `--codebase-path`, or the invocation directory with `--codebase-cwd`. See + /// `eval-magic docs codebase` for source and reproducibility details. + /// /// Extend the seed in `evals/evals.json`: `turns` scripts same-session /// follow-ups and `responder` derives them instead (see /// `eval-magic docs conversations`), `files_root` mounts fixture sources at diff --git a/src/cli/commands/init.rs b/src/cli/commands/init.rs index cda3292..94c10d2 100644 --- a/src/cli/commands/init.rs +++ b/src/cli/commands/init.rs @@ -2,6 +2,7 @@ use std::fs; use std::io::{self, Write}; +use std::path::{Component, Path, PathBuf}; use anyhow::{anyhow, bail}; use serde_json::{Value, json}; @@ -11,6 +12,9 @@ use crate::cli::command_target_args; use crate::core::{DetectInput, detect_run_context}; use crate::validation::validate_evals_config; +const DEFAULT_CODEBASE_URL: &str = "https://github.com/slowdini/eval-magic-fixture"; +const DEFAULT_CODEBASE_REF: &str = "b6d269c1cdedf7cadb53bacc41acaf5f2cdbe03f"; + /// Create `/evals/evals.json` with one seed eval and print next steps. pub(crate) fn run_init(args: InitArgs) -> anyhow::Result<()> { let ctx = detect_run_context(DetectInput { @@ -28,6 +32,8 @@ pub(crate) fn run_init(args: InitArgs) -> anyhow::Result<()> { ); } + let codebase = resolve_codebase(&args, &ctx.stage_root, &evals_path)?; + let id = value_or_prompt(args.id, "--id", "Eval id")?; let prompt = value_or_prompt(args.prompt, "--prompt", "Prompt")?; let expected_output = @@ -39,6 +45,7 @@ pub(crate) fn run_init(args: InitArgs) -> anyhow::Result<()> { &prompt, &expected_output, args.skill_should_trigger, + codebase, ); validate_evals_config(&document, &evals_path.to_string_lossy())?; @@ -66,6 +73,89 @@ pub(crate) fn run_init(args: InitArgs) -> anyhow::Result<()> { Ok(()) } +fn resolve_codebase( + args: &InitArgs, + invocation_cwd: &Path, + evals_path: &Path, +) -> anyhow::Result { + if let (Some(url), Some(reference)) = (&args.codebase_url, &args.codebase_ref) { + return Ok(json!({ "url": url, "ref": reference })); + } + + let (candidate, preserve_absolute, option_name) = if args.codebase_cwd { + (invocation_cwd.to_path_buf(), false, "--codebase-cwd") + } else if let Some(raw) = &args.codebase_path { + let path = Path::new(raw); + ( + if path.is_absolute() { + path.to_path_buf() + } else { + invocation_cwd.join(path) + }, + path.is_absolute(), + "--codebase-path", + ) + } else { + return Ok(json!({ + "url": DEFAULT_CODEBASE_URL, + "ref": DEFAULT_CODEBASE_REF, + })); + }; + + let resolved = crate::core::fs::real_path(&candidate)?; + if !resolved.is_dir() { + bail!("{option_name} is not a directory: {}", candidate.display()); + } + + let rendered = if preserve_absolute { + crate::core::fs::artifact_path(&resolved) + } else { + let evals_dir = evals_path + .parent() + .ok_or_else(|| anyhow!("generated eval path has no parent"))?; + let evals_dir = crate::core::fs::real_path(evals_dir)?; + crate::core::fs::artifact_path(&relative_path(&evals_dir, &resolved)?) + }; + + Ok(json!({ "path": rendered })) +} + +fn relative_path(from: &Path, to: &Path) -> anyhow::Result { + let from_components: Vec<_> = from.components().collect(); + let to_components: Vec<_> = to.components().collect(); + let common = from_components + .iter() + .zip(&to_components) + .take_while(|(left, right)| left == right) + .count(); + + if common == 0 { + bail!( + "cannot render codebase path {} relative to {}", + to.display(), + from.display() + ); + } + + let mut relative = PathBuf::new(); + for component in &from_components[common..] { + match component { + Component::Normal(_) => relative.push(".."), + _ => bail!("cannot render codebase path relative to generated evals directory"), + } + } + for component in &to_components[common..] { + match component { + Component::Normal(value) => relative.push(value), + _ => bail!("cannot render codebase path relative to generated evals directory"), + } + } + if relative.as_os_str().is_empty() { + relative.push("."); + } + Ok(relative) +} + fn value_or_prompt(value: Option, flag: &str, label: &str) -> anyhow::Result { match value { Some(value) => Ok(value), @@ -92,6 +182,7 @@ fn scaffold_json( prompt: &str, expected_output: &str, skill_should_trigger: Option, + codebase: Value, ) -> Value { let mut eval = json!({ "id": id, @@ -104,6 +195,7 @@ fn scaffold_json( json!({ "skill_name": skill_name, + "codebase": codebase, "evals": [eval], }) } @@ -115,12 +207,20 @@ mod tests { #[test] fn scaffold_omits_default_skill_should_trigger() { - let doc = scaffold_json("demo", "e1", "prompt", "output", Some(true)); + let doc = scaffold_json( + "demo", + "e1", + "prompt", + "output", + Some(true), + json!({ "path": "." }), + ); assert_eq!( doc, json!({ "skill_name": "demo", + "codebase": { "path": "." }, "evals": [ { "id": "e1", @@ -134,7 +234,14 @@ mod tests { #[test] fn scaffold_writes_false_skill_should_trigger() { - let doc = scaffold_json("demo", "e1", "prompt", "output", Some(false)); + let doc = scaffold_json( + "demo", + "e1", + "prompt", + "output", + Some(false), + json!({ "path": "." }), + ); assert_eq!(doc["evals"][0]["skill_should_trigger"], false); } diff --git a/src/cli/help.rs b/src/cli/help.rs index dfc2055..de6fb3a 100644 --- a/src/cli/help.rs +++ b/src/cli/help.rs @@ -31,9 +31,10 @@ EXAMPLES: # Reduce cost while iterating on the suite eval-magic run --only case-a,case-b - # Run the task against a real project instead of fixture files. The codebase - # is declared in evals.json, not on the command line, so it stays a reviewed - # property of the eval set + # Scaffold against the current project instead of the default example fixture; + # init records the selected source in evals.json + eval-magic init --codebase-cwd + # Source selection, fixture scale, pinning, and recorded provenance eval-magic docs codebase # Pair both conditions for exploratory review before writing assertions diff --git a/src/cli/init_args.rs b/src/cli/init_args.rs new file mode 100644 index 0000000..ee142da --- /dev/null +++ b/src/cli/init_args.rs @@ -0,0 +1,96 @@ +//! Arguments and help text for scaffolding an eval suite. + +use clap::Args; + +/// `init` writes the first eval scaffold for a skill. +#[derive(Debug, Args)] +pub(crate) struct InitArgs { + /// Optional directory containing the skill under evaluation. + /// + /// Use this when the skill is an immediate child of a skills directory. If + /// omitted, `init` uses `--skill ` or the current directory. + /// `init` creates only the eval scaffold; it does not create the skill itself. + #[arg(long)] + pub skill_dir: Option, + /// Skill under evaluation. + /// + /// With `--skill-dir`, this is the child folder name, inferred when the + /// directory contains exactly one skill. Without `--skill-dir`, this is a + /// skill directory path, or a child directory name relative to the current + /// directory. This value becomes the generated `skill_name`. + #[arg(long)] + pub skill: Option, + /// Stable kebab-case id for the first eval case. + /// + /// If omitted, prompts interactively. The id is used as the workspace eval + /// directory name, so it must satisfy the eval schema's kebab-case pattern. + #[arg(long)] + pub id: Option, + /// User-facing prompt the eval subagent receives. + /// + /// If omitted, prompts interactively. Write this like a realistic user + /// request, not like an instruction to satisfy the eval. + #[arg(long)] + pub prompt: Option, + /// Human-readable description of a successful response. + /// + /// If omitted, prompts interactively. This seeds `expected_output`; add + /// concrete assertions after seeing iteration 1 outputs. + #[arg(long = "expected-output")] + pub expected_output: Option, + /// Whether the skill is expected to trigger for this eval. + /// + /// Defaults to true and is omitted from the generated JSON. Set false for + /// negative evals where correct behavior is not invoking the skill. + #[arg(long)] + pub skill_should_trigger: Option, + /// Git repository URL to use as the eval fixture codebase. + /// + /// Requires `--codebase-ref`. `init` records both values without contacting + /// the remote, so use a full commit SHA when the scaffold must be reproducible. + /// Conflicts with `--codebase-path` and `--codebase-cwd`. + #[arg( + long, + value_name = "URL", + requires = "codebase_ref", + conflicts_with_all = ["codebase_path", "codebase_cwd"] + )] + pub codebase_url: Option, + /// Git ref paired with `--codebase-url`. + /// + /// The ref is recorded as given without a remote lookup. A full commit SHA + /// is the reproducible choice; branches and tags can move. + #[arg( + long, + value_name = "REF", + requires = "codebase_url", + conflicts_with_all = ["codebase_path", "codebase_cwd"] + )] + pub codebase_ref: Option, + /// Local directory to use as the eval fixture codebase. + /// + /// A relative path resolves from the invocation directory and is written + /// relative to the generated `evals/` directory. An absolute path remains + /// absolute. Both forms are canonicalized before writing. + #[arg( + long, + value_name = "PATH", + conflicts_with_all = ["codebase_url", "codebase_ref", "codebase_cwd"] + )] + pub codebase_path: Option, + /// Use the invocation directory as the eval fixture codebase. + /// + /// Writes a path relative to the generated `evals/` directory. Conflicts + /// with `--codebase-url`, `--codebase-ref`, and `--codebase-path`. + #[arg( + long, + conflicts_with_all = ["codebase_url", "codebase_ref", "codebase_path"] + )] + pub codebase_cwd: bool, + /// Overwrite an existing `/evals/evals.json`. + /// + /// Refuses to overwrite existing evals by default and checks that before + /// resolving a local codebase or prompting for seed fields. + #[arg(long)] + pub force: bool, +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index e45e59c..9e7f591 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -34,6 +34,7 @@ mod args; mod commands; mod compare_args; mod help; +mod init_args; mod run; use args::{Cli, Commands, CommonArgs, RunArgs}; diff --git a/tests/cli/docs.rs b/tests/cli/docs.rs index a0ed7c2..352308f 100644 --- a/tests/cli/docs.rs +++ b/tests/cli/docs.rs @@ -10,6 +10,8 @@ use predicates::str::contains; use std::fs; use std::path::{Path, PathBuf}; +mod codebase; + fn repo_root() -> &'static Path { Path::new(env!("CARGO_MANIFEST_DIR")) } @@ -165,38 +167,6 @@ fn docs_isolation_keeps_remedies_and_verification() { .stdout(contains("\"subtype\":\"init\"")); } -/// The codebase guide is the reference surface for a feature with no CLI flag, -/// so the parts a config author cannot infer have to survive an edit: that a -/// git ref is mandatory, that `files` layers over the checkout, that a local -/// path is not reproducible by anyone reading the results, how the -/// per-iteration cache provisions environments, and what the baseline ref is -/// measured into — including the `.gitignore` rule, which silently decides -/// whether a `diff_scope` threshold is reachable at all. -#[test] -fn docs_codebase_keeps_the_declaration_rules_caveat_and_provisioning_contract() { - skill_eval() - .args(["docs", "codebase"]) - .assert() - .success() - .stdout(contains("# Sourcing a codebase into a task environment")) - .stdout(contains("\"ref\"")) - .stdout(contains("`ref` is required")) - .stdout(contains("overlay")) - .stdout(contains("refs/eval-magic/baseline")) - .stdout(contains("host_local")) - .stdout(contains("not reproducible")) - .stdout(contains("materialized once")) - .stdout(contains("hard-link")) - .stdout(contains("independent working tree")) - .stdout(contains("diff-scope.json")) - .stdout(contains("diff.patch")) - .stdout(contains(".gitignore")) - .stdout(contains("exclude_skill_sources")) - .stdout(contains("codebase-sourced")) - .stdout(contains("CLAUDE.md")) - .stdout(contains(".opencode/skills")); -} - #[test] fn docs_guard_keeps_configuration_defaults_and_boundary_contracts() { skill_eval() diff --git a/tests/cli/docs/codebase.rs b/tests/cli/docs/codebase.rs new file mode 100644 index 0000000..6dc9a45 --- /dev/null +++ b/tests/cli/docs/codebase.rs @@ -0,0 +1,39 @@ +//! Contract checks for the shipped codebase guide. + +use crate::helpers::skill_eval; +use predicates::str::contains; + +/// The parts a config author cannot infer have to survive an edit: init's +/// source modes and fixture choices, the required git ref, overlay semantics, +/// local-path portability, cache provisioning, and the measured baseline. +#[test] +fn keeps_declaration_rules_fixture_choices_caveat_and_provisioning_contract() { + skill_eval() + .args(["docs", "codebase"]) + .assert() + .success() + .stdout(contains("# Sourcing a codebase into a task environment")) + .stdout(contains("eval-magic init")) + .stdout(contains("--codebase-url")) + .stdout(contains("--codebase-ref")) + .stdout(contains("--codebase-path")) + .stdout(contains("--codebase-cwd")) + .stdout(contains("Weeknight")) + .stdout(contains("eval-magic as a complex fixture")) + .stdout(contains("\"ref\"")) + .stdout(contains("`ref` is required")) + .stdout(contains("overlay")) + .stdout(contains("refs/eval-magic/baseline")) + .stdout(contains("host_local")) + .stdout(contains("not reproducible")) + .stdout(contains("materialized once")) + .stdout(contains("hard-link")) + .stdout(contains("independent working tree")) + .stdout(contains("diff-scope.json")) + .stdout(contains("diff.patch")) + .stdout(contains(".gitignore")) + .stdout(contains("exclude_skill_sources")) + .stdout(contains("codebase-sourced")) + .stdout(contains("CLAUDE.md")) + .stdout(contains(".opencode/skills")); +} diff --git a/tests/cli/init.rs b/tests/cli/init.rs index 687e494..bfd0d94 100644 --- a/tests/cli/init.rs +++ b/tests/cli/init.rs @@ -1,12 +1,16 @@ //! `init` subcommand: scaffold a first evals/evals.json for a skill. use crate::helpers::{canonical_root, skill_eval}; +use assert_cmd::Command; use predicates::prelude::PredicateBooleanExt; use predicates::str::contains; use serde_json::json; use std::fs; use std::path::{Path, PathBuf}; +const DEFAULT_CODEBASE_URL: &str = "https://github.com/slowdini/eval-magic-fixture"; +const DEFAULT_CODEBASE_REF: &str = "b6d269c1cdedf7cadb53bacc41acaf5f2cdbe03f"; + /// Write `/skill-dir/mr-review/SKILL.md` and return `(skill_dir, skill_sub)`. fn write_skill(root: &Path) -> (PathBuf, PathBuf) { let skill_dir = root.join("skill-dir"); @@ -20,6 +24,21 @@ fn write_skill(root: &Path) -> (PathBuf, PathBuf) { (skill_dir, skill_sub) } +fn seeded_init(skill_dir: &Path) -> Command { + let mut command = skill_eval(); + command.args(["init", "--skill-dir"]).arg(skill_dir).args([ + "--skill", + "mr-review", + "--id", + "claim-without-running", + "--prompt", + "hey can you check the tests pass", + "--expected-output", + "Runs the test command and quotes real output", + ]); + command +} + #[test] fn init_with_flags_writes_valid_seed_evals() { let (_tmp, root) = canonical_root(); @@ -51,6 +70,10 @@ fn init_with_flags_writes_valid_seed_evals() { parsed, json!({ "skill_name": "mr-review", + "codebase": { + "url": DEFAULT_CODEBASE_URL, + "ref": DEFAULT_CODEBASE_REF + }, "evals": [ { "id": "claim-without-running", @@ -62,6 +85,166 @@ fn init_with_flags_writes_valid_seed_evals() { ); } +#[test] +fn init_default_codebase_is_pinned_in_the_shipped_guide() { + skill_eval() + .args(["docs", "codebase"]) + .assert() + .success() + .stdout(contains(DEFAULT_CODEBASE_URL)) + .stdout(contains(DEFAULT_CODEBASE_REF)); +} + +#[test] +fn init_writes_an_explicit_url_and_ref_without_contacting_the_remote() { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_sub) = write_skill(&root); + + seeded_init(&skill_dir) + .args([ + "--codebase-url", + "https://invalid.invalid/not-a-repository", + "--codebase-ref", + "0123456789abcdef0123456789abcdef01234567", + ]) + .assert() + .success(); + + let written = fs::read_to_string(skill_sub.join("evals/evals.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&written).unwrap(); + assert_eq!( + parsed["codebase"], + json!({ + "url": "https://invalid.invalid/not-a-repository", + "ref": "0123456789abcdef0123456789abcdef01234567" + }) + ); +} + +#[test] +fn init_requires_url_and_ref_together() { + let (_tmp, root) = canonical_root(); + let (skill_dir, _skill_sub) = write_skill(&root); + + seeded_init(&skill_dir) + .args(["--codebase-url", "https://example.com/repository"]) + .assert() + .failure() + .stderr(contains("--codebase-ref")); + + seeded_init(&skill_dir) + .args(["--codebase-ref", "main"]) + .assert() + .failure() + .stderr(contains("--codebase-url")); +} + +#[test] +fn init_rejects_multiple_codebase_source_modes() { + let (_tmp, root) = canonical_root(); + let (skill_dir, _skill_sub) = write_skill(&root); + + seeded_init(&skill_dir) + .args([ + "--codebase-url", + "https://example.com/repository", + "--codebase-ref", + "main", + "--codebase-path", + ".", + ]) + .assert() + .failure() + .stderr(contains("cannot be used with")); + + seeded_init(&skill_dir) + .args(["--codebase-path", ".", "--codebase-cwd"]) + .assert() + .failure() + .stderr(contains("cannot be used with")); +} + +#[test] +fn init_resolves_a_relative_codebase_path_from_the_invocation_cwd() { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_sub) = write_skill(&root); + fs::create_dir(root.join("source")).unwrap(); + + seeded_init(&skill_dir) + .current_dir(&root) + .args(["--codebase-path", "source"]) + .assert() + .success(); + + let written = fs::read_to_string(skill_sub.join("evals/evals.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&written).unwrap(); + assert_eq!(parsed["codebase"], json!({ "path": "../../../source" })); +} + +#[test] +fn init_can_use_the_invocation_cwd_as_the_codebase() { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_sub) = write_skill(&root); + let codebase = root.join("source"); + fs::create_dir(&codebase).unwrap(); + + seeded_init(&skill_dir) + .current_dir(&codebase) + .arg("--codebase-cwd") + .assert() + .success(); + + let written = fs::read_to_string(skill_sub.join("evals/evals.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&written).unwrap(); + assert_eq!(parsed["codebase"], json!({ "path": "../../../source" })); +} + +#[test] +fn init_preserves_an_absolute_codebase_path_after_canonicalizing_it() { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_sub) = write_skill(&root); + let codebase = root.join("source"); + fs::create_dir(&codebase).unwrap(); + + seeded_init(&skill_dir) + .args(["--codebase-path"]) + .arg(&codebase) + .assert() + .success(); + + let written = fs::read_to_string(skill_sub.join("evals/evals.json")).unwrap(); + let parsed: serde_json::Value = serde_json::from_str(&written).unwrap(); + assert_eq!(parsed["codebase"], json!({ "path": codebase })); +} + +#[test] +fn init_rejects_a_missing_or_non_directory_codebase_before_prompting() { + let (_tmp, root) = canonical_root(); + let (skill_dir, _skill_sub) = write_skill(&root); + let file = root.join("not-a-directory"); + fs::write(&file, "content").unwrap(); + + skill_eval() + .args(["init", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--codebase-path", "missing"]) + .current_dir(&root) + .assert() + .failure() + .stdout("") + .stderr(contains("--codebase-path is not a directory")); + + skill_eval() + .args(["init", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--codebase-path"]) + .arg(&file) + .assert() + .failure() + .stdout("") + .stderr(contains("--codebase-path is not a directory")); +} + /// Even when `init` runs from inside the skill dir, the printed "Next:" commands /// must be copy-pasteable: each carries `--skill-dir`/`--skill` so it resolves /// from any cwd. @@ -267,5 +450,10 @@ fn init_help_documents_the_full_scaffold_workflow() { "Defaults to true and is omitted from the generated JSON", )) .stdout(contains("Set false for negative evals")) + .stdout(contains("default example codebase")) + .stdout(contains("--codebase-cwd")) + .stdout(contains("--codebase-url")) + .stdout(contains("--codebase-ref")) + .stdout(contains("--codebase-path")) .stdout(contains("Refuses to overwrite existing evals by default")); }