fix(run): hide framework-staged files from the codebase's own tooling (#296)

build.rs

@@ -23,37 +23,49 @@ fn main() {
     fs::write(out_dir.join("guide_topics.rs"), generated)
         .expect("failed to write generated guide topic table");
 
-    let profile_dir = manifest_dir.join("guard-profiles");
-    println!("cargo:rerun-if-changed={}", profile_dir.display());
-    let generated = render_guard_profiles(&profile_dir);
-    fs::write(out_dir.join("guard_profiles.rs"), generated)
-        .expect("failed to write generated guard profile table");
+    embed_profiles(
+        &manifest_dir.join("guard-profiles"),
+        &out_dir.join("guard_profiles.rs"),
+        "PACKAGED_GUARD_PROFILES",
+    );
+    embed_profiles(
+        &manifest_dir.join("ignore-profiles"),
+        &out_dir.join("ignore_profiles.rs"),
+        "PACKAGED_IGNORE_PROFILES",
+    );
 }
 
-fn render_guard_profiles(profile_dir: &Path) -> String {
+/// Render one directory of TOML profiles as a `&[(filename, body)]` table the
+/// crate `include!`s. Both packaged profile families — guard command policy and
+/// tool-ignore targets — ship this way, so a new profile is a new file and
+/// nothing else.
+fn embed_profiles(profile_dir: &Path, out_file: &Path, table_name: &str) {
+    println!("cargo:rerun-if-changed={}", profile_dir.display());
     let mut profiles: Vec<PathBuf> = fs::read_dir(profile_dir)
         .unwrap_or_else(|err| panic!("failed to read {}: {err}", profile_dir.display()))
-        .map(|entry| entry.expect("failed to read guard profile entry").path())
+        .map(|entry| entry.expect("failed to read profile entry").path())
         .filter(|path| path.extension().and_then(|value| value.to_str()) == Some("toml"))
         .collect();
     profiles.sort();
     assert!(
         !profiles.is_empty(),
-        "guard-profiles must contain a TOML profile"
+        "{} must contain a TOML profile",
+        profile_dir.display()
     );
 
-    let mut generated = String::from("const PACKAGED_GUARD_PROFILES: &[(&str, &str)] = &[\n");
+    let mut generated = format!("const {table_name}: &[(&str, &str)] = &[\n");
     for path in profiles {
         let name = path
             .file_name()
             .and_then(|value| value.to_str())
-            .unwrap_or_else(|| panic!("guard profile path is not UTF-8: {}", path.display()));
+            .unwrap_or_else(|| panic!("profile path is not UTF-8: {}", path.display()));
         let body = fs::read_to_string(&path)
             .unwrap_or_else(|err| panic!("failed to read {}: {err}", path.display()));
         generated.push_str(&format!("    ({name:?}, {body:?}),\n"));
     }
     generated.push_str("];\n");
-    generated
+    fs::write(out_file, generated)
+        .unwrap_or_else(|err| panic!("failed to write {}: {err}", out_file.display()));
 }
 
 fn discover_guides(guide_dir: &Path) -> Vec<Guide> {

docs/developer_overview.md

@@ -54,6 +54,10 @@ preconditions, handoffs, and recovery commands.
 - `schema/` contains the JSON schemas for user input and generated artifacts.
 - `harnesses/` contains built-in descriptors, descriptor scaffolding, and embedded harness assets.
 - `guard-profiles/` contains packaged command-policy defaults discovered and embedded by `build.rs`.
+- `ignore-profiles/` contains packaged tool-ignore targets, discovered and embedded the same way:
+  which ignore file a detected formatter or linter reads, so `run` can hide the framework's own
+  staged files from a sourced codebase's tooling. Both families detect through
+  `src/core/tree_profiles.rs`.
 - `profiles/` contains shared prompt profiles.
 - `tests/cli/` covers CLI and packaging contracts; `tests/run/` covers campaign behavior across
   the run boundary. Focused unit tests normally live beside the implementation.

docs/guides/byoh.md

@@ -128,6 +128,11 @@ path must be normalized, `/`-separated, and relative to the task repository. Its
 also appear in `config_dirs`, keeping discovery, sibling filtering, and task-repository baselining
 on one descriptor surface.
 
+`skills_dir` and the file your `[guard]` section stages (`hooks_file`, or `plugin_file` for a
+plugin engine) are also what `run` hides from a sourced codebase's own linters and formatters, by
+writing them into the project's ignore files. Declaring them correctly is all a descriptor has to
+do; see `eval-magic docs codebase`.
+
 ## Layer descriptors by field
 
 Descriptors load in this order:

docs/guides/codebase.md

@@ -172,6 +172,73 @@ cleanup. An explicit `--stage-name` remains stricter and refuses to clobber an o
 The effective `exclude_skill_sources` value is recorded with each codebase in `conditions.json`,
 every task in `dispatch.json`, every `run.json`, `benchmark.json`, and promoted `BASELINE.md`.
 
+## Framework files stay out of the project's own tooling
+
+Staged skills live *inside* the task repository, under the harness's skills dir. A project whose
+lint or format step globs the whole tree would otherwise report eval-magic's own artifacts as
+project failures — and only in the arm that stages a skill, because the control arm has no staged
+skills to find. That biases every `command_check` running the project's checks, and it hands the
+agent under test a red check listing files it did not create and must not touch.
+
+So `run` writes a delimited block into the project's own ignore files, naming the paths the runner
+placed:
+
+```
+# >>> eval-magic framework files >>>
+# Staged by `eval-magic run` so this project's own tooling does not report them.
+# See `eval-magic docs codebase`.
+/.eval-magic-outputs/
+/.claude/skills/
+/.claude/settings.local.json
+# <<< eval-magic framework files <<<
+```
+
+The paths come from the selected harness descriptor — its `skills_dir` and the file its write guard
+stages — plus the framework outputs directory, so a BYOH harness gets its own paths without
+configuring anything. The block is written into **every** environment: both arms, every repetition,
+revision mode, `--no-stage`, and `--dry-run`. An entry present in one arm only would trade one
+asymmetry for another.
+
+Which ignore files get the block is detected from the codebase's own tooling:
+
+| Detected | Ignore file | Created when the project has none |
+| --- | --- | --- |
+| Prettier | `.prettierignore` | yes |
+| ESLint | `.eslintignore` | no — ESLint 9's flat config no longer reads it |
+| Stylelint | `.stylelintignore` | yes |
+| markdownlint | `.markdownlintignore` | yes |
+| Docker | `.dockerignore` | yes |
+
+Detection reads config-file markers and `package.json` dependencies anywhere in the tree. The list
+is short because the ignore-file convention is: Python, Rust, and Go formatters have no ignore file
+and already skip dot-directories, so they never see the staged skills in the first place.
+
+Name the ignore files yourself when the project's are somewhere detection will not look, or when
+you want none at all:
+
+```json
+{
+  "codebase": {
+    "url": "https://github.com/slowdini/example-project",
+    "ref": "v1.4.0",
+    "ignore_files": ["tooling/.prettierignore"]
+  }
+}
+```
+
+A declared list replaces detection rather than extending it, and each path is created if missing.
+Paths are relative to the environment root and may not escape it. `"ignore_files": []` opts out
+entirely, leaving every ignore file exactly as the codebase wrote it.
+
+`.gitignore` is never a target. The baseline commit force-adds harness config dirs, so an entry
+there would hide nothing from Git — but it *would* hide the staged skills from every
+`.gitignore`-aware tool the agent uses, such as `rg`, which would damage the treatment arm instead
+of protecting it.
+
+Unlike `exclude_skill_sources`, no artifact records this: the written file is committed into each
+environment's baseline, so `git show refs/eval-magic/baseline:.prettierignore` inside any task
+environment is the evidence, per arm and per run. `run` also prints the files it wrote.
+
 ## What the environment contains
 
 Each dispatch gets its own private environment holding:

harnesses/template.toml

@@ -27,6 +27,9 @@ label = "{label}"
 ## additional_project_skill_dirs. They participate in sourced-codebase shadow detection and
 ## codebase.exclude_skill_sources but never receive staged skills. Each first path segment must
 ## also appear in config_dirs.
+## skills_dir and the [guard] staged file are also what `run` writes into a sourced codebase's own
+## ignore files (.prettierignore and friends), so the project's linters never report eval-magic's
+## staged skills — see `eval-magic docs codebase`.
 ## VERIFY: which directory does the harness actually scan for skills? Quote the doc or the
 ## observed behavior in the notes file.
 # skills_dir = ".{label}/skills"

ignore-profiles/tool-docker.toml

@@ -0,0 +1,8 @@
+# Not a linter: .dockerignore keeps the staged skills and the framework's own
+# outputs out of the build context a `docker build` command_check sends.
+id = "tool/docker"
+ignore_file = ".dockerignore"
+create_if_missing = true
+markers = ["Dockerfile", ".dockerignore"]
+marker_patterns = ["Dockerfile.*"]
+package_json_dependencies = []

ignore-profiles/tool-eslint.toml

@@ -0,0 +1,9 @@
+# ESLint 9's flat config dropped .eslintignore entirely, so creating one would
+# be inert noise in a modern project. A legacy project that already has the
+# file still gets the framework entries appended to it.
+id = "tool/eslint"
+ignore_file = ".eslintignore"
+create_if_missing = false
+markers = [".eslintignore"]
+marker_patterns = [".eslintrc*"]
+package_json_dependencies = ["eslint"]

ignore-profiles/tool-markdownlint.toml

@@ -0,0 +1,15 @@
+# markdownlint-cli reads .markdownlintignore; markdownlint-cli2 does not (it
+# takes globs in its own config), so its markers are deliberately absent — a
+# file that tool never reads would be noise, not protection.
+id = "tool/markdownlint"
+ignore_file = ".markdownlintignore"
+create_if_missing = true
+markers = [
+  ".markdownlintignore",
+  ".markdownlint.json",
+  ".markdownlint.jsonc",
+  ".markdownlint.yaml",
+  ".markdownlint.yml",
+]
+marker_patterns = [".markdownlintrc*"]
+package_json_dependencies = ["markdownlint-cli"]

ignore-profiles/tool-prettier.toml

@@ -0,0 +1,16 @@
+# Prettier reads .prettierignore (and .gitignore) but ignores nothing by
+# default, so `prettier --check .` walks straight into the staged skills dir.
+id = "tool/prettier"
+ignore_file = ".prettierignore"
+create_if_missing = true
+markers = [
+  ".prettierrc",
+  ".prettierrc.json",
+  ".prettierrc.json5",
+  ".prettierrc.yaml",
+  ".prettierrc.yml",
+  ".prettierrc.toml",
+  ".prettierignore",
+]
+marker_patterns = ["prettier.config.*"]
+package_json_dependencies = ["prettier"]

ignore-profiles/tool-stylelint.toml

@@ -0,0 +1,6 @@
+id = "tool/stylelint"
+ignore_file = ".stylelintignore"
+create_if_missing = true
+markers = [".stylelintignore"]
+marker_patterns = [".stylelintrc*", "stylelint.config.*"]
+package_json_dependencies = ["stylelint"]

schema/evals.schema.json

@@ -64,6 +64,11 @@
           "type": "boolean",
           "default": false,
           "description": "Move project-local skill roots discoverable by the selected harness out of every comparison environment before staging. Root instruction files and other harness configuration remain visible."
+        },
+        "ignore_files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "description": "Ignore files, relative to the task environment root, that run writes its framework-file block into, so the project's own linters and formatters never report eval-magic's staged skills, guard files, or outputs. Omit it to detect the project's tooling automatically (Prettier, ESLint, Stylelint, markdownlint, Docker); an empty array opts out entirely. Paths may not be absolute or escape the environment."
         }
       }
     },
@@ -86,6 +91,11 @@
           "type": "boolean",
           "default": false,
           "description": "Move project-local skill roots discoverable by the selected harness out of every comparison environment before staging. Root instruction files and other harness configuration remain visible."
+        },
+        "ignore_files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "description": "Ignore files, relative to the task environment root, that run writes its framework-file block into, so the project's own linters and formatters never report eval-magic's staged skills, guard files, or outputs. Omit it to detect the project's tooling automatically (Prettier, ESLint, Stylelint, markdownlint, Docker); an empty array opts out entirely. Paths may not be absolute or escape the environment."
         }
       }
     },

src/adapters/descriptor_adapter.rs

@@ -126,6 +126,19 @@ impl HarnessAdapter for DescriptorAdapter {
         })
     }
 
+    fn framework_ignore_paths(&self) -> Vec<String> {
+        let mut paths = vec![format!("/{}/", crate::sandbox::GUARD_DENIALS_DIR)];
+        if let Some(skills_dir) = &self.descriptor.skills_dir {
+            paths.push(format!("/{skills_dir}/"));
+        }
+        if let Some(guard) = &self.descriptor.guard
+            && let Some(staged) = crate::adapters::guard::guard_staged_file(guard)
+        {
+            paths.push(format!("/{staged}"));
+        }
+        paths
+    }
+
     fn project_skill_dirs(&self, repo_root: &Path) -> Vec<PathBuf> {
         self.descriptor
             .skills_dir

src/adapters/guard.rs

@@ -336,6 +336,16 @@ fn append_guard_denial(path: &Path, record: &GuardDenialRecord) -> io::Result<()
     file.write_all(&line)
 }
 
+/// The env-relative file this guard engine stages: the merged hook config for
+/// `json-hooks`, the project plugin for the plugin engines. One place decides
+/// it, so a new engine cannot teach half the codebase where its file lives.
+pub(crate) fn guard_staged_file(guard: &GuardSection) -> Option<&str> {
+    match guard.engine {
+        GuardEngine::JsonHooks => guard.hooks_file.as_deref(),
+        GuardEngine::OpencodePlugin | GuardEngine::ClinePlugin => guard.plugin_file.as_deref(),
+    }
+}
+
 /// The hook-surface dir the install created outside the skills dir, which
 /// teardown prunes when restoring the original file leaves it empty. Derived
 /// from the data: the engine's staged file's parent dir (`hooks_file` for
@@ -347,10 +357,7 @@ pub(crate) fn hook_cleanup_dir(
     skills_dir_rel: Option<&str>,
     stage_root: &Path,
 ) -> Option<PathBuf> {
-    let hook_file = match guard.engine {
-        GuardEngine::JsonHooks => guard.hooks_file.as_deref(),
-        GuardEngine::OpencodePlugin | GuardEngine::ClinePlugin => guard.plugin_file.as_deref(),
-    }?;
+    let hook_file = guard_staged_file(guard)?;
     let (parent, _) = hook_file.rsplit_once('/')?;
     if let Some(skills) = skills_dir_rel {
         // Component-wise ancestry: `.a` owns `.a/b/skills` but not `.ab/skills`.

src/adapters/harness.rs

@@ -81,6 +81,19 @@ pub trait HarnessAdapter {
         self.skills_dir(repo_root).into_iter().collect()
     }
 
+    /// Env-relative paths the runner itself places in every task environment,
+    /// as gitignore-style patterns.
+    ///
+    /// Staged skills sit *inside* the task repository, so a codebase whose lint
+    /// or format step globs the whole tree reports the framework's artifacts as
+    /// project failures — and only in the arm that stages a skill. `run` writes
+    /// these patterns into the project's own ignore files
+    /// ([`crate::workspace::tool_ignore`]) to keep that from happening. The
+    /// baseline is the framework outputs dir every harness produces.
+    fn framework_ignore_paths(&self) -> Vec<String> {
+        vec![format!("/{}/", crate::sandbox::GUARD_DENIALS_DIR)]
+    }
+
     // ── Run-option capabilities (defaulted) ──────────────────────────────────
 
     /// The run options the generic `run` preflight may accept for this
@@ -550,6 +563,49 @@ mod tests {
         );
     }
 
+    #[test]
+    fn framework_ignore_paths_cover_the_staged_skills_the_guard_file_and_the_outputs_dir() {
+        assert_eq!(
+            adapter_for(Harness::resolve("claude-code").unwrap()).framework_ignore_paths(),
+            vec![
+                "/.eval-magic-outputs/".to_string(),
+                "/.claude/skills/".to_string(),
+                "/.claude/settings.local.json".to_string(),
+            ]
+        );
+        // A plugin-engine harness contributes its plugin file, not a hooks file.
+        assert_eq!(
+            adapter_for(Harness::resolve("opencode").unwrap()).framework_ignore_paths(),
+            vec![
+                "/.eval-magic-outputs/".to_string(),
+                "/.opencode/skills/".to_string(),
+                "/.opencode/plugins/slow-powers-eval-guard.js".to_string(),
+            ]
+        );
+        assert_eq!(
+            adapter_for(Harness::resolve("codex").unwrap()).framework_ignore_paths(),
+            vec![
+                "/.eval-magic-outputs/".to_string(),
+                "/.agents/skills/".to_string(),
+                "/.codex/hooks.json".to_string(),
+            ]
+        );
+    }
+
+    #[test]
+    fn framework_ignore_paths_omit_what_a_bare_descriptor_never_stages() {
+        let descriptor =
+            crate::adapters::descriptor::load_descriptor("label = \"bare\"\n", "test.toml")
+                .unwrap();
+        let adapter =
+            crate::adapters::descriptor_adapter::DescriptorAdapter::from_descriptor(descriptor);
+
+        assert_eq!(
+            adapter.framework_ignore_paths(),
+            vec!["/.eval-magic-outputs/".to_string()]
+        );
+    }
+
     #[test]
     fn only_codex_and_opencode_rewrite_frontmatter() {
         assert!(!adapter_for(Harness::resolve("claude-code").unwrap()).rewrites_frontmatter_name());

src/cli/args.rs

@@ -602,6 +602,14 @@ pub(crate) enum Commands {
     /// selected harness's project skill roots symmetrically before staging. See
     /// `eval-magic docs codebase` for configuration and provenance, and
     /// `eval-magic docs isolation` for operator-source remedies and verification.
+    ///
+    /// Staged skills, guard files, and framework outputs sit inside the task
+    /// repository, so every environment also gets the project's own ignore files
+    /// (`.prettierignore` and friends, detected from the codebase's tooling)
+    /// taught to skip them — identically in both arms, so a project lint step
+    /// cannot fail in the treatment arm alone. Set the codebase's `ignore_files`
+    /// to name those files yourself, or to `[]` to opt out. See
+    /// `eval-magic docs codebase`.
     Run(RunArgs),
     /// Run every task in a prepared iteration through its harness CLI.
     ///