diff --git a/docs/codex-notes.md b/docs/codex-notes.md index 9fc1766..248271c 100644 --- a/docs/codex-notes.md +++ b/docs/codex-notes.md @@ -112,9 +112,16 @@ tool invocations: `command_execution`, `file_change`, `web_search`, and MCP item `agent_message` is preserved in event order for conversation gating. `transcript_check` matches these parsed items, and because the `[tools]` vocabulary declares `command_execution` and `file_change` beside their Claude-style spellings, a pattern authored against another harness's -tool names grades here through role aliasing. The JSONL exposes **no deterministic skill-tool -event**, so `transcript_surfaces_skill_invocation()` is false and the `__skill_invoked` meta-check -uses the LLM-judge fallback. +tool names grades here through role aliasing. + +The JSONL exposes no dedicated skill-tool event. A bounded Codex CLI 0.152.1 probe instead emitted +a successful `item.completed` `command_execution` whose `command` contained the exact staged +`SKILL.md` path as a literal argument to `sed`; the small synthetic event shape is preserved in +`tests/fixtures/codex/skill-access-0.152.1.jsonl`. The descriptor's +`[transcript.skill_access]` table therefore names `command_execution`, its command and exit-code +arguments, and the accepted read-command basenames. `__skill_invoked` passes locally only when one +of those commands exits zero with the task's exact staged path. A live source, another treatment +member's path, dynamic or partial path text, and response wording do not count. Codex token totals use its blended workload metric: diff --git a/docs/guides/judging.md b/docs/guides/judging.md index f2e6ec6..4f2fc74 100644 --- a/docs/guides/judging.md +++ b/docs/guides/judging.md @@ -173,13 +173,22 @@ Use `run --judge-samples N` to set a campaign-wide default. An assertion's `samp precedence over that default. The effective count must be at least one. The framework-injected `__skill_invoked` checks are not substantive grading and remain single-shot per treatment member. -For a multi-skill treatment, deterministic transcript grading checks every staged slug separately. +For a multi-skill treatment, deterministic transcript grading checks every member separately. A +native skill-tool signature compares its identifier with that member's staged slug. An exact-path +access signature instead requires a successful declared read command whose literal argument is +that member's task-specific staged `SKILL.md` path. Failed commands, skill-name text, final-message +phrasing, the live source path, and another treatment member's staged path do not count. A local +result has `confidence: 1.0` and emits no meta judge task. + The response files use `__skill_invoked__skill-N.json`, and each meta result names its -`skill_name`, so partial and complete invocation are distinguishable. Harness descriptors supply -the tool and argument signature; a harness without deterministic invocation events receives one -LLM fallback task per member. The suite-level `meta_summary.skill_invoked` value is true when any -treatment member was invoked. The `benchmark.json` file retains the suite rate and adds per-skill -counts and rates. A scalar treatment keeps the `__skill_invoked.json` filename and artifact shape. +`skill_name`, so partial and complete access or invocation are distinguishable. A run with no +usable deterministic signature—for example, a descriptor that exposes none—receives one clearly +labeled behavioral-influence fallback task per member. That fallback can estimate whether the +skill influenced the response, but a pass does not prove native invocation or staged-file access. +The compatibility field +`meta_summary.skill_invoked` is true when any treatment member passes its available check. The +`benchmark.json` file retains the suite rate and adds per-skill counts and rates. A scalar treatment +keeps the `__skill_invoked.json` filename and artifact shape. Each sample is a separate judge task and response, but every sample for a run receives the exact same bounded `judge-evidence.md`. The agent is not rerun, and eval-magic does not rebuild or expand diff --git a/docs/progressive-enhancements.md b/docs/progressive-enhancements.md index 072376d..03cf828 100644 --- a/docs/progressive-enhancements.md +++ b/docs/progressive-enhancements.md @@ -190,7 +190,10 @@ or use their documented grading fallback. *Descriptor fields:* the `[transcript]` table — `events_filename` (gate: an absent table means the ingest pipeline never reads a transcript), one primary summary reader, and -`surfaces_skill_invocation`. The primary reader is either `parser` or the summary outputs under +`surfaces_skill_invocation`. A deterministic native skill event uses `skill_tool` / `skill_arg`; +a successful exact-path shell read uses the mutually exclusive `skill_access` table with its tool, +command argument, exit-code argument, and declared read-command basenames. The primary reader is +either `parser` or the summary outputs under `extract`; validation rejects both, neither, and a surface-only extract. The `extract` sub-table is the declarative tier: equality `where` filters, final and ordered assistant-text picks, a session-id pick, flat tool-item mapping, token sum/subtract reduction, duration rule, and the auxiliary diff --git a/harnesses/codex.toml b/harnesses/codex.toml index 6cd15aa..cb47db2 100644 --- a/harnesses/codex.toml +++ b/harnesses/codex.toml @@ -34,17 +34,24 @@ unresolved_phrase = "If it does not load as a Codex skill" header = "## Skills\n" item = "\n- {name}: {description} (file: {path})" -# Codex's JSONL exposes no deterministic skill-tool event, so the -# `__skill_invoked` meta-check uses the LLM-judge fallback. +# Codex's JSONL exposes no deterministic skill-tool event. It does preserve a +# completed command's literal arguments and exit code, so the meta-check uses +# an exact staged-SKILL.md access signature instead. # # The JSONL stream is flat (every tool event self-contained), so its summary is # declarative. Permission denials are independent: the named reader correlates # the sibling stderr capture where Codex reports structural router rejections. [transcript] events_filename = "codex-events.jsonl" -surfaces_skill_invocation = false +surfaces_skill_invocation = true permission_denials_parser = "codex-items" +[transcript.skill_access] +tool = "command_execution" +command_arg = "command" +exit_code_arg = "exit_code" +read_commands = ["cat", "head", "sed", "tail"] + [transcript.extract.tools] where = { type = "item.completed" } item = "item" diff --git a/harnesses/template.toml b/harnesses/template.toml index 327d526..d44018d 100644 --- a/harnesses/template.toml +++ b/harnesses/template.toml @@ -124,18 +124,28 @@ label = "{label}" ## meanwhile. ## VERIFY: capture a real events file and check its line shapes against the reader you declare — ## an unknown parser name fails the schema gate listing the allowed values. Leave -## surfaces_skill_invocation false unless you verified the stream yields a deterministic -## skill-invocation event (false routes the __skill_invoked meta-check to the LLM judge). When -## true, skill_tool / skill_arg name the skill tool and the argument carrying the skill -## identifier (defaults: "Skill" / "skill" — Claude Code's spellings; OpenCode uses -## "skill" / "name"). +## Leave surfaces_skill_invocation false unless the stream yields deterministic native invocation +## or staged-file access evidence (false routes __skill_invoked to a clearly labeled behavioral- +## influence fallback). For a native event, skill_tool / skill_arg name the tool and identifier +## argument (defaults: "Skill" / "skill"; OpenCode uses "skill" / "name"). A harness that exposes +## only successful shell reads may instead declare [transcript.skill_access]: tool is a [tools] +## shell name, command_arg and exit_code_arg locate its fields, and read_commands lists literal +## executable basenames. The exact staged SKILL.md path must be one literal argument. # [transcript] # events_filename = "{label}-events.jsonl" # parser = "codex-items" # permission_denials_parser = "codex-items" -# surfaces_skill_invocation = false -# skill_tool = "Skill" -# skill_arg = "skill" +# surfaces_skill_invocation = true +# +# [transcript.skill_access] +# tool = "command_execution" +# command_arg = "command" +# exit_code_arg = "exit_code" +# read_commands = ["cat", "sed"] +# +## Native-tool alternative to [transcript.skill_access]: +## skill_tool = "Skill" +## skill_arg = "skill" # # [transcript.extract.session_surface] # where = { type = "system", subtype = "init" } diff --git a/schema/harness-descriptor.schema.json b/schema/harness-descriptor.schema.json index 885b5a1..e7af4ab 100644 --- a/schema/harness-descriptor.schema.json +++ b/schema/harness-descriptor.schema.json @@ -244,7 +244,7 @@ }, "surfaces_skill_invocation": { "type": "boolean", - "description": "Whether the transcript deterministically surfaces a skill invocation (false routes the meta-check to the LLM-judge fallback). For extract descriptors, leave false unless the mapping verifiably yields invocations named `Skill` carrying an `args.skill` field." + "description": "Whether the transcript deterministically surfaces native skill invocation or exact staged-file access (false routes the meta-check to a behavioral-influence fallback)." }, "skill_tool": { "type": "string", @@ -255,6 +255,24 @@ "type": "string", "minLength": 1, "description": "Argument of the skill-invocation tool that carries the staged slug (default \"skill\"; OpenCode declares \"name\")." + }, + "skill_access": { + "type": "object", + "required": ["tool", "command_arg", "exit_code_arg", "read_commands"], + "additionalProperties": false, + "description": "Deterministic exact-path access signature for a harness without a native skill tool. The named tool must complete successfully and its command argument must contain the task's exact staged SKILL.md path as a literal shell argument.", + "properties": { + "tool": { "type": "string", "minLength": 1 }, + "command_arg": { "type": "string", "minLength": 1 }, + "exit_code_arg": { "type": "string", "minLength": 1 }, + "read_commands": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "minLength": 1 }, + "description": "Literal executable basenames whose successful invocation with the exact staged path counts as a content read." + } + } } } }, diff --git a/schema/run-record.schema.json b/schema/run-record.schema.json index 27691b7..15f5b35 100644 --- a/schema/run-record.schema.json +++ b/schema/run-record.schema.json @@ -25,7 +25,11 @@ }, "skill_path": { "type": ["string", "null"], - "description": "Absolute path to the SKILL.md the subagent could load, or null if no skill was provided (without_skill condition)." + "description": "Absolute path to the campaign-owned SKILL.md input, or null if no skill was provided. The task-specific staged location is staged_skill_path." + }, + "staged_skill_path": { + "type": "string", + "description": "Exact task-environment path to the staged scalar SKILL.md. Absent for control, unstaged, and historical runs." }, "skills": { "type": "array", @@ -107,7 +111,11 @@ "properties": { "name": { "type": "string" }, "skill_path": { "type": "string" }, - "staged_skill_slug": { "type": ["string", "null"] } + "staged_skill_slug": { "type": ["string", "null"] }, + "staged_skill_path": { + "type": "string", + "description": "Exact task-environment path to this member's staged SKILL.md. Absent before a task env is selected and in historical records." + } } }, "responderOutcome": { diff --git a/src/adapters/descriptor/transcript_section.rs b/src/adapters/descriptor/transcript_section.rs index abe4ffd..5bb4f5e 100644 --- a/src/adapters/descriptor/transcript_section.rs +++ b/src/adapters/descriptor/transcript_section.rs @@ -9,6 +9,16 @@ use serde::{Deserialize, Serialize}; use super::{default_true, is_true}; +/// A deterministic staged-skill access encoded in a successful native command +/// item rather than a dedicated skill tool event. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct SkillAccessSection { + pub tool: String, + pub command_arg: String, + pub exit_code_arg: String, + pub read_commands: Vec, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct TranscriptSection { pub events_filename: String, @@ -27,15 +37,19 @@ pub struct TranscriptSection { pub extract: Option, #[serde(default = "default_true", skip_serializing_if = "is_true")] pub surfaces_skill_invocation: bool, - /// Tool name of the deterministic skill-invocation event the - /// `__skill_invoked` meta-check matches (default `"Skill"` — Claude - /// Code's Skill tool). + /// Tool name of a deterministic native skill-invocation event (default + /// `"Skill"` — Claude Code's Skill tool). Mutually exclusive with + /// [`Self::skill_access`]. #[serde(skip_serializing_if = "Option::is_none")] pub skill_tool: Option, /// Argument of the skill-invocation tool that carries the staged slug /// (default `"skill"`). #[serde(skip_serializing_if = "Option::is_none")] pub skill_arg: Option, + /// Successful shell-command signature whose literal argument must equal + /// the task's exact staged `SKILL.md` path. + #[serde(skip_serializing_if = "Option::is_none")] + pub skill_access: Option, } impl TranscriptSection { diff --git a/src/adapters/descriptor/validation.rs b/src/adapters/descriptor/validation.rs index 712a52e..1f98bf3 100644 --- a/src/adapters/descriptor/validation.rs +++ b/src/adapters/descriptor/validation.rs @@ -37,6 +37,7 @@ const CHECKS: &[Check] = &[ check_guard_engine_fields, check_guard_verdict_template, transcript::check_tool_vocabulary, + transcript::check_skill_evidence, transcript::check_tiers, conversation::validate, check_tool_roles_disjoint, diff --git a/src/adapters/descriptor/validation/tests/transcript.rs b/src/adapters/descriptor/validation/tests/transcript.rs index a6e8d90..4afb468 100644 --- a/src/adapters/descriptor/validation/tests/transcript.rs +++ b/src/adapters/descriptor/validation/tests/transcript.rs @@ -112,6 +112,72 @@ fn accepts_an_independent_permission_denials_parser_with_extract_ingest() { load_descriptor(&descriptor, "test.toml").expect("denial parsing should compose"); } +#[test] +fn accepts_a_successful_exact_path_skill_access_signature() { + let descriptor = format!( + "{TOOLED}\n[transcript]\nevents_filename = \"demo-events.jsonl\"\n\ + parser = \"codex-items\"\nsurfaces_skill_invocation = true\n\n\ + [transcript.skill_access]\n\ + tool = \"command_execution\"\n\ + command_arg = \"command\"\n\ + exit_code_arg = \"exit_code\"\n\ + read_commands = [\"cat\", \"sed\"]\n" + ); + + load_descriptor(&descriptor, "test.toml") + .expect("an exact-path access signature should be descriptor-declared"); +} + +#[test] +fn rejects_skill_access_that_is_not_a_true_exclusive_shell_signal() { + for (extra_transcript, expected) in [ + ( + "surfaces_skill_invocation = false\n", + "surfaces_skill_invocation", + ), + ( + "surfaces_skill_invocation = true\nskill_tool = \"Skill\"\n", + "skill_tool", + ), + ] { + let err = err_of(&format!( + "{TOOLED}\n[transcript]\nevents_filename = \"demo-events.jsonl\"\n\ + parser = \"codex-items\"\n{extra_transcript}\n\ + [transcript.skill_access]\n\ + tool = \"command_execution\"\n\ + command_arg = \"command\"\n\ + exit_code_arg = \"exit_code\"\n\ + read_commands = [\"cat\", \"sed\"]\n" + )); + assert!(err.contains(expected), "{err}"); + } + + let err = err_of(&format!( + "{TOOLED}\n[transcript]\nevents_filename = \"demo-events.jsonl\"\n\ + parser = \"codex-items\"\nsurfaces_skill_invocation = true\n\n\ + [transcript.skill_access]\n\ + tool = \"file_change\"\n\ + command_arg = \"command\"\n\ + exit_code_arg = \"exit_code\"\n\ + read_commands = [\"cat\", \"sed\"]\n" + )); + assert!(err.contains("tools.shell"), "{err}"); +} + +#[test] +fn rejects_skill_access_read_commands_that_are_not_literal_basenames() { + let err = err_of(&format!( + "{TOOLED}\n[transcript]\nevents_filename = \"demo-events.jsonl\"\n\ + parser = \"codex-items\"\nsurfaces_skill_invocation = true\n\n\ + [transcript.skill_access]\n\ + tool = \"command_execution\"\n\ + command_arg = \"command\"\n\ + exit_code_arg = \"exit_code\"\n\ + read_commands = [\"/bin/cat\"]\n" + )); + assert!(err.contains("literal executable basenames"), "{err}"); +} + #[test] fn rejects_transcript_with_neither_parser_nor_extract() { for auxiliary in ["", "permission_denials_parser = \"codex-items\"\n"] { diff --git a/src/adapters/descriptor/validation/transcript.rs b/src/adapters/descriptor/validation/transcript.rs index f25505b..b7e619b 100644 --- a/src/adapters/descriptor/validation/transcript.rs +++ b/src/adapters/descriptor/validation/transcript.rs @@ -13,6 +13,60 @@ pub(super) fn check_tool_vocabulary(d: &HarnessDescriptor) -> Result<(), String> Ok(()) } +/// Exact-path access is an alternative to a native skill-tool signature. Its +/// tool must be declared as shell so the descriptor cannot assign command-line +/// semantics to an unrelated transcript item. +pub(super) fn check_skill_evidence(d: &HarnessDescriptor) -> Result<(), String> { + let Some(transcript) = &d.transcript else { + return Ok(()); + }; + let Some(access) = &transcript.skill_access else { + return Ok(()); + }; + if !transcript.surfaces_skill_invocation { + return Err( + "transcript.skill_access requires surfaces_skill_invocation = true; otherwise the \ + descriptor declares deterministic evidence and disables it at the same time" + .into(), + ); + } + if transcript.skill_tool.is_some() || transcript.skill_arg.is_some() { + return Err( + "transcript.skill_access cannot be combined with skill_tool or skill_arg; declare \ + either exact-path access or a native skill-tool signature" + .into(), + ); + } + if !d.tools.shell.contains(&access.tool) { + return Err(format!( + "transcript.skill_access.tool {:?} is not declared in tools.shell; exact-path \ + command evidence must use a shell tool", + access.tool + )); + } + if access.command_arg == access.exit_code_arg { + return Err( + "transcript.skill_access command_arg and exit_code_arg must name distinct fields" + .into(), + ); + } + if access.read_commands.iter().any(|command| { + command.is_empty() + || command == "." + || command == ".." + || !command + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '+' | '.')) + }) { + return Err( + "transcript.skill_access.read_commands must contain literal executable basenames; \ + paths, whitespace, and shell syntax are not allowed" + .into(), + ); + } + Ok(()) +} + /// Transcript ingest has one primary summary reader: a named code parser or /// declarative summary outputs. Independent denial and session-surface readers /// may compose around it. diff --git a/src/adapters/descriptor_adapter.rs b/src/adapters/descriptor_adapter.rs index aa80ba8..98219b2 100644 --- a/src/adapters/descriptor_adapter.rs +++ b/src/adapters/descriptor_adapter.rs @@ -303,6 +303,7 @@ impl HarnessAdapter for DescriptorAdapter { fn transcript_skill_invocation(&self) -> Option<(String, String)> { match &self.descriptor.transcript { Some(t) if !t.surfaces_skill_invocation => None, + Some(t) if t.skill_access.is_some() => None, Some(t) => Some(( t.skill_tool.clone().unwrap_or_else(|| "Skill".to_string()), t.skill_arg.clone().unwrap_or_else(|| "skill".to_string()), @@ -313,6 +314,33 @@ impl HarnessAdapter for DescriptorAdapter { } } + fn transcript_skill_evidence(&self) -> Option { + use crate::adapters::SkillEvidenceSignature; + + let transcript = self.descriptor.transcript.as_ref()?; + if !transcript.surfaces_skill_invocation { + return None; + } + if let Some(access) = &transcript.skill_access { + return Some(SkillEvidenceSignature::StagedPathAccess { + tool: access.tool.clone(), + command_arg: access.command_arg.clone(), + exit_code_arg: access.exit_code_arg.clone(), + read_commands: access.read_commands.clone(), + }); + } + Some(SkillEvidenceSignature::Invocation { + tool: transcript + .skill_tool + .clone() + .unwrap_or_else(|| "Skill".to_string()), + arg: transcript + .skill_arg + .clone() + .unwrap_or_else(|| "skill".to_string()), + }) + } + fn cli_model_flag(&self) -> Option { self.model_flag().map(str::to_string) } diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 4d16c45..ce34b3f 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -28,6 +28,21 @@ use crate::sandbox::GuardMarker; use super::skill_shadow::{PluginShadowReport, ShadowSource}; use super::{PermissionDenial, SessionSurface, TranscriptSummary}; +/// Deterministic transcript evidence for the `__skill_invoked` meta-check. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SkillEvidenceSignature { + /// A native skill tool whose argument equals the staged slug. + Invocation { tool: String, arg: String }, + /// A successful command carrying the exact staged `SKILL.md` path as a + /// literal argument. + StagedPathAccess { + tool: String, + command_arg: String, + exit_code_arg: String, + read_commands: Vec, + }, +} + /// The role a tool name plays in a harness's vocabulary. A descriptor's roles /// are validated disjoint (`descriptor::validation::check_tool_roles_disjoint`), /// so one native name maps to at most one role. @@ -302,15 +317,22 @@ pub trait HarnessAdapter { )) } - /// **Enhancement: transcript ingest.** The deterministic skill-invocation - /// signature the `__skill_invoked` meta-check matches: `(tool name, arg - /// carrying the staged slug)` — Claude Code's `Skill`/`skill`, OpenCode's - /// `skill`/`name`. `None` for Codex (its JSONL has no skill-tool event), - /// which routes the meta-check to the LLM-judge fallback. + /// **Enhancement: transcript ingest.** The native skill-invocation part of + /// the `__skill_invoked` contract: `(tool name, arg carrying the staged + /// slug)` — Claude Code's `Skill`/`skill`, OpenCode's `skill`/`name`. + /// Harnesses using exact staged-path access return `None` here and expose + /// that alternative through [`Self::transcript_skill_evidence`]. fn transcript_skill_invocation(&self) -> Option<(String, String)> { Some(("Skill".to_string(), "skill".to_string())) } + /// Deterministic invocation/access evidence exposed by this harness. The + /// default preserves the historical native skill-tool contract. + fn transcript_skill_evidence(&self) -> Option { + self.transcript_skill_invocation() + .map(|(tool, arg)| SkillEvidenceSignature::Invocation { tool, arg }) + } + /// **Enhancement: transcript denial reader.** Whether this harness's /// transcript identifies tool calls it refused to run. `false` for /// harnesses whose refusals are not distinguishable from ordinary tool @@ -688,6 +710,15 @@ mod tests { adapter_for(Harness::resolve("codex").unwrap()).transcript_skill_invocation(), None ); + assert_eq!( + adapter_for(Harness::resolve("codex").unwrap()).transcript_skill_evidence(), + Some(SkillEvidenceSignature::StagedPathAccess { + tool: "command_execution".to_string(), + command_arg: "command".to_string(), + exit_code_arg: "exit_code".to_string(), + read_commands: ["cat", "head", "sed", "tail"].map(str::to_string).to_vec(), + }) + ); assert_eq!( adapter_for(Harness::resolve("opencode").unwrap()).transcript_skill_invocation(), Some(("skill".to_string(), "name".to_string())) diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 4006ef9..bb27252 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -40,7 +40,7 @@ pub mod transcript; pub use harness::{ CliDispatchContext, CliManifestContext, EMPTY_TOOL_VOCABULARY, HarnessAdapter, - RUNBOOK_TEMPLATE, TokenUsageAggregation, ToolRole, ToolVocabulary, + RUNBOOK_TEMPLATE, SkillEvidenceSignature, TokenUsageAggregation, ToolRole, ToolVocabulary, }; pub use registry::{ DEFAULT_HARNESS_NAME, UnknownHarnessError, adapter_for, all_config_dir_names, diff --git a/src/cli/run/dispatch.rs b/src/cli/run/dispatch.rs index 3ea4133..e1474ab 100644 --- a/src/cli/run/dispatch.rs +++ b/src/cli/run/dispatch.rs @@ -39,6 +39,8 @@ pub struct DispatchTask { pub skill_path: Option, pub staged_skill_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_skill_path: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub skills: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub available_skills: Option>, @@ -252,6 +254,7 @@ pub fn build_dispatch_task(opts: &DispatchTaskOpts) -> Result::to_vec), available_skills: opts.skills.map(|_| staged_skills), user_prompt: opts.user_prompt.to_string(), diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index 800bae8..bd287fc 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -30,7 +30,7 @@ use crate::core::fs::{artifact_path, write_json}; mod ignore; mod roster; -use roster::condition_roster; +use roster::{condition_roster, staged_skill_path_for, task_roster}; /// Build every `(eval, condition)` dispatch task and write `conditions.json`, /// `dispatch-manifest.md`, the per-task prompt files, and `dispatch.json`. @@ -100,16 +100,6 @@ pub(super) fn write_dispatch( // result can only be tied to a skill revision if each record names one. let skill_source_record = r.skill.record(); - let staged_skill_path_for = |env_root: &Path, cond_slug: Option<&str>| -> Option { - cond_slug.map(|slug| { - artifact_path( - &skills_dir_for_harness(env_root, ctx.harness) - .join(slug) - .join("SKILL.md"), - ) - }) - }; - // availableSkills for a condition in a given env = siblings + the // skill-under-test when that condition loads it. Paths are task-env-specific. let available_skills_for = |env_root: &Path, @@ -146,7 +136,7 @@ pub(super) fn write_dispatch( path: treatment .staged_skill_slug .as_deref() - .and_then(|slug| staged_skill_path_for(env_root, Some(slug))) + .and_then(|slug| staged_skill_path_for(env_root, ctx.harness, Some(slug))) .unwrap_or_else(|| treatment.skill_path.clone()), description: get_skill_description(Path::new(&treatment.skill_path)), }); @@ -160,7 +150,8 @@ pub(super) fn write_dispatch( }; skills.push(AvailableSkill { name, - path: staged_skill_path_for(env_root, cond_slug).unwrap_or_else(|| csp.to_string()), + path: staged_skill_path_for(env_root, ctx.harness, cond_slug) + .unwrap_or_else(|| csp.to_string()), description: get_skill_description(Path::new(csp)), }); } @@ -229,13 +220,10 @@ pub(super) fn write_dispatch( env_run_index, ); let env_root_str = env_root.to_string_lossy().into_owned(); - let staged_path = staged_skill_path_for(&env_root, cond_slug); - let available_skills = available_skills_for( - &env_root, - cond_skill_path, - cond_slug, - condition_roster, - ); + let staged_path = staged_skill_path_for(&env_root, ctx.harness, cond_slug); + let task_roster = task_roster(condition_roster, &env_root, ctx.harness); + let available_skills = + available_skills_for(&env_root, cond_skill_path, cond_slug, &task_roster); // Create the per-run meta dir (run.json / timing.json), which // lives above the env. fs::create_dir_all(&run_dir)?; @@ -263,7 +251,7 @@ pub(super) fn write_dispatch( skill_path: cond_skill_path, staged_skill_slug: cond_slug, staged_skill_path: staged_path.as_deref(), - skills: multi_skill.then_some(condition_roster), + skills: multi_skill.then_some(task_roster.as_slice()), treatment_names: multi_skill.then_some(treatment_names.as_slice()), user_prompt: &ev.prompt, files, diff --git a/src/cli/run/orchestrate/build/roster.rs b/src/cli/run/orchestrate/build/roster.rs index c5bd7cb..1bd0dd1 100644 --- a/src/cli/run/orchestrate/build/roster.rs +++ b/src/cli/run/orchestrate/build/roster.rs @@ -1,8 +1,9 @@ use std::path::Path; -use crate::core::ConditionSkill; use crate::core::fs::artifact_path; +use crate::core::{ConditionSkill, Harness}; +use super::super::super::staging::skills_dir_for_harness; use super::super::StagedTreatmentSkill; pub(super) fn condition_roster( @@ -18,6 +19,39 @@ pub(super) fn condition_roster( .iter() .find(|skill| &skill.name == name) .and_then(|skill| skill.slug.clone()), + staged_skill_path: None, + }) + .collect() +} + +pub(super) fn staged_skill_path_for( + env_root: &Path, + harness: Harness, + staged_slug: Option<&str>, +) -> Option { + staged_slug.map(|slug| { + artifact_path( + &skills_dir_for_harness(env_root, harness) + .join(slug) + .join("SKILL.md"), + ) + }) +} + +pub(super) fn task_roster( + condition_roster: &[ConditionSkill], + env_root: &Path, + harness: Harness, +) -> Vec { + condition_roster + .iter() + .map(|skill| ConditionSkill { + staged_skill_path: staged_skill_path_for( + env_root, + harness, + skill.staged_skill_slug.as_deref(), + ), + ..skill.clone() }) .collect() } diff --git a/src/core/types.rs b/src/core/types.rs index 361146b..95dab0a 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -418,6 +418,11 @@ pub struct ConditionSkill { pub name: String, pub skill_path: String, pub staged_skill_slug: Option, + /// Exact task-environment path to the staged `SKILL.md`. Absent in + /// conditions-level and historical records, where no single task env owns + /// the roster yet. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_skill_path: Option, } /// One condition in a comparison run. @@ -546,6 +551,10 @@ pub struct RunRecord { pub eval_id: String, pub condition: String, pub skill_path: Option, + /// Exact task-environment path to the staged scalar skill. Absent for + /// unstaged/control runs and historical records. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_skill_path: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub skills: Option>, pub prompt: String, @@ -839,6 +848,7 @@ mod tests { eval_id: "e".into(), condition: "with-skill".into(), skill_path: None, + staged_skill_path: None, skills: None, prompt: "p".into(), files: vec![], diff --git a/src/pipeline/grade/judge_tasks.rs b/src/pipeline/grade/judge_tasks.rs index 0bfe918..985797a 100644 --- a/src/pipeline/grade/judge_tasks.rs +++ b/src/pipeline/grade/judge_tasks.rs @@ -2,10 +2,10 @@ //! //! For each //! `(eval, condition)` it builds judge prompts for `llm_judge` assertions and a -//! skill-invocation meta-check (code-checked from the transcript when possible, -//! else emitted as an LLM judge task), writing `judge-tasks.json` plus the -//! per-assertion prompt files. `transcript_check` assertions are not dispatched -//! here — they are graded directly in `finalize`. +//! skill invocation/access meta-check (code-checked from the transcript when +//! possible, else emitted as a behavioral-influence judge task), writing +//! `judge-tasks.json` plus the per-assertion prompt files. `transcript_check` +//! assertions are not dispatched here — they are graded directly in `finalize`. use std::collections::HashMap; use std::fs; @@ -14,8 +14,9 @@ use std::path::Path; use serde::Serialize; use serde_json::json; +use crate::adapters::SkillEvidenceSignature; use crate::core::fs::{artifact_path, write_json}; -use crate::core::{Assertion, ConditionSkill, RunRecord, SKILL_INVOKED_META_ID, ToolInvocation}; +use crate::core::{Assertion, ConditionSkill, RunRecord, SKILL_INVOKED_META_ID}; use crate::pipeline::error::PipelineError; use crate::pipeline::io::now_iso8601; use crate::pipeline::slots::run_slots; @@ -25,6 +26,11 @@ use super::GradeContext; use super::evidence::{EvidenceBundleRef, JUDGE_PROMPT_BYTE_LIMIT, build_evidence_bundle}; use super::stale_verdicts; +mod skill_evidence; + +pub use skill_evidence::check_skill_invoked_from_transcript; +use skill_evidence::{check_skill_evidence_from_transcript, skill_invoked_rubric}; + /// One judge task. `dispatch_prompt` carries the full prompt in memory but is /// stripped from the serialized `judge-tasks.json` (the orchestrator reads it /// from `dispatch_prompt_path` instead). `model` is always present (null or a @@ -86,30 +92,6 @@ pub struct EmitSummary { pub warnings: Vec, } -/// True when the transcript shows the harness's skill tool invoked with the -/// staged slug: the invocation named `skill_tool` whose `skill_arg` argument -/// equals the slug (Claude Code's `Skill`/`skill`, OpenCode's -/// `skill`/`name`). -pub fn check_skill_invoked_from_transcript( - invocations: &[ToolInvocation], - staged_slug: Option<&str>, - skill_tool: &str, - skill_arg: &str, -) -> bool { - let Some(slug) = staged_slug else { - return false; - }; - invocations.iter().any(|inv| { - inv.name == skill_tool - && inv - .args - .as_ref() - .and_then(|a| a.get(skill_arg)) - .and_then(|v| v.as_str()) - == Some(slug) - }) -} - pub(super) fn meta_response_stem(index: usize, multi_skill: bool) -> String { if multi_skill { format!("{SKILL_INVOKED_META_ID}__skill-{}", index + 1) @@ -118,49 +100,6 @@ pub(super) fn meta_response_stem(index: usize, multi_skill: bool) -> String { } } -/// The meta-check rubric asking a judge whether the agent actually applied the -/// skill (separate from correctness). -fn skill_invoked_rubric(skill_name: &str, skill_content: Option<&str>) -> String { - let mut lines: Vec = vec![ - format!( - "The agent had access to the **{skill_name}** skill. This meta-check asks whether \ - there is evidence the agent actually applied the skill in this run — separate from \ - whether the response was correct." - ), - String::new(), - ]; - if let Some(content) = skill_content { - lines.push("# Skill content".to_string()); - lines.push(String::new()); - lines.push("```markdown".to_string()); - lines.push(content.trim().to_string()); - lines.push("```".to_string()); - lines.push(String::new()); - } - lines.extend( - [ - "Evidence the skill WAS applied:", - "- The agent cites the skill by name or references specific named sections (e.g. \"Iron Law\", \"Red Flags\", \"Gate Function\", or any other distinctive heading from the skill).", - "- The agent's response uses distinctive vocabulary or phrasing taken from the skill content.", - "- The agent's behavior follows a specific procedural step prescribed by the skill in a way that mirrors the skill's phrasing — not just generic best practice.", - "- The agent explicitly acknowledges following the skill's guidance.", - "", - "Evidence the skill was NOT applied:", - "- The response uses only generic best-practice language unrelated to the skill's specific framing.", - "- No vocabulary, structure, or rules from the skill content appear anywhere in the response.", - "- The response would read identically with or without the skill loaded.", - "", - "Compare the agent's `final_message`, conversation transcript, and tool invocation summary against the skill content. Look for stylistic and procedural fingerprints.", - "", - "PASS if there is observable evidence the skill influenced the response.", - "FAIL if there is no observable evidence — the response is indistinguishable from baseline behavior.", - ] - .iter() - .map(|s| s.to_string()), - ); - lines.join("\n") -} - /// Assemble one bounded judge prompt around its persisted evidence bundle. fn build_judge_prompt( assertion_id: &str, @@ -231,6 +170,7 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result Result crate::adapters::adapter_for(harness).transcript_skill_evidence(), + None => Some(SkillEvidenceSignature::Invocation { + tool: "Skill".to_string(), + arg: "skill".to_string(), + }), + }; let default_judge_model = ctx.conditions.judge_model.clone(); let tasks_path = ctx.iteration_dir.join("judge-tasks.json"); @@ -367,43 +309,78 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result { + treatment.staged_skill_slug.is_some() + } + SkillEvidenceSignature::StagedPathAccess { .. } => { + staged_path.is_some() + } + }); + if deterministic_target_available { + let signature = skill_signature + .as_ref() + .expect("target availability requires a signature"); + let invoked = check_skill_evidence_from_transcript( &run_record.tool_invocations, treatment.staged_skill_slug.as_deref(), - &skill_tool, - &skill_arg, + staged_path, + signature, ); - let evidence = if invoked { - if *multi_skill { + let evidence = match signature { + SkillEvidenceSignature::Invocation { .. } if invoked => { + if *multi_skill { + format!( + "Skill '{}' invocation verified from transcript.", + treatment.name + ) + } else { + "Skill invocation verified from transcript.".to_string() + } + } + SkillEvidenceSignature::Invocation { .. } if *multi_skill => { + format!( + "No invocation of skill '{}' found in transcript across {} transcript invocation(s).", + treatment.name, + run_record.tool_invocations.len() + ) + } + SkillEvidenceSignature::Invocation { .. } => format!( + "No skill invocation found in transcript across {} transcript invocation(s).", + run_record.tool_invocations.len() + ), + SkillEvidenceSignature::StagedPathAccess { .. } if invoked => { format!( - "Skill '{}' invocation verified from transcript.", + "Skill '{}' access verified from a successful transcript command reading its exact staged SKILL.md path.", treatment.name ) - } else { - "Skill invocation verified from transcript.".to_string() } - } else if *multi_skill { - format!( - "No invocation of skill '{}' found in transcript across {} transcript invocation(s).", + SkillEvidenceSignature::StagedPathAccess { .. } => format!( + "No deterministic access to skill '{}': no successful transcript command read its exact staged SKILL.md path across {} transcript invocation(s).", treatment.name, run_record.tool_invocations.len() - ) - } else { - format!( - "No skill invocation found in transcript across {} transcript invocation(s).", - run_record.tool_invocations.len() - ) + ), }; let mut response = json!({ "passed": invoked, @@ -486,6 +463,7 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result, ordinal: u32) -> ToolInvocation { @@ -543,20 +521,23 @@ mod tests { let second = "second-treatment-slug"; for harness_name in ["claude-code", "cline", "opencode"] { let harness = crate::core::Harness::resolve(harness_name).unwrap(); - let (tool, arg) = crate::adapters::adapter_for(harness) - .transcript_skill_invocation() + let signature = crate::adapters::adapter_for(harness) + .transcript_skill_evidence() .expect("these built-in harnesses expose deterministic invocation events"); - let invocations = [inv(&tool, Some(json!({arg.clone(): second})), 0)]; + let SkillEvidenceSignature::Invocation { tool, arg } = &signature else { + panic!("{harness_name} must retain its native skill-tool signature"); + }; + let invocations = [inv(tool, Some(json!({arg.clone(): second})), 0)]; assert!( - !check_skill_invoked_from_transcript(&invocations, Some(first), &tool, &arg), + !check_skill_evidence_from_transcript(&invocations, Some(first), None, &signature,), "{harness_name} falsely attributed the second skill to the first" ); - assert!(check_skill_invoked_from_transcript( + assert!(check_skill_evidence_from_transcript( &invocations, Some(second), - &tool, - &arg + None, + &signature, )); } } diff --git a/src/pipeline/grade/judge_tasks/skill_evidence.rs b/src/pipeline/grade/judge_tasks/skill_evidence.rs new file mode 100644 index 0000000..918a10c --- /dev/null +++ b/src/pipeline/grade/judge_tasks/skill_evidence.rs @@ -0,0 +1,178 @@ +use crate::adapters::SkillEvidenceSignature; +use crate::core::ToolInvocation; + +/// True when the transcript shows the harness's skill tool invoked with the +/// staged slug: the invocation named `skill_tool` whose `skill_arg` argument +/// equals the slug (Claude Code's `Skill`/`skill`, OpenCode's +/// `skill`/`name`). +pub fn check_skill_invoked_from_transcript( + invocations: &[ToolInvocation], + staged_slug: Option<&str>, + skill_tool: &str, + skill_arg: &str, +) -> bool { + let Some(slug) = staged_slug else { + return false; + }; + invocations.iter().any(|inv| { + inv.name == skill_tool + && inv + .args + .as_ref() + .and_then(|a| a.get(skill_arg)) + .and_then(|v| v.as_str()) + == Some(slug) + }) +} + +pub(super) fn check_skill_evidence_from_transcript( + invocations: &[ToolInvocation], + staged_slug: Option<&str>, + staged_skill_path: Option<&str>, + signature: &SkillEvidenceSignature, +) -> bool { + match signature { + SkillEvidenceSignature::Invocation { tool, arg } => { + check_skill_invoked_from_transcript(invocations, staged_slug, tool, arg) + } + SkillEvidenceSignature::StagedPathAccess { + tool, + command_arg, + exit_code_arg, + read_commands, + } => { + let Some(staged_path) = staged_skill_path else { + return false; + }; + invocations.iter().any(|inv| { + if inv.name != *tool { + return false; + } + let Some(args) = inv.args.as_ref() else { + return false; + }; + let succeeded = args + .get(exit_code_arg) + .is_some_and(|code| code.as_i64() == Some(0) || code.as_u64() == Some(0)); + succeeded + && args + .get(command_arg) + .and_then(|command| command.as_str()) + .is_some_and(|command| { + crate::sandbox::command_reads_literal_path( + command, + staged_path, + read_commands, + ) + }) + }) + } + } +} + +/// Behavioral-influence fallback for a run without usable deterministic +/// invocation/access evidence. This is deliberately weaker than the native +/// `__skill_invoked` contract and says so in the prompt. +pub(super) fn skill_invoked_rubric(skill_name: &str, skill_content: Option<&str>) -> String { + let mut lines: Vec = vec![ + format!( + "This run exposes no usable deterministic skill invocation/access signal for the \ + **{skill_name}** skill. This behavioral-influence fallback asks whether the skill \ + appears to have influenced the run — separate from whether the response was \ + correct. A fallback PASS does not prove native invocation or access." + ), + String::new(), + ]; + if let Some(content) = skill_content { + lines.push("# Skill content".to_string()); + lines.push(String::new()); + lines.push("```markdown".to_string()); + lines.push(content.trim().to_string()); + lines.push("```".to_string()); + lines.push(String::new()); + } + lines.extend( + [ + "Evidence the skill DID influence behavior:", + "- The agent cites the skill by name or references specific named sections (e.g. \"Iron Law\", \"Red Flags\", \"Gate Function\", or any other distinctive heading from the skill).", + "- The agent's response uses distinctive vocabulary or phrasing taken from the skill content.", + "- The agent's behavior follows a specific procedural step prescribed by the skill in a way that mirrors the skill's phrasing — not just generic best practice.", + "- The agent explicitly acknowledges following the skill's guidance.", + "", + "Evidence the skill DID NOT observably influence behavior:", + "- The response uses only generic best-practice language unrelated to the skill's specific framing.", + "- No vocabulary, structure, or rules from the skill content appear anywhere in the response.", + "- The response would read identically with or without the skill loaded.", + "", + "Compare the agent's `final_message`, conversation transcript, and tool invocation summary against the skill content. Look for stylistic and procedural fingerprints.", + "", + "PASS only as behavioral-influence evidence when the skill observably influenced the response.", + "FAIL when there is no observable influence — the response is indistinguishable from baseline behavior.", + ] + .iter() + .map(|s| s.to_string()), + ); + lines.join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn invocation(command: String, exit_code: i64) -> ToolInvocation { + ToolInvocation { + name: "command_execution".to_string(), + args: Some(json!({"command": command, "exit_code": exit_code})), + result: None, + ordinal: 0, + } + } + + #[test] + fn exact_path_access_requires_a_successful_literal_match() { + let staged = "/work/env/.agents/skills/target/SKILL.md"; + let signature = SkillEvidenceSignature::StagedPathAccess { + tool: "command_execution".to_string(), + command_arg: "command".to_string(), + exit_code_arg: "exit_code".to_string(), + read_commands: vec!["cat".to_string(), "sed".to_string()], + }; + let check = |command: String, exit_code| { + check_skill_evidence_from_transcript( + &[invocation(command, exit_code)], + Some("target"), + Some(staged), + &signature, + ) + }; + + assert!(check( + format!("/usr/bin/zsh -lc \"sed -n '1,240p' {staged}\""), + 0 + )); + for command in [ + "cat /work/live/target/SKILL.md".to_string(), + "cat /work/env/.agents/skills/other/SKILL.md".to_string(), + format!("cat {staged}.backup"), + "cat $STAGED_SKILL_PATH".to_string(), + format!("echo {staged}"), + format!("echo \"cat {staged}\""), + format!("cat {staged} || true"), + ] { + assert!(!check(command, 0)); + } + assert!(!check(format!("cat {staged}"), 1)); + } + + #[test] + fn fallback_rubric_labels_behavioral_influence_without_claiming_native_invocation() { + let rubric = skill_invoked_rubric("mr-review", Some("# Review carefully")); + assert!(rubric.contains("behavioral-influence fallback"), "{rubric}"); + assert!( + rubric.contains("does not prove native invocation or access"), + "{rubric}" + ); + assert!(!rubric.contains("actually applied the skill"), "{rubric}"); + } +} diff --git a/src/pipeline/record_runs.rs b/src/pipeline/record_runs.rs index 405dadc..c2368df 100644 --- a/src/pipeline/record_runs.rs +++ b/src/pipeline/record_runs.rs @@ -58,6 +58,8 @@ struct DispatchTask { run_index: Option, skill_path: Option, #[serde(default)] + staged_skill_path: Option, + #[serde(default)] skills: Option>, user_prompt: String, #[serde(alias = "fixtures")] @@ -280,6 +282,7 @@ pub fn record_runs( eval_id: task.eval_id.clone(), condition: task.condition.clone(), skill_path: task.skill_path.clone(), + staged_skill_path: task.staged_skill_path.clone(), skills: task.skills.clone(), prompt: task.user_prompt.clone(), files: task.files.clone(), diff --git a/src/pipeline/record_runs/tests/assembly.rs b/src/pipeline/record_runs/tests/assembly.rs index 600d848..d2de193 100644 --- a/src/pipeline/record_runs/tests/assembly.rs +++ b/src/pipeline/record_runs/tests/assembly.rs @@ -38,6 +38,14 @@ fn assembles_run_and_timing_for_every_task_from_disk() { assert_eq!(run.tool_invocations.len(), 1); assert_eq!(run.tool_invocations[0].name, "Bash"); assert_eq!(run.tool_invocations[0].ordinal, 1); + let run_json: Value = serde_json::from_str( + &fs::read_to_string(iter.join("eval-crash/with_skill/run.json")).unwrap(), + ) + .unwrap(); + assert_eq!( + run_json["staged_skill_path"], + json!("/task/.agents/skills/test-slug/SKILL.md") + ); assert!( read_run(&iter, "crash", "without_skill") diff --git a/src/pipeline/record_runs/tests/mod.rs b/src/pipeline/record_runs/tests/mod.rs index 6dad494..51e1355 100644 --- a/src/pipeline/record_runs/tests/mod.rs +++ b/src/pipeline/record_runs/tests/mod.rs @@ -148,6 +148,7 @@ fn write_iteration(iteration_dir: &Path, tasks: &[FixtureTask]) -> Vec bool { + fn is_assignment(word: &ShellWord) -> bool { + !word.dynamic + && word + .value + .split_once('=') + .is_some_and(|(name, _)| !name.is_empty() && !name.contains('/')) + } + + fn segment_reads(words: &[ShellWord], expected: &str, read_commands: &[String]) -> bool { + let Some(command_index) = words.iter().position(|word| !is_assignment(word)) else { + return false; + }; + let command = &words[command_index]; + if command.dynamic { + return false; + } + let Some(name) = Path::new(&command.value) + .file_name() + .and_then(|name| name.to_str()) + else { + return false; + }; + read_commands.iter().any(|allowed| allowed == name) + && words[command_index + 1..].iter().any(|word| { + !word.dynamic && crate::core::fs::normalize_separators(&word.value) == expected + }) + } + + fn literal_wrapper_script(words: &[ShellWord]) -> Option<&str> { + let command_index = words.iter().position(|word| !is_assignment(word))?; + let command = &words[command_index]; + if command.dynamic { + return None; + } + let name = Path::new(&command.value).file_name()?.to_str()?; + if !matches!(name, "sh" | "bash" | "dash" | "ksh" | "zsh") { + return None; + } + + let args = &words[command_index + 1..]; + let command_option = args.iter().position(|word| { + !word.dynamic + && word.value.starts_with('-') + && word.value != "--" + && word.value[1..].contains('c') + })?; + let script = args.get(command_option + 1)?; + (!script.dynamic).then_some(script.value.as_str()) + } + + fn scan(command: &str, expected: &str, read_commands: &[String], depth: usize) -> bool { + let lexed = lex_shell(command); + if lexed.malformed { + return false; + } + let mut segment = Vec::new(); + let mut skip_output_target = false; + for token in lexed.tokens { + match token { + ShellToken::Word(word) => { + if skip_output_target { + skip_output_target = false; + continue; + } + segment.push(word); + } + // One native exit code cannot establish which member of a + // compound command succeeded, so deterministic evidence is + // limited to a single command segment. + ShellToken::Pipe | ShellToken::Separator => return false, + ShellToken::OutputRedirect => skip_output_target = true, + ShellToken::FdDuplicate | ShellToken::InputRedirect => {} + } + } + segment_reads(&segment, expected, read_commands) + || depth < 2 + && literal_wrapper_script(&segment) + .is_some_and(|script| scan(script, expected, read_commands, depth + 1)) + } + + let expected = crate::core::fs::normalize_separators(expected); + scan(command, &expected, read_commands, 0) +} diff --git a/tests/fixtures/codex/skill-access-0.152.1.jsonl b/tests/fixtures/codex/skill-access-0.152.1.jsonl new file mode 100644 index 0000000..fc1a14d --- /dev/null +++ b/tests/fixtures/codex/skill-access-0.152.1.jsonl @@ -0,0 +1,2 @@ +{"type":"item.completed","item":{"id":"item_0","type":"command_execution","command":"/usr/bin/zsh -lc \"sed -n '1,240p' __STAGED_SKILL_PATH__\"","aggregated_output":"---\nname: synthetic-access\ndescription: synthetic Codex skill-access fixture\n---\n\n# Synthetic access probe\n","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"I used mr-review; the requested result is ready."}} diff --git a/tests/run/codex.rs b/tests/run/codex.rs index aa7cda5..55f537e 100644 --- a/tests/run/codex.rs +++ b/tests/run/codex.rs @@ -6,6 +6,8 @@ use predicates::str::contains; use std::fs; use std::path::Path; +mod skill_access; + #[test] fn codex_no_stage_keeps_inline_fallback() { let tmp = tempfile::TempDir::new().unwrap(); @@ -90,6 +92,10 @@ fn codex_stages_repo_local_skills_under_agents() { .iter() .find(|t| t["condition"] == "with_skill") .unwrap(); + assert_eq!( + task["staged_skill_path"], + serde_json::json!(wire_path(&codex_skills.join(slug).join("SKILL.md"))) + ); let prompt = read_str(Path::new(task["dispatch_prompt_path"].as_str().unwrap())); assert!(prompt.contains("## Skills")); assert!(prompt.contains(&format!("- {slug}: review merge requests"))); diff --git a/tests/run/codex/skill_access.rs b/tests/run/codex/skill_access.rs new file mode 100644 index 0000000..9893f86 --- /dev/null +++ b/tests/run/codex/skill_access.rs @@ -0,0 +1,52 @@ +use super::*; + +#[test] +fn without_exact_path_evidence_fails_locally_despite_final_message_wording() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--harness", "codex", "--dry-run"]) + .assert() + .success(); + + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + for task in dispatch["tasks"].as_array().unwrap() { + write_task_transcript( + &cwd, + task, + "codex-events.jsonl", + concat!( + r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"I invoked mr-review and followed every instruction."}}"#, + "\n" + ), + ); + } + + skill_eval() + .current_dir(&cwd) + .args(["ingest", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--harness", + "codex", + "--iteration", + "1", + ]) + .assert() + .success(); + + let response = read_json( + &iteration_dir(&cwd).join("eval-e1/with_skill/judge-responses/__skill_invoked.json"), + ); + assert_eq!(response["passed"], false, "{response}"); + assert_eq!(response["confidence"], 1.0, "{response}"); + assert_eq!(response["grader"], "transcript_check", "{response}"); + assert!(response["evidence"].as_str().unwrap().contains("access")); + let tasks = read_json(&iteration_dir(&cwd).join("judge-tasks.json")); + assert_eq!(tasks["meta_tasks_injected"], 0, "{tasks}"); +} diff --git a/tests/run/multi_skill.rs b/tests/run/multi_skill.rs index 41240d6..0702611 100644 --- a/tests/run/multi_skill.rs +++ b/tests/run/multi_skill.rs @@ -6,6 +6,8 @@ use serde_json::{Value, json}; use std::fs; use std::path::Path; +mod skill_evidence; + fn add_skill(skill_dir: &Path, name: &str) { let root = skill_dir.join(name); fs::create_dir_all(&root).unwrap(); @@ -93,7 +95,20 @@ fn conditions_and_dispatch_record_the_ordered_treatment_roster() { dispatch["skill_name"], json!(["mr-review", "supporting-skill"]) ); - assert_eq!(dispatch["tasks"][0]["skills"], *treatment); + let task_treatment = dispatch["tasks"][0]["skills"].as_array().unwrap(); + assert_eq!(task_treatment.len(), 2); + for (task_skill, condition_skill) in task_treatment.iter().zip(treatment.as_array().unwrap()) { + assert_eq!(task_skill["name"], condition_skill["name"]); + assert_eq!(task_skill["skill_path"], condition_skill["skill_path"]); + assert_eq!( + task_skill["staged_skill_slug"], + condition_skill["staged_skill_slug"] + ); + assert!(condition_skill.get("staged_skill_path").is_none()); + let staged_path = task_skill["staged_skill_path"].as_str().unwrap(); + assert!(staged_path.contains("/env-g1-with_skill/.claude/skills/")); + assert!(staged_path.ends_with("/SKILL.md")); + } assert_eq!(dispatch["tasks"][1]["skills"], json!([])); assert_eq!( dispatch["tasks"][0]["available_skills"] @@ -128,7 +143,7 @@ fn conditions_and_dispatch_record_the_ordered_treatment_roster() { } #[test] -fn a_one_member_list_still_uses_list_artifacts_and_indexed_meta_files() { +fn a_one_member_list_still_uses_list_artifacts_and_indexed_skill_evidence_files() { let tmp = tempfile::TempDir::new().unwrap(); let evals = json!({ "skill_name": ["mr-review"], @@ -198,19 +213,20 @@ fn a_one_member_list_still_uses_list_artifacts_and_indexed_meta_files() { .assert() .success(); let tasks = read_json(&iteration.join("judge-tasks.json")); - let meta = tasks["tasks"] - .as_array() - .unwrap() - .iter() - .find(|task| task["is_meta"] == true) - .unwrap(); - assert_eq!(meta["skill_name"], "mr-review"); assert!( - meta["response_path"] - .as_str() + tasks["tasks"] + .as_array() .unwrap() - .ends_with("__skill_invoked__skill-1.json") + .iter() + .all(|task| task["is_meta"] != true), + "Codex skill access is graded locally" + ); + let evidence = read_json( + &iteration.join("eval-e1/with_skill/judge-responses/__skill_invoked__skill-1.json"), ); + assert_eq!(evidence["passed"], false); + assert_eq!(evidence["grader"], "transcript_check"); + assert!(evidence["evidence"].as_str().unwrap().contains("mr-review")); skill_eval() .current_dir(&cwd) @@ -546,69 +562,6 @@ fn no_stage_inlines_every_treatment_member_without_staging_ambient_skills() { ); } -#[test] -fn codex_fallback_emits_one_named_llm_meta_task_per_treatment_member() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), &multi_evals().to_string()); - add_skill(&skill_dir, "supporting-skill"); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--harness", "codex", "--dry-run"]) - .assert() - .success(); - - let iteration = iteration_dir(&cwd); - let dispatch = read_json(&iteration.join("dispatch.json")); - for task in dispatch["tasks"].as_array().unwrap() { - let run_path = Path::new(task["run_record_path"].as_str().unwrap()); - fs::write( - run_path, - serde_json::to_vec_pretty(&json!({ - "eval_id": task["eval_id"], - "condition": task["condition"], - "skill_path": task["skill_path"], - "skills": task["skills"], - "prompt": task["user_prompt"], - "files": task["files"], - "final_message": "done", - "tool_invocations": [], - "total_tokens": null, - "duration_ms": null - })) - .unwrap(), - ) - .unwrap(); - } - skill_eval() - .current_dir(&cwd) - .args(["grade", "--skill-dir"]) - .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "codex", - "--iteration", - "1", - ]) - .assert() - .success(); - - let tasks = read_json(&iteration.join("judge-tasks.json")); - let meta = tasks["tasks"] - .as_array() - .unwrap() - .iter() - .filter(|task| task["is_meta"] == true) - .collect::>(); - assert_eq!(meta.len(), 2); - assert_eq!(meta[0]["skill_name"], "mr-review"); - assert_eq!(meta[1]["skill_name"], "supporting-skill"); - assert_ne!(meta[0]["response_path"], meta[1]["response_path"]); -} - #[test] fn live_source_detection_checks_every_treatment_member() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/tests/run/multi_skill/skill_evidence.rs b/tests/run/multi_skill/skill_evidence.rs new file mode 100644 index 0000000..5b9a638 --- /dev/null +++ b/tests/run/multi_skill/skill_evidence.rs @@ -0,0 +1,162 @@ +use super::*; + +#[test] +fn codex_exact_path_access_grades_each_treatment_member_without_meta_judges() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = prepare( + tmp.path(), + multi_evals(), + &["--mode", "new-skill", "--harness", "codex"], + ); + + let iteration = iteration_dir(&cwd); + let dispatch = read_json(&iteration.join("dispatch.json")); + for task in dispatch["tasks"].as_array().unwrap() { + let events = if task["condition"] == "with_skill" { + let staged_path = task["skills"][1]["staged_skill_path"] + .as_str() + .expect("each Codex treatment member records its exact staged path"); + include_str!("../../fixtures/codex/skill-access-0.152.1.jsonl") + .replace("__STAGED_SKILL_PATH__", staged_path) + } else { + concat!( + r#"{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"done"}}"#, + "\n" + ) + .to_string() + }; + write_task_transcript(&cwd, task, "codex-events.jsonl", &events); + } + skill_eval() + .current_dir(&cwd) + .args(["ingest", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--harness", + "codex", + "--iteration", + "1", + ]) + .assert() + .success(); + + let tasks = read_json(&iteration.join("judge-tasks.json")); + let meta = tasks["tasks"] + .as_array() + .unwrap() + .iter() + .filter(|task| task["is_meta"] == true) + .collect::>(); + assert!( + meta.is_empty(), + "Codex exact-path evidence is graded locally" + ); + + let responses = iteration.join("eval-e1/with_skill/judge-responses"); + let first = read_json(&responses.join("__skill_invoked__skill-1.json")); + let second = read_json(&responses.join("__skill_invoked__skill-2.json")); + assert_eq!( + first["passed"], false, + "final-message wording is not evidence" + ); + assert_eq!(second["passed"], true, "the exact staged path was read"); + for response in [&first, &second] { + assert_eq!(response["grader"], "transcript_check"); + assert_eq!(response["confidence"], 1.0); + assert!(response["evidence"].as_str().unwrap().contains("access")); + } +} + +#[test] +fn descriptor_without_deterministic_evidence_retains_one_labeled_fallback_per_member() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), &multi_evals().to_string()); + add_skill(&skill_dir, "supporting-skill"); + let descriptor_dir = cwd.join(".eval-magic/harnesses"); + fs::create_dir_all(&descriptor_dir).unwrap(); + fs::write( + descriptor_dir.join("fallback.toml"), + r#"label = "fallback" +skills_dir = ".agents/skills" +config_dirs = [".agents"] + +[tools] +write = ["file_change"] +shell = ["command_execution"] + +[transcript] +events_filename = "fallback-events.jsonl" +parser = "codex-items" +surfaces_skill_invocation = false + +[dispatch] +exec_template = "fallback --cd > /fallback-events.jsonl" +"#, + ) + .unwrap(); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--harness", "fallback", "--dry-run"]) + .assert() + .success(); + + let iteration = iteration_dir(&cwd); + let dispatch = read_json(&iteration.join("dispatch.json")); + for task in dispatch["tasks"].as_array().unwrap() { + let run_path = Path::new(task["run_record_path"].as_str().unwrap()); + fs::write( + run_path, + serde_json::to_vec_pretty(&json!({ + "eval_id": task["eval_id"], + "condition": task["condition"], + "skill_path": task["skill_path"], + "skills": [], + "prompt": task["user_prompt"], + "files": task["files"], + "final_message": "done", + "tool_invocations": [], + "total_tokens": null, + "duration_ms": null + })) + .unwrap(), + ) + .unwrap(); + } + skill_eval() + .current_dir(&cwd) + .args(["grade", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--harness", + "fallback", + "--iteration", + "1", + ]) + .assert() + .success(); + + let tasks = read_json(&iteration.join("judge-tasks.json")); + let meta = tasks["tasks"] + .as_array() + .unwrap() + .iter() + .filter(|task| task["is_meta"] == true) + .collect::>(); + assert_eq!(meta.len(), 2, "{tasks}"); + assert_eq!(meta[0]["skill_name"], "mr-review"); + assert_eq!(meta[1]["skill_name"], "supporting-skill"); + for task in meta { + let rubric = task["rubric"].as_str().unwrap(); + assert!(rubric.contains("behavioral-influence fallback"), "{rubric}"); + assert!( + rubric.contains("does not prove native invocation or access"), + "{rubric}" + ); + } +}