From 906e52012eed3778f084acc83a601022fe6fb468 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:36:33 +0800 Subject: [PATCH 1/2] refactor(validation): separate project plan and evidence authority --- .../src/webcodex_runner/validation/project.rs | 13 +- .../validation/project_tests.rs | 141 ++++++++++++-- crates/webcodex-validation/src/adapters/go.rs | 135 ++++++------- .../webcodex-validation/src/adapters/mod.rs | 42 +++-- .../src/adapters/python.rs | 55 +++--- .../webcodex-validation/src/adapters/rust.rs | 83 ++++---- crates/webcodex-validation/src/evidence.rs | 37 ++-- crates/webcodex-validation/src/lib.rs | 7 +- .../webcodex-validation/src/profile_tests.rs | 55 ++++++ src/tool_runtime/jobs.rs | 85 +++++---- src/tool_runtime/tests/jobs.rs | 177 ++++++++++++++++++ .../validation_handoff/project_validation.rs | 65 +++++++ src/tool_runtime/validation.rs | 32 ++-- src/tool_runtime/validation_profile/mod.rs | 5 +- 14 files changed, 692 insertions(+), 240 deletions(-) diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project.rs index adc666c89..913e1e28e 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project.rs @@ -59,11 +59,13 @@ pub(crate) fn plan( .test .as_ref() .and_then(|test| test.filter.as_deref()); + // The operation supplies semantic identity and normalization only. The + // resolved recipe below is the sole authority for the executable step. let operation = project_validation_operation(backend.as_str(), action, packages, all_packages) .and_then(|operation| operation.with_dependency_policy(request.dependency_policy)) .and_then(|operation| operation.with_test_filter(filter)) .map_err(|code| unavailable(code, Some(backend.as_str())))?; - let adapter = operation.adapter(); + let profile = operation.compatibility_profile(); let resolved = resolve_project_validation_recipe( &root, request.cwd.as_deref(), @@ -90,13 +92,6 @@ pub(crate) fn plan( } else { identity }; - debug_assert_eq!( - operation - .build_readonly_plan() - .ok() - .map(|plan| plan.structured_step), - resolved.steps.first().cloned() - ); let cwd = root.join(&resolved.recipe_root_relative); Ok(( ProjectValidationPlan { @@ -108,7 +103,7 @@ pub(crate) fn plan( manifest_digest: resolved.manifest_digest, invocation_digest: resolved.invocation_digest, }, - adapter: adapter.tool_identity().into(), + adapter: profile.tool_identity.into(), step: resolved .steps .into_iter() diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs index 5ec75f86b..d0ee2df90 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs @@ -90,6 +90,12 @@ fn project_validation_runner_resolves_all_production_actions() { ("Cargo.toml", Test, "cargo_test", vec!["test"]), ("go.mod", Check, "go_vet", vec!["vet", "./..."]), ("go.mod", Test, "go_test", vec!["test", "-json", "./..."]), + ( + "pyproject.toml", + Test, + "python:pytest:test", + vec!["-m", "pytest", "--color=no", "-rA"], + ), ] { let (_tmp, root, registry, policy) = fixture(marker); let (plan, cwd) = project::plan(&policy, ®istry, &request(action)).unwrap(); @@ -102,31 +108,25 @@ fn project_validation_runner_resolves_all_production_actions() { Check => webcodex_validation::SemanticCheck::Check, Test => webcodex_validation::SemanticCheck::Test, }; - let canonical_adapter = webcodex_validation::validation_adapter_for_recipe( - plan.provenance.backend.as_str(), - semantic_check, + let resolved = webcodex_validation::resolve_project_validation_recipe( + &root, + None, + None, + &[semantic_check], + None, + None, + false, + None, ) .unwrap(); - let canonical_options = if plan.provenance.backend == "rust" && action == FormatCheck { - webcodex_validation::ValidationCommandOptions { - check: true, - ..Default::default() - } - } else { - webcodex_validation::ValidationCommandOptions::default() - }; - let canonical_plan = canonical_adapter - .build_readonly_plan(canonical_options) - .unwrap(); assert_eq!( - plan.step, canonical_plan.structured_step, - "{adapter} gateway plan must use the canonical adapter step" + std::slice::from_ref(&plan.step), + resolved.steps.as_slice(), + "{adapter} gateway plan must use the resolved recipe step" ); let invocation_digest = format!( "{:x}", - sha2::Sha256::digest( - serde_json::to_vec(&vec![canonical_plan.structured_step]).unwrap() - ) + sha2::Sha256::digest(serde_json::to_vec(&resolved.steps).unwrap()) ); assert_eq!( plan.provenance.invocation_digest, invocation_digest, @@ -138,6 +138,109 @@ fn project_validation_runner_resolves_all_production_actions() { .contains(root.to_str().unwrap())); } } +#[test] +fn project_validation_recipe_step_and_identity_preserve_normalized_selection() { + for (marker, backend, tool, filter, packages) in [ + ( + "Cargo.toml", + "rust", + "cargo_test", + "selected", + vec!["b", "a", "a"], + ), + ( + "go.mod", + "go", + "go_test", + " ^TestA/sub$ ", + vec!["./b", "./a", "./a"], + ), + ( + "pyproject.toml", + "python", + "python:pytest:test", + " selected and not slow ", + vec![], + ), + ] { + let (_tmp, root, registry, policy) = fixture(marker); + let mut req = request(ProjectValidationAction::Test); + req.test = Some(ProjectValidationTestOptions { + filter: Some(if backend == "rust" { + format!(" {filter} ") + } else { + filter.into() + }), + ..Default::default() + }); + if !packages.is_empty() { + req.scope = Some(ProjectValidationScope { + packages: packages.into_iter().map(str::to_string).collect(), + all_packages: false, + }); + req.dependency_policy = Some(ProjectDependencyPolicy { + mode: ProjectDependencyMode::Locked, + }); + } + let (plan, _) = project::plan(&policy, ®istry, &req).unwrap(); + let resolved = webcodex_validation::resolve_project_validation_recipe( + &root, + None, + None, + &[webcodex_validation::SemanticCheck::Test], + req.test.as_ref().unwrap().filter.as_deref(), + req.scope + .as_ref() + .and_then(ProjectValidationScope::explicit_packages), + false, + req.dependency_policy, + ) + .unwrap(); + assert_eq!(std::slice::from_ref(&plan.step), resolved.steps.as_slice()); + assert_eq!(plan.adapter, tool); + assert_eq!( + plan.provenance.invocation_digest, + resolved.invocation_digest + ); + + req.test.as_mut().unwrap().filter = Some(filter.into()); + if backend == "rust" { + let scope = req.scope.as_mut().unwrap(); + scope.packages.sort(); + scope.packages.dedup(); + } + let (normalized, _) = project::plan(&policy, ®istry, &req).unwrap(); + assert_eq!(plan.step, normalized.step); + assert_eq!(plan.validation_target_id, normalized.validation_target_id); + let operation = webcodex_validation::project_validation_operation( + backend, + webcodex_validation::SemanticCheck::Test, + req.scope + .as_ref() + .and_then(ProjectValidationScope::explicit_packages) + .map(<[String]>::to_vec), + false, + ) + .unwrap() + .with_dependency_policy(req.dependency_policy) + .unwrap() + .with_test_filter(Some(filter)) + .unwrap(); + let identity = operation + .validation_target_id(Some(&resolved.recipe_root_relative)) + .unwrap(); + let expected = if backend == "go" { + webcodex_core::validation_identity::contextualize_structured_validation_target_identity( + &identity, + webcodex_core::validation_identity::StructuredValidationExecutionContext::GoProjectSingleModuleV1, + ).unwrap() + } else { + identity + }; + assert_eq!(plan.validation_target_id, expected); + } +} + #[test] fn go_project_validation_identity_is_single_module_domain_separated() { for action in [ diff --git a/crates/webcodex-validation/src/adapters/go.rs b/crates/webcodex-validation/src/adapters/go.rs index a3853540e..6960d7700 100644 --- a/crates/webcodex-validation/src/adapters/go.rs +++ b/crates/webcodex-validation/src/adapters/go.rs @@ -1,6 +1,7 @@ use super::{ read_only_validation_plan, validate_package_scope_exclusivity, ReadOnlyValidationPlan, - ValidationAdapter, ValidationCommandOptions, ValidationFailureEvidence, ValidationPlanArg, + ValidationAdapter, ValidationCommandOptions, ValidationEvidenceProfile, + ValidationFailureEvidence, ValidationPlanArg, }; use webcodex_core::runner_protocol::normalize_go_test_packages; use webcodex_core::validation_evidence::{parse_go_test_diagnostics, ValidationDiagnostics}; @@ -25,7 +26,7 @@ pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { &GO_TEST_ADAPTER } -impl ValidationAdapter for GoTestValidationAdapter { +impl ValidationEvidenceProfile for GoTestValidationAdapter { fn validation_kind(&self) -> &'static str { "test" } @@ -34,54 +35,6 @@ impl ValidationAdapter for GoTestValidationAdapter { "go_test" } - fn build_readonly_plan( - &self, - options: ValidationCommandOptions, - ) -> Result { - if options.check - || options.lib.is_some() - || options.all_targets.is_some() - || options.all_features.is_some() - || options.no_default_features.is_some() - || options.features.is_some() - || options.package.is_some() - || options.cargo_packages.is_some() - || options.no_run.is_some() - { - return Err("go_test does not accept Cargo validation command options".to_string()); - } - let explicit_packages = options.go_packages.is_some(); - validate_package_scope_exclusivity(explicit_packages, options.all_packages)?; - let packages = normalize_go_test_packages(options.go_packages.as_deref()) - .map_err(|reason| format!("packages {reason}"))?; - let mut args = vec![ - ValidationPlanArg::Literal("test"), - ValidationPlanArg::Literal("-json"), - ]; - if options.dependency_mode - == Some(webcodex_core::project_validation::ProjectDependencyMode::Locked) - { - args.push(ValidationPlanArg::Literal("-mod=readonly")); - } - if let Some(filter) = options - .filter - .as_deref() - .map(webcodex_core::runner_protocol::normalize_go_test_filter) - .transpose()? - .flatten() - { - args.push(ValidationPlanArg::Literal("-run")); - args.push(ValidationPlanArg::Value(filter)); - } - if explicit_packages && !options.all_packages { - args.extend(packages.into_iter().map(ValidationPlanArg::Value)); - } else { - debug_assert_eq!(packages.as_slice(), ["./..."]); - args.push(ValidationPlanArg::Literal("./...")); - } - read_only_validation_plan("test", "go", args) - } - fn parse( &self, stdout_excerpt: &str, @@ -132,21 +85,12 @@ impl ValidationAdapter for GoTestValidationAdapter { } } -struct GoVetValidationAdapter; -static GO_VET_ADAPTER: GoVetValidationAdapter = GoVetValidationAdapter; -impl ValidationAdapter for GoVetValidationAdapter { - fn validation_kind(&self) -> &'static str { - "check" - } - fn tool_identity(&self) -> &'static str { - "go_vet" - } +impl ValidationAdapter for GoTestValidationAdapter { fn build_readonly_plan( &self, options: ValidationCommandOptions, ) -> Result { if options.check - || options.filter.is_some() || options.lib.is_some() || options.all_targets.is_some() || options.all_features.is_some() @@ -156,29 +100,56 @@ impl ValidationAdapter for GoVetValidationAdapter { || options.cargo_packages.is_some() || options.no_run.is_some() { - return Err("go_vet does not accept Cargo validation command options".to_string()); + return Err("go_test does not accept Cargo validation command options".to_string()); } let explicit_packages = options.go_packages.is_some(); validate_package_scope_exclusivity(explicit_packages, options.all_packages)?; let packages = normalize_go_test_packages(options.go_packages.as_deref()) .map_err(|reason| format!("packages {reason}"))?; - let mut args = vec![ValidationPlanArg::Literal("vet")]; + let mut args = vec![ + ValidationPlanArg::Literal("test"), + ValidationPlanArg::Literal("-json"), + ]; if options.dependency_mode == Some(webcodex_core::project_validation::ProjectDependencyMode::Locked) { args.push(ValidationPlanArg::Literal("-mod=readonly")); } + if let Some(filter) = options + .filter + .as_deref() + .map(webcodex_core::runner_protocol::normalize_go_test_filter) + .transpose()? + .flatten() + { + args.push(ValidationPlanArg::Literal("-run")); + args.push(ValidationPlanArg::Value(filter)); + } if explicit_packages && !options.all_packages { args.extend(packages.into_iter().map(ValidationPlanArg::Value)); } else { debug_assert_eq!(packages.as_slice(), ["./..."]); args.push(ValidationPlanArg::Literal("./...")); } - read_only_validation_plan("check", "go", args) + read_only_validation_plan("test", "go", args) } +} + +struct GoVetValidationAdapter; +static GO_VET_ADAPTER: GoVetValidationAdapter = GoVetValidationAdapter; +impl ValidationEvidenceProfile for GoVetValidationAdapter { + fn validation_kind(&self) -> &'static str { + "check" + } + + fn tool_identity(&self) -> &'static str { + "go_vet" + } + fn parse(&self, _stdout: &str, stderr: &str, truncated: bool) -> ValidationDiagnostics { webcodex_core::validation_evidence::parse_go_vet_diagnostics(stderr, truncated) } + fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { if evidence.success { "unknown" @@ -192,3 +163,41 @@ impl ValidationAdapter for GoVetValidationAdapter { } } } + +impl ValidationAdapter for GoVetValidationAdapter { + fn build_readonly_plan( + &self, + options: ValidationCommandOptions, + ) -> Result { + if options.check + || options.filter.is_some() + || options.lib.is_some() + || options.all_targets.is_some() + || options.all_features.is_some() + || options.no_default_features.is_some() + || options.features.is_some() + || options.package.is_some() + || options.cargo_packages.is_some() + || options.no_run.is_some() + { + return Err("go_vet does not accept Cargo validation command options".to_string()); + } + let explicit_packages = options.go_packages.is_some(); + validate_package_scope_exclusivity(explicit_packages, options.all_packages)?; + let packages = normalize_go_test_packages(options.go_packages.as_deref()) + .map_err(|reason| format!("packages {reason}"))?; + let mut args = vec![ValidationPlanArg::Literal("vet")]; + if options.dependency_mode + == Some(webcodex_core::project_validation::ProjectDependencyMode::Locked) + { + args.push(ValidationPlanArg::Literal("-mod=readonly")); + } + if explicit_packages && !options.all_packages { + args.extend(packages.into_iter().map(ValidationPlanArg::Value)); + } else { + debug_assert_eq!(packages.as_slice(), ["./..."]); + args.push(ValidationPlanArg::Literal("./...")); + } + read_only_validation_plan("check", "go", args) + } +} diff --git a/crates/webcodex-validation/src/adapters/mod.rs b/crates/webcodex-validation/src/adapters/mod.rs index aa92da28b..d4922874a 100644 --- a/crates/webcodex-validation/src/adapters/mod.rs +++ b/crates/webcodex-validation/src/adapters/mod.rs @@ -508,21 +508,12 @@ pub struct ValidationFailureEvidence<'a> { pub stderr_excerpt: &'a str, } -pub trait ValidationAdapter: Sync { +/// Result interpretation without command-building authority. +pub trait ValidationEvidenceProfile: Sync { fn validation_kind(&self) -> &'static str; fn tool_identity(&self) -> &'static str; - fn build_readonly_plan( - &self, - options: ValidationCommandOptions, - ) -> Result; - - fn build_command(&self, options: ValidationCommandOptions) -> Result { - self.build_readonly_plan(options) - .map(|plan| plan.compatibility_command) - } - fn parse( &self, stdout_excerpt: &str, @@ -537,6 +528,18 @@ pub trait ValidationAdapter: Sync { } } +pub trait ValidationAdapter: ValidationEvidenceProfile { + fn build_readonly_plan( + &self, + options: ValidationCommandOptions, + ) -> Result; + + fn build_command(&self, options: ValidationCommandOptions) -> Result { + self.build_readonly_plan(options) + .map(|plan| plan.compatibility_command) + } +} + /// Canonical mapping from structured validation classification to execution /// evidence intent. Tool selection determines this purpose; callers do not. pub fn execution_purpose_for_validation_kind(validation_kind: &str) -> ExecutionPurpose { @@ -572,3 +575,20 @@ pub fn validation_adapter_for_recipe( _ => return None, }) } + +/// Evidence-only view of the canonical adapter registry. +pub fn validation_evidence_profile_for_tool( + tool_identity: &str, +) -> Option<&'static dyn ValidationEvidenceProfile> { + validation_adapter_for_tool(tool_identity) + .map(|adapter| adapter as &dyn ValidationEvidenceProfile) +} + +/// Evidence-only view of the canonical backend/action mapping. +pub fn validation_evidence_profile_for_recipe( + backend: &str, + action: crate::SemanticCheck, +) -> Option<&'static dyn ValidationEvidenceProfile> { + validation_adapter_for_recipe(backend, action) + .map(|adapter| adapter as &dyn ValidationEvidenceProfile) +} diff --git a/crates/webcodex-validation/src/adapters/python.rs b/crates/webcodex-validation/src/adapters/python.rs index 00b8da12a..93d405d74 100644 --- a/crates/webcodex-validation/src/adapters/python.rs +++ b/crates/webcodex-validation/src/adapters/python.rs @@ -1,6 +1,6 @@ use super::{ read_only_validation_plan, ReadOnlyValidationPlan, ValidationAdapter, ValidationCommandOptions, - ValidationFailureEvidence, ValidationPlanArg, + ValidationEvidenceProfile, ValidationFailureEvidence, ValidationPlanArg, }; use webcodex_core::validation_evidence::{parse_pytest_diagnostics, ValidationDiagnostics}; @@ -10,13 +10,42 @@ pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { &PYTEST_ADAPTER } -impl ValidationAdapter for PytestValidationAdapter { +impl ValidationEvidenceProfile for PytestValidationAdapter { fn validation_kind(&self) -> &'static str { "test" } + fn tool_identity(&self) -> &'static str { "python:pytest:test" } + + fn parse(&self, stdout: &str, _stderr: &str, truncated: bool) -> ValidationDiagnostics { + parse_pytest_diagnostics(stdout, truncated) + } + + fn reports_test_run_metadata(&self) -> bool { + true + } + + fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { + if evidence.success { + return "unknown"; + } + if matches!( + evidence.reported_failure_kind, + Some("timeout" | "timed_out" | "command_timeout") + ) { + return "timeout"; + } + if evidence.exit_code == Some(1) { + "test_failure" + } else { + "process_exit" + } + } +} + +impl ValidationAdapter for PytestValidationAdapter { fn build_readonly_plan( &self, options: ValidationCommandOptions, @@ -52,26 +81,4 @@ impl ValidationAdapter for PytestValidationAdapter { } read_only_validation_plan("test", "python", args) } - fn parse(&self, stdout: &str, _stderr: &str, truncated: bool) -> ValidationDiagnostics { - parse_pytest_diagnostics(stdout, truncated) - } - fn reports_test_run_metadata(&self) -> bool { - true - } - fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { - if evidence.success { - return "unknown"; - } - if matches!( - evidence.reported_failure_kind, - Some("timeout" | "timed_out" | "command_timeout") - ) { - return "timeout"; - } - if evidence.exit_code == Some(1) { - "test_failure" - } else { - "process_exit" - } - } } diff --git a/crates/webcodex-validation/src/adapters/rust.rs b/crates/webcodex-validation/src/adapters/rust.rs index 5dc885833..aa1bbf11f 100644 --- a/crates/webcodex-validation/src/adapters/rust.rs +++ b/crates/webcodex-validation/src/adapters/rust.rs @@ -1,6 +1,7 @@ use super::{ read_only_validation_plan, validate_package_scope_exclusivity, ReadOnlyValidationPlan, - ValidationAdapter, ValidationCommandOptions, ValidationFailureEvidence, ValidationPlanArg, + ValidationAdapter, ValidationCommandOptions, ValidationEvidenceProfile, + ValidationFailureEvidence, ValidationPlanArg, }; use webcodex_core::runner_protocol::{ normalize_cargo_packages, normalize_cargo_value, normalize_rust_test_filter, @@ -55,7 +56,7 @@ pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { &CARGO_TEST_ADAPTER } -impl ValidationAdapter for RustValidationAdapter { +impl ValidationEvidenceProfile for RustValidationAdapter { fn validation_kind(&self) -> &'static str { match self.kind { RustAdapterKind::Format => "format", @@ -68,44 +69,6 @@ impl ValidationAdapter for RustValidationAdapter { self.tool_identity } - fn build_readonly_plan( - &self, - options: ValidationCommandOptions, - ) -> Result { - if options.go_packages.is_some() { - return Err("Cargo validation does not accept go_test packages".to_string()); - } - match self.kind { - RustAdapterKind::Format => { - if !options.check { - return Err("cargo_fmt mutation is not read-only validation".to_string()); - } - read_only_validation_plan( - "format", - "cargo", - vec![ - ValidationPlanArg::Literal("fmt"), - ValidationPlanArg::Literal("--"), - ValidationPlanArg::Literal("--check"), - ], - ) - } - RustAdapterKind::Check => cargo_check_plan(options), - RustAdapterKind::Test => cargo_test_plan(options), - } - } - - fn build_command(&self, options: ValidationCommandOptions) -> Result { - if self.kind == RustAdapterKind::Format && !options.check { - if options.go_packages.is_some() { - return Err("Cargo validation does not accept go_test packages".to_string()); - } - return Ok("cargo fmt".to_string()); - } - self.build_readonly_plan(options) - .map(|plan| plan.compatibility_command) - } - fn parse( &self, stdout_excerpt: &str, @@ -189,6 +152,46 @@ impl ValidationAdapter for RustValidationAdapter { } } +impl ValidationAdapter for RustValidationAdapter { + fn build_readonly_plan( + &self, + options: ValidationCommandOptions, + ) -> Result { + if options.go_packages.is_some() { + return Err("Cargo validation does not accept go_test packages".to_string()); + } + match self.kind { + RustAdapterKind::Format => { + if !options.check { + return Err("cargo_fmt mutation is not read-only validation".to_string()); + } + read_only_validation_plan( + "format", + "cargo", + vec![ + ValidationPlanArg::Literal("fmt"), + ValidationPlanArg::Literal("--"), + ValidationPlanArg::Literal("--check"), + ], + ) + } + RustAdapterKind::Check => cargo_check_plan(options), + RustAdapterKind::Test => cargo_test_plan(options), + } + } + + fn build_command(&self, options: ValidationCommandOptions) -> Result { + if self.kind == RustAdapterKind::Format && !options.check { + if options.go_packages.is_some() { + return Err("Cargo validation does not accept go_test packages".to_string()); + } + return Ok("cargo fmt".to_string()); + } + self.build_readonly_plan(options) + .map(|plan| plan.compatibility_command) + } +} + fn cargo_check_plan(options: ValidationCommandOptions) -> Result { validate_package_scope_exclusivity( options.package.is_some() || options.cargo_packages.is_some(), diff --git a/crates/webcodex-validation/src/evidence.rs b/crates/webcodex-validation/src/evidence.rs index 7579e1248..e4330c339 100644 --- a/crates/webcodex-validation/src/evidence.rs +++ b/crates/webcodex-validation/src/evidence.rs @@ -188,8 +188,8 @@ pub struct CurrentValidationEvidenceProjection { } use crate::adapters::{ - execution_purpose_for_validation_kind, validation_adapter_for_tool, ValidationAdapter, - ValidationFailureEvidence, + execution_purpose_for_validation_kind, validation_evidence_profile_for_tool, + ValidationEvidenceProfile, ValidationFailureEvidence, }; pub fn skipped_validation_summary() -> Value { @@ -751,7 +751,7 @@ fn validation_event_decides_historical_failure_status(event: &ValidationEvent) - // `validation_kind = test` is only an intent/category for generic execution. // Only an actual first-class structured test tool opts into count/zero-test proof. fn structured_test_requires_execution_proof(event: &ValidationEvent) -> bool { - validation_adapter_for_tool(&event.tool_name).is_some_and(|adapter| { + validation_evidence_profile_for_tool(&event.tool_name).is_some_and(|adapter| { adapter.validation_kind() == "test" && adapter.reports_test_run_metadata() }) } @@ -1005,7 +1005,7 @@ pub fn event_is_job_acceptance_only(event: &SessionEvent) -> bool { } pub fn validation_kind_for_tool(tool_name: &str) -> Option<&'static str> { - if let Some(adapter) = validation_adapter_for_tool(tool_name) { + if let Some(adapter) = validation_evidence_profile_for_tool(tool_name) { return Some(adapter.validation_kind()); } None @@ -1070,7 +1070,7 @@ fn validation_event_from_finished( finished: &SessionEvent, started: Option<&SessionEvent>, ) -> Option { - let adapter = validation_adapter_for_execution(finished, started); + let adapter = validation_evidence_profile_for_execution(finished, started); let purpose = started .and_then(execution_purpose) .or_else(|| execution_purpose(finished))?; @@ -1278,18 +1278,18 @@ fn validation_event_failure_class(event: &ValidationEvent) -> &'static str { } } -fn validation_adapter_for_execution( +fn validation_evidence_profile_for_execution( finished: &SessionEvent, started: Option<&SessionEvent>, -) -> Option<&'static dyn ValidationAdapter> { - validation_adapter_for_tool(&finished.tool_name) +) -> Option<&'static dyn ValidationEvidenceProfile> { + validation_evidence_profile_for_tool(&finished.tool_name) .or_else(|| { finished .validation_output_summary .as_ref() .and_then(|v| v.get("adapter")) .and_then(Value::as_str) - .and_then(validation_adapter_for_tool) + .and_then(validation_evidence_profile_for_tool) }) .or_else(|| { started @@ -1310,7 +1310,7 @@ fn validation_adapter_for_execution( .and_then(|summary| summary.get("validation_tool")) .and_then(Value::as_str) }) - .and_then(validation_adapter_for_tool) + .and_then(validation_evidence_profile_for_tool) }) .or_else(|| { let command = started @@ -1330,9 +1330,11 @@ fn validation_adapter_for_execution( })?; let mut words = command.split_whitespace(); match (words.next(), words.next()) { - (Some("cargo"), Some("fmt")) => validation_adapter_for_tool("cargo_fmt"), - (Some("cargo"), Some("check")) => validation_adapter_for_tool("cargo_check"), - (Some("cargo"), Some("test")) => validation_adapter_for_tool("cargo_test"), + (Some("cargo"), Some("fmt")) => validation_evidence_profile_for_tool("cargo_fmt"), + (Some("cargo"), Some("check")) => { + validation_evidence_profile_for_tool("cargo_check") + } + (Some("cargo"), Some("test")) => validation_evidence_profile_for_tool("cargo_test"), _ => None, } }) @@ -1505,7 +1507,7 @@ fn validation_failure_kind( finished: &SessionEvent, success: bool, diagnostics: Option<&ValidationDiagnostics>, - adapter: Option<&dyn ValidationAdapter>, + adapter: Option<&dyn ValidationEvidenceProfile>, ) -> &'static str { if let Some(adapter) = adapter { let (stdout_excerpt, stderr_excerpt, _) = @@ -1614,7 +1616,7 @@ fn parser_summary_for_events(events: &[ValidationEvent]) -> ValidationParserSumm fn validation_diagnostics_from_summary( finished: &SessionEvent, - adapter: &dyn ValidationAdapter, + adapter: &dyn ValidationEvidenceProfile, ) -> Option { let (stdout_excerpt, stderr_excerpt, truncated) = validation_output_excerpts(finished)?; Some(adapter.parse(stdout_excerpt, stderr_excerpt, truncated)) @@ -1643,7 +1645,7 @@ fn validation_output_excerpts(finished: &SessionEvent) -> Option<(&str, &str, bo fn validation_test_run_metadata( finished: &SessionEvent, - adapter: Option<&dyn ValidationAdapter>, + adapter: Option<&dyn ValidationEvidenceProfile>, diagnostics: Option<&ValidationDiagnostics>, ) -> ( Option, @@ -1664,7 +1666,8 @@ fn validation_test_run_metadata( .into_iter() .any(|field| value.get(field).is_some()) }); - if !adapter.is_some_and(ValidationAdapter::reports_test_run_metadata) && !explicit_test_metadata + if !adapter.is_some_and(ValidationEvidenceProfile::reports_test_run_metadata) + && !explicit_test_metadata { return (None, None, None, None, None); } diff --git a/crates/webcodex-validation/src/lib.rs b/crates/webcodex-validation/src/lib.rs index 23f20b022..2a67d66b6 100644 --- a/crates/webcodex-validation/src/lib.rs +++ b/crates/webcodex-validation/src/lib.rs @@ -22,11 +22,12 @@ mod recipe_tests; pub use adapters::{ execution_purpose_for_validation_kind, project_validation_operation, - validation_adapter_for_recipe, validation_adapter_for_tool, CargoCheckOptions, - CargoReadOnlyValidationOperation, CargoTestOptions, GoCheckOptions, + validation_adapter_for_recipe, validation_adapter_for_tool, + validation_evidence_profile_for_recipe, validation_evidence_profile_for_tool, + CargoCheckOptions, CargoReadOnlyValidationOperation, CargoTestOptions, GoCheckOptions, GoReadOnlyValidationOperation, GoTestOptions, PythonTestOptions, ReadOnlyValidationOperation, ReadOnlyValidationPlan, ValidationAdapter, ValidationCommandOptions, - ValidationCompatibilityProfile, ValidationFailureEvidence, + ValidationCompatibilityProfile, ValidationEvidenceProfile, ValidationFailureEvidence, }; #[cfg(any(feature = "session-evidence", test))] pub use evidence::{ diff --git a/crates/webcodex-validation/src/profile_tests.rs b/crates/webcodex-validation/src/profile_tests.rs index f66d25f0a..32cb354e9 100644 --- a/crates/webcodex-validation/src/profile_tests.rs +++ b/crates/webcodex-validation/src/profile_tests.rs @@ -890,3 +890,58 @@ fn project_validation_python_pytest_owns_filter_plan_and_parser() { assert!(!is_known_tool_name("pytest")); assert!(!is_known_tool_name("python:pytest:test")); } + +#[test] +fn evidence_profiles_share_direct_adapter_registry_and_recipe_mapping() { + use crate::{validation_evidence_profile_for_recipe, validation_evidence_profile_for_tool}; + for (backend, action, tool) in [ + ("rust", SemanticCheck::Format, "cargo_fmt"), + ("rust", SemanticCheck::Check, "cargo_check"), + ("rust", SemanticCheck::Test, "cargo_test"), + ("go", SemanticCheck::Check, "go_vet"), + ("go", SemanticCheck::Test, "go_test"), + ("python", SemanticCheck::Test, "python:pytest:test"), + ] { + let adapter = validation_adapter_for_tool(tool).unwrap(); + let profile = validation_evidence_profile_for_tool(tool).unwrap(); + let recipe = validation_evidence_profile_for_recipe(backend, action).unwrap(); + assert_eq!(profile.tool_identity(), adapter.tool_identity()); + assert_eq!(recipe.tool_identity(), tool); + assert_eq!(profile.validation_kind(), adapter.validation_kind()); + assert_eq!( + profile.reports_test_run_metadata(), + adapter.reports_test_run_metadata() + ); + for truncated in [false, true] { + let stdout = "test result: ok. 2 passed; 0 failed; 0 ignored\n2 passed in 0.01s\n"; + let stderr = "error: example failure"; + let diagnostics = profile.parse(stdout, stderr, truncated); + assert_eq!( + serde_json::to_value(&diagnostics).unwrap(), + serde_json::to_value(adapter.parse(stdout, stderr, truncated)).unwrap() + ); + for success in [false, true] { + let evidence = || ValidationFailureEvidence { + success, + reported_failure_kind: None, + exit_code: Some(if success { 0 } else { 1 }), + diagnostics: Some(&diagnostics), + stdout_excerpt: stdout, + stderr_excerpt: stderr, + }; + assert_eq!( + profile.map_failure_kind(evidence()), + adapter.map_failure_kind(evidence()) + ); + } + } + assert!(adapter + .build_readonly_plan(ValidationCommandOptions { + check: tool == "cargo_fmt", + ..Default::default() + }) + .is_ok()); + } + assert!(validation_evidence_profile_for_tool("unknown_test").is_none()); + assert!(validation_evidence_profile_for_recipe("go", SemanticCheck::Format).is_none()); +} diff --git a/src/tool_runtime/jobs.rs b/src/tool_runtime/jobs.rs index 35127975f..bdbdb7469 100644 --- a/src/tool_runtime/jobs.rs +++ b/src/tool_runtime/jobs.rs @@ -427,10 +427,11 @@ pub(crate) fn structured_validation_evidence( truncated: bool, ) -> StructuredValidationEvidence { let combined = format!("{stdout}\n{stderr}"); - let diagnostics = matches!(kind, "check" | "test") - .then(|| super::validation_profile::validation_adapter_for_tool(tool)) + let profile = matches!(kind, "check" | "test") + .then(|| super::validation_profile::validation_evidence_profile_for_tool(tool)) .flatten() - .map(|adapter| adapter.parse(stdout, stderr, truncated)); + .filter(|adapter| adapter.validation_kind() == kind); + let diagnostics = profile.map(|adapter| adapter.parse(stdout, stderr, truncated)); let mut evidence = StructuredValidationEvidence { diagnostics, tests_detected: None, @@ -442,8 +443,8 @@ pub(crate) fn structured_validation_evidence( warnings_count: None, errors_count: None, }; - match kind { - "test" if tool == "python:pytest:test" => { + match profile.map(|adapter| adapter.tool_identity()) { + Some("python:pytest:test") => { let summary = evidence .diagnostics .as_ref() @@ -463,7 +464,7 @@ pub(crate) fn structured_validation_evidence( evidence.zero_tests_run = evidence.tests_run_count.map(|count| count == 0); } } - "test" if tool == "go_test" => { + Some("go_test") => { let test_summary = evidence .diagnostics .as_ref() @@ -482,7 +483,7 @@ pub(crate) fn structured_validation_evidence( evidence.zero_tests_run = evidence.tests_run_count.map(|count| count == 0); } } - "test" => { + Some("cargo_test") => { let metadata = super::cargo::parse_cargo_test_run_metadata(&combined); evidence.tests_detected = Some(metadata.tests_detected); evidence.test_count_evidence_reason = Some(if truncated { @@ -497,7 +498,9 @@ pub(crate) fn structured_validation_evidence( evidence.zero_tests_run = metadata.zero_tests_run; } } - "check" if !truncated => { + // Preserve the existing check counter projection for known Cargo/Go + // adapters; unknown check identities must not inherit Rustc parsing. + Some("cargo_check" | "go_vet") if !truncated => { evidence.warnings_count = Some(super::cargo::count_rustc_diagnostics(&combined, "warning:") as u64); evidence.errors_count = @@ -548,11 +551,8 @@ pub(crate) fn validation_job_projection_with_policy( no_run: Option, ) -> Option { let tool = tool?; - let kind = kind.unwrap_or(match tool { - "cargo_test" | "go_test" | "python:pytest:test" => "test", - "cargo_fmt" => "format", - _ => "check", - }); + let profile = super::validation_profile::validation_evidence_profile_for_tool(tool); + let kind = kind.unwrap_or_else(|| profile.map_or("check", |profile| profile.validation_kind())); let lifecycle = RunnerJobLifecycle::from_wire(status).ok(); if !lifecycle.is_some_and(RunnerJobLifecycle::is_terminal) { let mut value = json!({ @@ -610,7 +610,7 @@ pub(crate) fn validation_job_projection_with_policy( } let process_passed = lifecycle == Some(RunnerJobLifecycle::Completed) && exit_code == Some(0); let mut evidence = structured_validation_evidence(tool, kind, stdout, stderr, truncated); - if tool == "cargo_test" { + if tool == "cargo_test" && kind == "test" { if let Some(authoritative) = authoritative_test_count.filter(|evidence| evidence.is_valid()) { evidence.tests_detected = Some(authoritative.tests_detected); @@ -650,7 +650,8 @@ pub(crate) fn validation_job_projection_with_policy( diagnostics.reason = Some("pytest summary contradicts process outcome"); } } - let mut passed = process_passed && !pytest_contradictory; + let profile_mismatch = profile.is_some_and(|profile| profile.validation_kind() != kind); + let mut passed = process_passed && !pytest_contradictory && !profile_mismatch; let mut value = json!({ "tool": tool, "kind": kind, @@ -668,31 +669,6 @@ pub(crate) fn validation_job_projection_with_policy( value["tests_passed"] = json!(evidence.tests_passed); value["tests_failed"] = json!(evidence.tests_failed); value["zero_tests_run"] = json!(evidence.zero_tests_run); - if matches!(tool, "cargo_test" | "go_test" | "python:pytest:test") && process_passed { - if let Some(minimum_tests) = minimum_tests { - let (status, reason_code) = match evidence.tests_run_count { - Some(actual) if actual >= minimum_tests => ("passed", "minimum_satisfied"), - Some(_) => { - passed = false; - ("failed", "minimum_not_met") - } - None => { - passed = false; - ("unproven", "test_count_unproven") - } - }; - value["passed"] = json!(passed); - value["test_count_assertion"] = json!({ - "minimum_tests": minimum_tests, - "actual_tests_run": evidence.tests_run_count, - "status": status, - "reason_code": reason_code, - "evidence_reason_code": evidence - .test_count_evidence_reason - .unwrap_or("no_complete_summary"), - }); - } - } } "check" => { value["warnings_count"] = json!(evidence.warnings_count); @@ -700,6 +676,33 @@ pub(crate) fn validation_job_projection_with_policy( } _ => {} } + // A requested minimum requires proven counts, including for unknown tools + // or mismatched kinds. Unknown tools without a minimum retain process success. + if process_passed { + if let Some(minimum_tests) = minimum_tests { + let (status, reason_code) = match evidence.tests_run_count { + Some(actual) if actual >= minimum_tests => ("passed", "minimum_satisfied"), + Some(_) => { + passed = false; + ("failed", "minimum_not_met") + } + None => { + passed = false; + ("unproven", "test_count_unproven") + } + }; + value["passed"] = json!(passed); + value["test_count_assertion"] = json!({ + "minimum_tests": minimum_tests, + "actual_tests_run": evidence.tests_run_count, + "status": status, + "reason_code": reason_code, + "evidence_reason_code": evidence + .test_count_evidence_reason + .unwrap_or("no_complete_summary"), + }); + } + } apply_cargo_test_execution_policy(&mut value, tool, require_tests, no_run); Some(value) } @@ -932,7 +935,7 @@ impl ToolRuntime { let kind = metadata .map(|metadata| metadata.kind.as_str()) .or_else(|| { - super::validation_profile::validation_adapter_for_tool(tool) + super::validation_profile::validation_evidence_profile_for_tool(tool) .map(|adapter| adapter.validation_kind()) }) .or(job.purpose.as_deref()); diff --git a/src/tool_runtime/tests/jobs.rs b/src/tool_runtime/tests/jobs.rs index 7dd07b4fc..444dd9336 100644 --- a/src/tool_runtime/tests/jobs.rs +++ b/src/tool_runtime/tests/jobs.rs @@ -11,6 +11,183 @@ use crate::runner_protocol::{ }; use serde_json::json; +#[test] +fn validation_evidence_unknown_profiles_preserve_only_process_success() { + assert_unproven_validation_profiles( + &[("unknown_test", "test"), ("unknown_check", "check")], + true, + ); +} + +#[test] +fn validation_evidence_known_mismatched_profiles_fail_even_without_minimum() { + assert_unproven_validation_profiles( + &[ + ("cargo_check", "test"), + ("cargo_fmt", "test"), + ("cargo_test", "check"), + ("go_vet", "test"), + ("go_test", "check"), + ("python:pytest:test", "check"), + ("cargo_test", "format"), + ], + false, + ); +} + +fn assert_unproven_validation_profiles(profiles: &[(&str, &str)], generic_success: bool) { + use crate::tool_runtime::jobs::{structured_validation_evidence, validation_job_projection}; + + let stdout = "test result: ok. 20 passed; 0 failed; 0 ignored\n"; + let stderr = "warning: unused import\nerror[E0308]: mismatched types\n"; + for &(tool, kind) in profiles { + for truncated in [false, true] { + let evidence = structured_validation_evidence(tool, kind, stdout, stderr, truncated); + assert!(evidence.diagnostics.is_none(), "{tool}/{kind}"); + assert!(evidence.tests_detected.is_none()); + assert!(evidence.tests_run_count.is_none()); + assert!(evidence.tests_passed.is_none()); + assert!(evidence.tests_failed.is_none()); + assert!(evidence.zero_tests_run.is_none()); + assert!(evidence.warnings_count.is_none()); + assert!(evidence.errors_count.is_none()); + for minimum in [None, Some(0), Some(1)] { + let value = validation_job_projection( + Some(tool), + Some(kind), + "completed", + Some(0), + stdout, + stderr, + truncated, + minimum, + ) + .unwrap(); + assert_eq!( + value["passed"], + generic_success && minimum.is_none(), + "{tool}/{kind}: {value}" + ); + if let Some(minimum) = minimum { + assert_eq!(value["test_count_assertion"]["status"], "unproven"); + assert_eq!(value["test_count_assertion"]["minimum_tests"], minimum); + assert_eq!( + value["test_count_assertion"]["reason_code"], + "test_count_unproven" + ); + assert!(value["test_count_assertion"]["actual_tests_run"].is_null()); + } else { + assert!(value.get("test_count_assertion").is_none()); + } + } + } + } +} + +#[test] +fn validation_evidence_go_vet_preserves_historic_counters() { + use crate::tool_runtime::jobs::structured_validation_evidence; + let stderr = "warning: unused import\nerror[E0308]: mismatched types\n"; + // Preserve known Go check counter compatibility while unknown tools + // cannot borrow the old generic Rustc counter fallback. + let go = structured_validation_evidence("go_vet", "check", "", stderr, false); + assert!(go.diagnostics.is_some()); + assert_eq!(go.warnings_count, Some(1)); + assert_eq!(go.errors_count, Some(1)); +} + +#[test] +fn validation_evidence_authoritative_cargo_count_requires_matching_kind_and_tool() { + use crate::runner_protocol::ShellJobTestCountEvidence; + use webcodex_core::validation_evidence::CargoTestCountEvidenceStatus; + + let count = ShellJobTestCountEvidence { + tests_detected: true, + tests_run_count: Some(20), + status: CargoTestCountEvidenceStatus::CompleteSummary, + }; + for (tool, kind) in [ + ("cargo_test", "check"), + ("unknown_test", "test"), + ("go_test", "test"), + ] { + let value = crate::tool_runtime::jobs::validation_job_projection_with_policy( + Some(tool), + Some(kind), + "completed", + Some(0), + "", + "", + true, + Some(&count), + Some(1), + None, + None, + ) + .unwrap(); + assert_eq!(value["passed"], false, "{value}"); + assert_eq!(value["test_count_assertion"]["status"], "unproven"); + assert!(value["test_count_assertion"]["actual_tests_run"].is_null()); + } +} + +#[test] +fn validation_evidence_known_profiles_preserve_counts_failures_zero_and_truncation() { + use crate::tool_runtime::jobs::validation_job_projection; + + for (tool, stdout, passed, failed, status, exit) in [ + ("cargo_test", "test result: ok. 2 passed; 0 failed; 0 ignored\n", 2, 0, "completed", 0), + ("cargo_test", "test result: FAILED. 1 passed; 1 failed; 0 ignored\n", 1, 1, "failed", 101), + ("cargo_test", "test result: ok. 0 passed; 0 failed; 0 ignored\n", 0, 0, "completed", 0), + ("go_test", "{\"Action\":\"pass\",\"Package\":\"example\",\"Test\":\"TestOne\"}\n{\"Action\":\"pass\",\"Package\":\"example\"}\n", 1, 0, "completed", 0), + ("go_test", "{\"Action\":\"fail\",\"Package\":\"example\",\"Test\":\"TestOne\"}\n{\"Action\":\"fail\",\"Package\":\"example\"}\n", 0, 1, "failed", 1), + ("python:pytest:test", "2 passed in 0.01s\n", 2, 0, "completed", 0), + ("python:pytest:test", "1 failed, 1 passed in 0.01s\n", 1, 1, "failed", 1), + ("python:pytest:test", "no tests ran in 0.01s\n", 0, 0, "failed", 5), + ] { + for truncated in [false, true] { + let value = validation_job_projection( + Some(tool), Some("test"), status, Some(exit), stdout, "", truncated, Some(1), + ).unwrap(); + if truncated { + assert!(value["tests_run_count"].is_null(), "{value}"); + assert!(value["tests_passed"].is_null()); + assert!(value["tests_failed"].is_null()); + assert!(value["zero_tests_run"].is_null()); + } else { + assert_eq!(value["tests_run_count"], passed + failed, "{value}"); + assert_eq!(value["tests_passed"], passed, "{value}"); + assert_eq!(value["tests_failed"], failed, "{value}"); + assert_eq!(value["zero_tests_run"], passed + failed == 0); + assert_eq!(value["diagnostics"]["available"], true); + } + assert_eq!(value["passed"], exit == 0 && passed > 0 && !truncated, "{value}"); + if exit != 0 { + assert!(value.get("test_count_assertion").is_none()); + } else { + assert_eq!(value["test_count_assertion"]["status"], + if truncated { "unproven" } else if passed > 0 { "passed" } else { "failed" }); + } + } + } + // Package-only Go output cannot prove zero; preserve that existing contract. + let go_no_tests = validation_job_projection( + Some("go_test"), + Some("test"), + "completed", + Some(0), + "{\"Action\":\"pass\",\"Package\":\"example\"}\n", + "", + false, + Some(1), + ) + .unwrap(); + assert!(go_no_tests["tests_run_count"].is_null()); + assert!(go_no_tests["zero_tests_run"].is_null()); + assert_eq!(go_no_tests["passed"], false); + assert_eq!(go_no_tests["test_count_assertion"]["status"], "unproven"); +} + fn run_jobs_test_in_large_stack_thread(test: F) where F: FnOnce() -> Fut + Send + 'static, diff --git a/src/tool_runtime/tests/validation_handoff/project_validation.rs b/src/tool_runtime/tests/validation_handoff/project_validation.rs index 22417ae9a..a3eb3f06a 100644 --- a/src/tool_runtime/tests/validation_handoff/project_validation.rs +++ b/src/tool_runtime/tests/validation_handoff/project_validation.rs @@ -939,3 +939,68 @@ async fn project_validation_all_packages_dispatches_and_completes_same_wire_job( .is_none()); } } + +#[tokio::test] +async fn project_validation_mismatched_evidence_profile_starts_nothing() { + for (backend, identity) in [ + ("rust", "cargo_test"), + ("rust", "go_vet"), + ("go", "cargo_check"), + ("python", "cargo_test"), + ("rust", "unknown_check"), + ] { + let runtime = setup(1).await; + let task = tokio::spawn({ + let runtime = runtime.clone(); + async move { + runtime + .dispatch_with_auth( + call(ProjectValidationAction::Check, None), + Some(&auth_context(None, true)), + ) + .await + } + }); + let request = wait_for_runner_request(&runtime, "project-validation").await; + assert_eq!(request.kind, "plan_project_validation"); + let semantic: ProjectValidationRequest = + serde_json::from_str(request.content.as_deref().unwrap()).unwrap(); + let operation = webcodex_validation::project_validation_operation( + "rust", + webcodex_validation::SemanticCheck::Check, + None, + false, + ) + .unwrap(); + let plan = ProjectValidationPlan { + provenance: ProjectValidationProvenance { + request: semantic, + backend: backend.into(), + recipe_root: ".".into(), + root_digest: "a".repeat(64), + manifest_digest: "b".repeat(64), + invocation_digest: "c".repeat(64), + }, + adapter: identity.into(), + step: operation.build_readonly_plan().unwrap().structured_step, + validation_target_id: operation.validation_target_id(Some(".")).unwrap(), + }; + complete_sync_shell_lifecycle( + &runtime, + "project-validation", + request.request_id, + ShellCommandExecutionState::Completed, + Some(0), + &serde_json::to_string(&ProjectValidationPlanningResult::Ready { plan }).unwrap(), + "", + None, + ) + .await; + let result = task.await.unwrap(); + assert!(!result.success, "{backend}/{identity}"); + assert!(runtime.runner_registry.list_jobs(Some(10)).await.is_empty()); + assert!(probe_patch_agent_request(&runtime, "project-validation") + .await + .is_none()); + } +} diff --git a/src/tool_runtime/validation.rs b/src/tool_runtime/validation.rs index 754867080..a729e3c19 100644 --- a/src/tool_runtime/validation.rs +++ b/src/tool_runtime/validation.rs @@ -15,13 +15,13 @@ use super::structured_execution::{ StructuredExecutionBudget, StructuredJobHandoffFailure, STRUCTURED_EXECUTION_SYNC_WAIT_SECS, }; use super::tool_result::ToolResult; -#[cfg(test)] -use super::validation_profile::ValidationCommandOptions; use super::validation_profile::{ - requires_multi_package_cargo_check, runtime_profile, validation_adapter_for_tool, - CargoReadOnlyValidationOperation, ReadOnlyValidationOperation, ValidationAdapter, + requires_multi_package_cargo_check, runtime_profile, validation_evidence_profile_for_recipe, + CargoReadOnlyValidationOperation, ReadOnlyValidationOperation, ValidationEvidenceProfile, ValidationFailureEvidence, }; +#[cfg(test)] +use super::validation_profile::{validation_adapter_for_tool, ValidationCommandOptions}; use super::ToolRuntime; use crate::auth::AuthContext; use crate::runner_http::{RunnerFeature, ShellJobStartMetadata}; @@ -63,7 +63,7 @@ pub(super) fn terminal_test_count_assertion_failure_message( } pub(super) fn cargo_fmt_check_is_stable_diff( - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, output: &ProjectCommandOutput, ) -> bool { if output.execution_state != ShellCommandExecutionState::Completed @@ -533,11 +533,21 @@ impl ToolRuntime { if !plan.provenance.is_valid() || plan.provenance.request != request || !canonical_step { return ToolResult::err("invalid Runner project validation plan"); } - let Some(adapter) = validation_adapter_for_tool(&plan.adapter) else { + let semantic_action = match action { + ProjectValidationAction::FormatCheck => webcodex_validation::SemanticCheck::Format, + ProjectValidationAction::Check => webcodex_validation::SemanticCheck::Check, + ProjectValidationAction::Test => webcodex_validation::SemanticCheck::Test, + }; + let Some(adapter) = + validation_evidence_profile_for_recipe(&plan.provenance.backend, semantic_action) + else { return ToolResult::err( "validation adapter unavailable; upgrade Server and Runner together", ); }; + if adapter.tool_identity() != plan.adapter || adapter.validation_kind() != action.kind() { + return ToolResult::err("invalid Runner project validation evidence profile"); + } let source_fence = self.validation_sources.capture(&resolved.resolved_id); let provenance = plan.provenance; let relative_root = provenance.recipe_root.clone(); @@ -760,7 +770,7 @@ impl ToolRuntime { config: &crate::projects::ProjectConfig, cwd: Option<&str>, command: &str, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, step: ShellJobValidationStep, purpose: ExecutionPurpose, timeout_secs: u64, @@ -898,7 +908,7 @@ impl ToolRuntime { &self, job_id: String, sync_wait_secs: u64, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, handoff: ValidationHandoff, ) -> ToolResult { let mut guard = ValidationCleanupGuard::new( @@ -1050,7 +1060,7 @@ impl ToolRuntime { async fn recover_validation_handoff( &self, job_id: String, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, handoff: ValidationHandoff, guard: &mut ValidationCleanupGuard, ) -> ToolResult { @@ -1078,7 +1088,7 @@ impl ToolRuntime { async fn validation_terminal_result( &self, job_id: String, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, job_status: &str, handoff: ValidationHandoff, ) -> ToolResult { @@ -1231,7 +1241,7 @@ impl ToolRuntime { command: &str, cwd: Option<&str>, config: &crate::projects::ProjectConfig, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, output: ProjectCommandOutput, timeout_secs: u64, sync_wait_secs: u64, diff --git a/src/tool_runtime/validation_profile/mod.rs b/src/tool_runtime/validation_profile/mod.rs index ba810503d..02103a8b1 100644 --- a/src/tool_runtime/validation_profile/mod.rs +++ b/src/tool_runtime/validation_profile/mod.rs @@ -4,9 +4,10 @@ use super::helpers::{ }; pub(crate) use webcodex_validation::{ - validation_adapter_for_tool, CargoCheckOptions, CargoReadOnlyValidationOperation, + validation_adapter_for_tool, validation_evidence_profile_for_recipe, + validation_evidence_profile_for_tool, CargoCheckOptions, CargoReadOnlyValidationOperation, CargoTestOptions, GoReadOnlyValidationOperation, GoTestOptions, ReadOnlyValidationOperation, - ValidationAdapter, ValidationCommandOptions, ValidationFailureEvidence, + ValidationCommandOptions, ValidationEvidenceProfile, ValidationFailureEvidence, }; pub(crate) struct ValidationRuntimeProfile { From 6f6dd077cda7d0b84e44ab3c77bd7dc5c6e8e7a4 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:50:27 +0800 Subject: [PATCH 2/2] test(validation): align passive evidence profile fixture --- src/tool_runtime/tests/jobs.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/tool_runtime/tests/jobs.rs b/src/tool_runtime/tests/jobs.rs index 444dd9336..59d2da9f8 100644 --- a/src/tool_runtime/tests/jobs.rs +++ b/src/tool_runtime/tests/jobs.rs @@ -376,6 +376,17 @@ fn passive_validation_projection_separates_execution_result_from_source_freshnes metadata.kind = "test".to_string(); job.exit_code = Some(0); job.test_count_evidence = None; + let mismatched = runtime + .passive_job_validation_projection(&job, None) + .expect("mismatched validation profile"); + assert_eq!(mismatched["passed"], false); + + // A consistent Cargo test identity without count evidence remains + // inconclusive, independently of source freshness. + let metadata = job.validation.as_mut().unwrap(); + metadata.adapter = "cargo_test".to_string(); + metadata.steps[0].name = "test".to_string(); + metadata.steps[0].args = vec!["test".to_string()]; let inconclusive = runtime .passive_job_validation_projection(&job, None) .expect("test validation projection");