From 906e52012eed3778f084acc83a601022fe6fb468 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:36:33 +0800 Subject: [PATCH 1/5] refactor(validation): separate project plan and evidence authority --- .../src/webcodex_runner/validation/project.rs | 13 +- .../validation/project_tests.rs | 141 ++++++++++++-- crates/webcodex-validation/src/adapters/go.rs | 135 ++++++------- .../webcodex-validation/src/adapters/mod.rs | 42 +++-- .../src/adapters/python.rs | 55 +++--- .../webcodex-validation/src/adapters/rust.rs | 83 ++++---- crates/webcodex-validation/src/evidence.rs | 37 ++-- crates/webcodex-validation/src/lib.rs | 7 +- .../webcodex-validation/src/profile_tests.rs | 55 ++++++ src/tool_runtime/jobs.rs | 85 +++++---- src/tool_runtime/tests/jobs.rs | 177 ++++++++++++++++++ .../validation_handoff/project_validation.rs | 65 +++++++ src/tool_runtime/validation.rs | 32 ++-- src/tool_runtime/validation_profile/mod.rs | 5 +- 14 files changed, 692 insertions(+), 240 deletions(-) diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project.rs index adc666c89..913e1e28e 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project.rs @@ -59,11 +59,13 @@ pub(crate) fn plan( .test .as_ref() .and_then(|test| test.filter.as_deref()); + // The operation supplies semantic identity and normalization only. The + // resolved recipe below is the sole authority for the executable step. let operation = project_validation_operation(backend.as_str(), action, packages, all_packages) .and_then(|operation| operation.with_dependency_policy(request.dependency_policy)) .and_then(|operation| operation.with_test_filter(filter)) .map_err(|code| unavailable(code, Some(backend.as_str())))?; - let adapter = operation.adapter(); + let profile = operation.compatibility_profile(); let resolved = resolve_project_validation_recipe( &root, request.cwd.as_deref(), @@ -90,13 +92,6 @@ pub(crate) fn plan( } else { identity }; - debug_assert_eq!( - operation - .build_readonly_plan() - .ok() - .map(|plan| plan.structured_step), - resolved.steps.first().cloned() - ); let cwd = root.join(&resolved.recipe_root_relative); Ok(( ProjectValidationPlan { @@ -108,7 +103,7 @@ pub(crate) fn plan( manifest_digest: resolved.manifest_digest, invocation_digest: resolved.invocation_digest, }, - adapter: adapter.tool_identity().into(), + adapter: profile.tool_identity.into(), step: resolved .steps .into_iter() diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs index 5ec75f86b..d0ee2df90 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs @@ -90,6 +90,12 @@ fn project_validation_runner_resolves_all_production_actions() { ("Cargo.toml", Test, "cargo_test", vec!["test"]), ("go.mod", Check, "go_vet", vec!["vet", "./..."]), ("go.mod", Test, "go_test", vec!["test", "-json", "./..."]), + ( + "pyproject.toml", + Test, + "python:pytest:test", + vec!["-m", "pytest", "--color=no", "-rA"], + ), ] { let (_tmp, root, registry, policy) = fixture(marker); let (plan, cwd) = project::plan(&policy, ®istry, &request(action)).unwrap(); @@ -102,31 +108,25 @@ fn project_validation_runner_resolves_all_production_actions() { Check => webcodex_validation::SemanticCheck::Check, Test => webcodex_validation::SemanticCheck::Test, }; - let canonical_adapter = webcodex_validation::validation_adapter_for_recipe( - plan.provenance.backend.as_str(), - semantic_check, + let resolved = webcodex_validation::resolve_project_validation_recipe( + &root, + None, + None, + &[semantic_check], + None, + None, + false, + None, ) .unwrap(); - let canonical_options = if plan.provenance.backend == "rust" && action == FormatCheck { - webcodex_validation::ValidationCommandOptions { - check: true, - ..Default::default() - } - } else { - webcodex_validation::ValidationCommandOptions::default() - }; - let canonical_plan = canonical_adapter - .build_readonly_plan(canonical_options) - .unwrap(); assert_eq!( - plan.step, canonical_plan.structured_step, - "{adapter} gateway plan must use the canonical adapter step" + std::slice::from_ref(&plan.step), + resolved.steps.as_slice(), + "{adapter} gateway plan must use the resolved recipe step" ); let invocation_digest = format!( "{:x}", - sha2::Sha256::digest( - serde_json::to_vec(&vec![canonical_plan.structured_step]).unwrap() - ) + sha2::Sha256::digest(serde_json::to_vec(&resolved.steps).unwrap()) ); assert_eq!( plan.provenance.invocation_digest, invocation_digest, @@ -138,6 +138,109 @@ fn project_validation_runner_resolves_all_production_actions() { .contains(root.to_str().unwrap())); } } +#[test] +fn project_validation_recipe_step_and_identity_preserve_normalized_selection() { + for (marker, backend, tool, filter, packages) in [ + ( + "Cargo.toml", + "rust", + "cargo_test", + "selected", + vec!["b", "a", "a"], + ), + ( + "go.mod", + "go", + "go_test", + " ^TestA/sub$ ", + vec!["./b", "./a", "./a"], + ), + ( + "pyproject.toml", + "python", + "python:pytest:test", + " selected and not slow ", + vec![], + ), + ] { + let (_tmp, root, registry, policy) = fixture(marker); + let mut req = request(ProjectValidationAction::Test); + req.test = Some(ProjectValidationTestOptions { + filter: Some(if backend == "rust" { + format!(" {filter} ") + } else { + filter.into() + }), + ..Default::default() + }); + if !packages.is_empty() { + req.scope = Some(ProjectValidationScope { + packages: packages.into_iter().map(str::to_string).collect(), + all_packages: false, + }); + req.dependency_policy = Some(ProjectDependencyPolicy { + mode: ProjectDependencyMode::Locked, + }); + } + let (plan, _) = project::plan(&policy, ®istry, &req).unwrap(); + let resolved = webcodex_validation::resolve_project_validation_recipe( + &root, + None, + None, + &[webcodex_validation::SemanticCheck::Test], + req.test.as_ref().unwrap().filter.as_deref(), + req.scope + .as_ref() + .and_then(ProjectValidationScope::explicit_packages), + false, + req.dependency_policy, + ) + .unwrap(); + assert_eq!(std::slice::from_ref(&plan.step), resolved.steps.as_slice()); + assert_eq!(plan.adapter, tool); + assert_eq!( + plan.provenance.invocation_digest, + resolved.invocation_digest + ); + + req.test.as_mut().unwrap().filter = Some(filter.into()); + if backend == "rust" { + let scope = req.scope.as_mut().unwrap(); + scope.packages.sort(); + scope.packages.dedup(); + } + let (normalized, _) = project::plan(&policy, ®istry, &req).unwrap(); + assert_eq!(plan.step, normalized.step); + assert_eq!(plan.validation_target_id, normalized.validation_target_id); + let operation = webcodex_validation::project_validation_operation( + backend, + webcodex_validation::SemanticCheck::Test, + req.scope + .as_ref() + .and_then(ProjectValidationScope::explicit_packages) + .map(<[String]>::to_vec), + false, + ) + .unwrap() + .with_dependency_policy(req.dependency_policy) + .unwrap() + .with_test_filter(Some(filter)) + .unwrap(); + let identity = operation + .validation_target_id(Some(&resolved.recipe_root_relative)) + .unwrap(); + let expected = if backend == "go" { + webcodex_core::validation_identity::contextualize_structured_validation_target_identity( + &identity, + webcodex_core::validation_identity::StructuredValidationExecutionContext::GoProjectSingleModuleV1, + ).unwrap() + } else { + identity + }; + assert_eq!(plan.validation_target_id, expected); + } +} + #[test] fn go_project_validation_identity_is_single_module_domain_separated() { for action in [ diff --git a/crates/webcodex-validation/src/adapters/go.rs b/crates/webcodex-validation/src/adapters/go.rs index a3853540e..6960d7700 100644 --- a/crates/webcodex-validation/src/adapters/go.rs +++ b/crates/webcodex-validation/src/adapters/go.rs @@ -1,6 +1,7 @@ use super::{ read_only_validation_plan, validate_package_scope_exclusivity, ReadOnlyValidationPlan, - ValidationAdapter, ValidationCommandOptions, ValidationFailureEvidence, ValidationPlanArg, + ValidationAdapter, ValidationCommandOptions, ValidationEvidenceProfile, + ValidationFailureEvidence, ValidationPlanArg, }; use webcodex_core::runner_protocol::normalize_go_test_packages; use webcodex_core::validation_evidence::{parse_go_test_diagnostics, ValidationDiagnostics}; @@ -25,7 +26,7 @@ pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { &GO_TEST_ADAPTER } -impl ValidationAdapter for GoTestValidationAdapter { +impl ValidationEvidenceProfile for GoTestValidationAdapter { fn validation_kind(&self) -> &'static str { "test" } @@ -34,54 +35,6 @@ impl ValidationAdapter for GoTestValidationAdapter { "go_test" } - fn build_readonly_plan( - &self, - options: ValidationCommandOptions, - ) -> Result { - if options.check - || options.lib.is_some() - || options.all_targets.is_some() - || options.all_features.is_some() - || options.no_default_features.is_some() - || options.features.is_some() - || options.package.is_some() - || options.cargo_packages.is_some() - || options.no_run.is_some() - { - return Err("go_test does not accept Cargo validation command options".to_string()); - } - let explicit_packages = options.go_packages.is_some(); - validate_package_scope_exclusivity(explicit_packages, options.all_packages)?; - let packages = normalize_go_test_packages(options.go_packages.as_deref()) - .map_err(|reason| format!("packages {reason}"))?; - let mut args = vec![ - ValidationPlanArg::Literal("test"), - ValidationPlanArg::Literal("-json"), - ]; - if options.dependency_mode - == Some(webcodex_core::project_validation::ProjectDependencyMode::Locked) - { - args.push(ValidationPlanArg::Literal("-mod=readonly")); - } - if let Some(filter) = options - .filter - .as_deref() - .map(webcodex_core::runner_protocol::normalize_go_test_filter) - .transpose()? - .flatten() - { - args.push(ValidationPlanArg::Literal("-run")); - args.push(ValidationPlanArg::Value(filter)); - } - if explicit_packages && !options.all_packages { - args.extend(packages.into_iter().map(ValidationPlanArg::Value)); - } else { - debug_assert_eq!(packages.as_slice(), ["./..."]); - args.push(ValidationPlanArg::Literal("./...")); - } - read_only_validation_plan("test", "go", args) - } - fn parse( &self, stdout_excerpt: &str, @@ -132,21 +85,12 @@ impl ValidationAdapter for GoTestValidationAdapter { } } -struct GoVetValidationAdapter; -static GO_VET_ADAPTER: GoVetValidationAdapter = GoVetValidationAdapter; -impl ValidationAdapter for GoVetValidationAdapter { - fn validation_kind(&self) -> &'static str { - "check" - } - fn tool_identity(&self) -> &'static str { - "go_vet" - } +impl ValidationAdapter for GoTestValidationAdapter { fn build_readonly_plan( &self, options: ValidationCommandOptions, ) -> Result { if options.check - || options.filter.is_some() || options.lib.is_some() || options.all_targets.is_some() || options.all_features.is_some() @@ -156,29 +100,56 @@ impl ValidationAdapter for GoVetValidationAdapter { || options.cargo_packages.is_some() || options.no_run.is_some() { - return Err("go_vet does not accept Cargo validation command options".to_string()); + return Err("go_test does not accept Cargo validation command options".to_string()); } let explicit_packages = options.go_packages.is_some(); validate_package_scope_exclusivity(explicit_packages, options.all_packages)?; let packages = normalize_go_test_packages(options.go_packages.as_deref()) .map_err(|reason| format!("packages {reason}"))?; - let mut args = vec![ValidationPlanArg::Literal("vet")]; + let mut args = vec![ + ValidationPlanArg::Literal("test"), + ValidationPlanArg::Literal("-json"), + ]; if options.dependency_mode == Some(webcodex_core::project_validation::ProjectDependencyMode::Locked) { args.push(ValidationPlanArg::Literal("-mod=readonly")); } + if let Some(filter) = options + .filter + .as_deref() + .map(webcodex_core::runner_protocol::normalize_go_test_filter) + .transpose()? + .flatten() + { + args.push(ValidationPlanArg::Literal("-run")); + args.push(ValidationPlanArg::Value(filter)); + } if explicit_packages && !options.all_packages { args.extend(packages.into_iter().map(ValidationPlanArg::Value)); } else { debug_assert_eq!(packages.as_slice(), ["./..."]); args.push(ValidationPlanArg::Literal("./...")); } - read_only_validation_plan("check", "go", args) + read_only_validation_plan("test", "go", args) } +} + +struct GoVetValidationAdapter; +static GO_VET_ADAPTER: GoVetValidationAdapter = GoVetValidationAdapter; +impl ValidationEvidenceProfile for GoVetValidationAdapter { + fn validation_kind(&self) -> &'static str { + "check" + } + + fn tool_identity(&self) -> &'static str { + "go_vet" + } + fn parse(&self, _stdout: &str, stderr: &str, truncated: bool) -> ValidationDiagnostics { webcodex_core::validation_evidence::parse_go_vet_diagnostics(stderr, truncated) } + fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { if evidence.success { "unknown" @@ -192,3 +163,41 @@ impl ValidationAdapter for GoVetValidationAdapter { } } } + +impl ValidationAdapter for GoVetValidationAdapter { + fn build_readonly_plan( + &self, + options: ValidationCommandOptions, + ) -> Result { + if options.check + || options.filter.is_some() + || options.lib.is_some() + || options.all_targets.is_some() + || options.all_features.is_some() + || options.no_default_features.is_some() + || options.features.is_some() + || options.package.is_some() + || options.cargo_packages.is_some() + || options.no_run.is_some() + { + return Err("go_vet does not accept Cargo validation command options".to_string()); + } + let explicit_packages = options.go_packages.is_some(); + validate_package_scope_exclusivity(explicit_packages, options.all_packages)?; + let packages = normalize_go_test_packages(options.go_packages.as_deref()) + .map_err(|reason| format!("packages {reason}"))?; + let mut args = vec![ValidationPlanArg::Literal("vet")]; + if options.dependency_mode + == Some(webcodex_core::project_validation::ProjectDependencyMode::Locked) + { + args.push(ValidationPlanArg::Literal("-mod=readonly")); + } + if explicit_packages && !options.all_packages { + args.extend(packages.into_iter().map(ValidationPlanArg::Value)); + } else { + debug_assert_eq!(packages.as_slice(), ["./..."]); + args.push(ValidationPlanArg::Literal("./...")); + } + read_only_validation_plan("check", "go", args) + } +} diff --git a/crates/webcodex-validation/src/adapters/mod.rs b/crates/webcodex-validation/src/adapters/mod.rs index aa92da28b..d4922874a 100644 --- a/crates/webcodex-validation/src/adapters/mod.rs +++ b/crates/webcodex-validation/src/adapters/mod.rs @@ -508,21 +508,12 @@ pub struct ValidationFailureEvidence<'a> { pub stderr_excerpt: &'a str, } -pub trait ValidationAdapter: Sync { +/// Result interpretation without command-building authority. +pub trait ValidationEvidenceProfile: Sync { fn validation_kind(&self) -> &'static str; fn tool_identity(&self) -> &'static str; - fn build_readonly_plan( - &self, - options: ValidationCommandOptions, - ) -> Result; - - fn build_command(&self, options: ValidationCommandOptions) -> Result { - self.build_readonly_plan(options) - .map(|plan| plan.compatibility_command) - } - fn parse( &self, stdout_excerpt: &str, @@ -537,6 +528,18 @@ pub trait ValidationAdapter: Sync { } } +pub trait ValidationAdapter: ValidationEvidenceProfile { + fn build_readonly_plan( + &self, + options: ValidationCommandOptions, + ) -> Result; + + fn build_command(&self, options: ValidationCommandOptions) -> Result { + self.build_readonly_plan(options) + .map(|plan| plan.compatibility_command) + } +} + /// Canonical mapping from structured validation classification to execution /// evidence intent. Tool selection determines this purpose; callers do not. pub fn execution_purpose_for_validation_kind(validation_kind: &str) -> ExecutionPurpose { @@ -572,3 +575,20 @@ pub fn validation_adapter_for_recipe( _ => return None, }) } + +/// Evidence-only view of the canonical adapter registry. +pub fn validation_evidence_profile_for_tool( + tool_identity: &str, +) -> Option<&'static dyn ValidationEvidenceProfile> { + validation_adapter_for_tool(tool_identity) + .map(|adapter| adapter as &dyn ValidationEvidenceProfile) +} + +/// Evidence-only view of the canonical backend/action mapping. +pub fn validation_evidence_profile_for_recipe( + backend: &str, + action: crate::SemanticCheck, +) -> Option<&'static dyn ValidationEvidenceProfile> { + validation_adapter_for_recipe(backend, action) + .map(|adapter| adapter as &dyn ValidationEvidenceProfile) +} diff --git a/crates/webcodex-validation/src/adapters/python.rs b/crates/webcodex-validation/src/adapters/python.rs index 00b8da12a..93d405d74 100644 --- a/crates/webcodex-validation/src/adapters/python.rs +++ b/crates/webcodex-validation/src/adapters/python.rs @@ -1,6 +1,6 @@ use super::{ read_only_validation_plan, ReadOnlyValidationPlan, ValidationAdapter, ValidationCommandOptions, - ValidationFailureEvidence, ValidationPlanArg, + ValidationEvidenceProfile, ValidationFailureEvidence, ValidationPlanArg, }; use webcodex_core::validation_evidence::{parse_pytest_diagnostics, ValidationDiagnostics}; @@ -10,13 +10,42 @@ pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { &PYTEST_ADAPTER } -impl ValidationAdapter for PytestValidationAdapter { +impl ValidationEvidenceProfile for PytestValidationAdapter { fn validation_kind(&self) -> &'static str { "test" } + fn tool_identity(&self) -> &'static str { "python:pytest:test" } + + fn parse(&self, stdout: &str, _stderr: &str, truncated: bool) -> ValidationDiagnostics { + parse_pytest_diagnostics(stdout, truncated) + } + + fn reports_test_run_metadata(&self) -> bool { + true + } + + fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { + if evidence.success { + return "unknown"; + } + if matches!( + evidence.reported_failure_kind, + Some("timeout" | "timed_out" | "command_timeout") + ) { + return "timeout"; + } + if evidence.exit_code == Some(1) { + "test_failure" + } else { + "process_exit" + } + } +} + +impl ValidationAdapter for PytestValidationAdapter { fn build_readonly_plan( &self, options: ValidationCommandOptions, @@ -52,26 +81,4 @@ impl ValidationAdapter for PytestValidationAdapter { } read_only_validation_plan("test", "python", args) } - fn parse(&self, stdout: &str, _stderr: &str, truncated: bool) -> ValidationDiagnostics { - parse_pytest_diagnostics(stdout, truncated) - } - fn reports_test_run_metadata(&self) -> bool { - true - } - fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { - if evidence.success { - return "unknown"; - } - if matches!( - evidence.reported_failure_kind, - Some("timeout" | "timed_out" | "command_timeout") - ) { - return "timeout"; - } - if evidence.exit_code == Some(1) { - "test_failure" - } else { - "process_exit" - } - } } diff --git a/crates/webcodex-validation/src/adapters/rust.rs b/crates/webcodex-validation/src/adapters/rust.rs index 5dc885833..aa1bbf11f 100644 --- a/crates/webcodex-validation/src/adapters/rust.rs +++ b/crates/webcodex-validation/src/adapters/rust.rs @@ -1,6 +1,7 @@ use super::{ read_only_validation_plan, validate_package_scope_exclusivity, ReadOnlyValidationPlan, - ValidationAdapter, ValidationCommandOptions, ValidationFailureEvidence, ValidationPlanArg, + ValidationAdapter, ValidationCommandOptions, ValidationEvidenceProfile, + ValidationFailureEvidence, ValidationPlanArg, }; use webcodex_core::runner_protocol::{ normalize_cargo_packages, normalize_cargo_value, normalize_rust_test_filter, @@ -55,7 +56,7 @@ pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { &CARGO_TEST_ADAPTER } -impl ValidationAdapter for RustValidationAdapter { +impl ValidationEvidenceProfile for RustValidationAdapter { fn validation_kind(&self) -> &'static str { match self.kind { RustAdapterKind::Format => "format", @@ -68,44 +69,6 @@ impl ValidationAdapter for RustValidationAdapter { self.tool_identity } - fn build_readonly_plan( - &self, - options: ValidationCommandOptions, - ) -> Result { - if options.go_packages.is_some() { - return Err("Cargo validation does not accept go_test packages".to_string()); - } - match self.kind { - RustAdapterKind::Format => { - if !options.check { - return Err("cargo_fmt mutation is not read-only validation".to_string()); - } - read_only_validation_plan( - "format", - "cargo", - vec![ - ValidationPlanArg::Literal("fmt"), - ValidationPlanArg::Literal("--"), - ValidationPlanArg::Literal("--check"), - ], - ) - } - RustAdapterKind::Check => cargo_check_plan(options), - RustAdapterKind::Test => cargo_test_plan(options), - } - } - - fn build_command(&self, options: ValidationCommandOptions) -> Result { - if self.kind == RustAdapterKind::Format && !options.check { - if options.go_packages.is_some() { - return Err("Cargo validation does not accept go_test packages".to_string()); - } - return Ok("cargo fmt".to_string()); - } - self.build_readonly_plan(options) - .map(|plan| plan.compatibility_command) - } - fn parse( &self, stdout_excerpt: &str, @@ -189,6 +152,46 @@ impl ValidationAdapter for RustValidationAdapter { } } +impl ValidationAdapter for RustValidationAdapter { + fn build_readonly_plan( + &self, + options: ValidationCommandOptions, + ) -> Result { + if options.go_packages.is_some() { + return Err("Cargo validation does not accept go_test packages".to_string()); + } + match self.kind { + RustAdapterKind::Format => { + if !options.check { + return Err("cargo_fmt mutation is not read-only validation".to_string()); + } + read_only_validation_plan( + "format", + "cargo", + vec![ + ValidationPlanArg::Literal("fmt"), + ValidationPlanArg::Literal("--"), + ValidationPlanArg::Literal("--check"), + ], + ) + } + RustAdapterKind::Check => cargo_check_plan(options), + RustAdapterKind::Test => cargo_test_plan(options), + } + } + + fn build_command(&self, options: ValidationCommandOptions) -> Result { + if self.kind == RustAdapterKind::Format && !options.check { + if options.go_packages.is_some() { + return Err("Cargo validation does not accept go_test packages".to_string()); + } + return Ok("cargo fmt".to_string()); + } + self.build_readonly_plan(options) + .map(|plan| plan.compatibility_command) + } +} + fn cargo_check_plan(options: ValidationCommandOptions) -> Result { validate_package_scope_exclusivity( options.package.is_some() || options.cargo_packages.is_some(), diff --git a/crates/webcodex-validation/src/evidence.rs b/crates/webcodex-validation/src/evidence.rs index 7579e1248..e4330c339 100644 --- a/crates/webcodex-validation/src/evidence.rs +++ b/crates/webcodex-validation/src/evidence.rs @@ -188,8 +188,8 @@ pub struct CurrentValidationEvidenceProjection { } use crate::adapters::{ - execution_purpose_for_validation_kind, validation_adapter_for_tool, ValidationAdapter, - ValidationFailureEvidence, + execution_purpose_for_validation_kind, validation_evidence_profile_for_tool, + ValidationEvidenceProfile, ValidationFailureEvidence, }; pub fn skipped_validation_summary() -> Value { @@ -751,7 +751,7 @@ fn validation_event_decides_historical_failure_status(event: &ValidationEvent) - // `validation_kind = test` is only an intent/category for generic execution. // Only an actual first-class structured test tool opts into count/zero-test proof. fn structured_test_requires_execution_proof(event: &ValidationEvent) -> bool { - validation_adapter_for_tool(&event.tool_name).is_some_and(|adapter| { + validation_evidence_profile_for_tool(&event.tool_name).is_some_and(|adapter| { adapter.validation_kind() == "test" && adapter.reports_test_run_metadata() }) } @@ -1005,7 +1005,7 @@ pub fn event_is_job_acceptance_only(event: &SessionEvent) -> bool { } pub fn validation_kind_for_tool(tool_name: &str) -> Option<&'static str> { - if let Some(adapter) = validation_adapter_for_tool(tool_name) { + if let Some(adapter) = validation_evidence_profile_for_tool(tool_name) { return Some(adapter.validation_kind()); } None @@ -1070,7 +1070,7 @@ fn validation_event_from_finished( finished: &SessionEvent, started: Option<&SessionEvent>, ) -> Option { - let adapter = validation_adapter_for_execution(finished, started); + let adapter = validation_evidence_profile_for_execution(finished, started); let purpose = started .and_then(execution_purpose) .or_else(|| execution_purpose(finished))?; @@ -1278,18 +1278,18 @@ fn validation_event_failure_class(event: &ValidationEvent) -> &'static str { } } -fn validation_adapter_for_execution( +fn validation_evidence_profile_for_execution( finished: &SessionEvent, started: Option<&SessionEvent>, -) -> Option<&'static dyn ValidationAdapter> { - validation_adapter_for_tool(&finished.tool_name) +) -> Option<&'static dyn ValidationEvidenceProfile> { + validation_evidence_profile_for_tool(&finished.tool_name) .or_else(|| { finished .validation_output_summary .as_ref() .and_then(|v| v.get("adapter")) .and_then(Value::as_str) - .and_then(validation_adapter_for_tool) + .and_then(validation_evidence_profile_for_tool) }) .or_else(|| { started @@ -1310,7 +1310,7 @@ fn validation_adapter_for_execution( .and_then(|summary| summary.get("validation_tool")) .and_then(Value::as_str) }) - .and_then(validation_adapter_for_tool) + .and_then(validation_evidence_profile_for_tool) }) .or_else(|| { let command = started @@ -1330,9 +1330,11 @@ fn validation_adapter_for_execution( })?; let mut words = command.split_whitespace(); match (words.next(), words.next()) { - (Some("cargo"), Some("fmt")) => validation_adapter_for_tool("cargo_fmt"), - (Some("cargo"), Some("check")) => validation_adapter_for_tool("cargo_check"), - (Some("cargo"), Some("test")) => validation_adapter_for_tool("cargo_test"), + (Some("cargo"), Some("fmt")) => validation_evidence_profile_for_tool("cargo_fmt"), + (Some("cargo"), Some("check")) => { + validation_evidence_profile_for_tool("cargo_check") + } + (Some("cargo"), Some("test")) => validation_evidence_profile_for_tool("cargo_test"), _ => None, } }) @@ -1505,7 +1507,7 @@ fn validation_failure_kind( finished: &SessionEvent, success: bool, diagnostics: Option<&ValidationDiagnostics>, - adapter: Option<&dyn ValidationAdapter>, + adapter: Option<&dyn ValidationEvidenceProfile>, ) -> &'static str { if let Some(adapter) = adapter { let (stdout_excerpt, stderr_excerpt, _) = @@ -1614,7 +1616,7 @@ fn parser_summary_for_events(events: &[ValidationEvent]) -> ValidationParserSumm fn validation_diagnostics_from_summary( finished: &SessionEvent, - adapter: &dyn ValidationAdapter, + adapter: &dyn ValidationEvidenceProfile, ) -> Option { let (stdout_excerpt, stderr_excerpt, truncated) = validation_output_excerpts(finished)?; Some(adapter.parse(stdout_excerpt, stderr_excerpt, truncated)) @@ -1643,7 +1645,7 @@ fn validation_output_excerpts(finished: &SessionEvent) -> Option<(&str, &str, bo fn validation_test_run_metadata( finished: &SessionEvent, - adapter: Option<&dyn ValidationAdapter>, + adapter: Option<&dyn ValidationEvidenceProfile>, diagnostics: Option<&ValidationDiagnostics>, ) -> ( Option, @@ -1664,7 +1666,8 @@ fn validation_test_run_metadata( .into_iter() .any(|field| value.get(field).is_some()) }); - if !adapter.is_some_and(ValidationAdapter::reports_test_run_metadata) && !explicit_test_metadata + if !adapter.is_some_and(ValidationEvidenceProfile::reports_test_run_metadata) + && !explicit_test_metadata { return (None, None, None, None, None); } diff --git a/crates/webcodex-validation/src/lib.rs b/crates/webcodex-validation/src/lib.rs index 23f20b022..2a67d66b6 100644 --- a/crates/webcodex-validation/src/lib.rs +++ b/crates/webcodex-validation/src/lib.rs @@ -22,11 +22,12 @@ mod recipe_tests; pub use adapters::{ execution_purpose_for_validation_kind, project_validation_operation, - validation_adapter_for_recipe, validation_adapter_for_tool, CargoCheckOptions, - CargoReadOnlyValidationOperation, CargoTestOptions, GoCheckOptions, + validation_adapter_for_recipe, validation_adapter_for_tool, + validation_evidence_profile_for_recipe, validation_evidence_profile_for_tool, + CargoCheckOptions, CargoReadOnlyValidationOperation, CargoTestOptions, GoCheckOptions, GoReadOnlyValidationOperation, GoTestOptions, PythonTestOptions, ReadOnlyValidationOperation, ReadOnlyValidationPlan, ValidationAdapter, ValidationCommandOptions, - ValidationCompatibilityProfile, ValidationFailureEvidence, + ValidationCompatibilityProfile, ValidationEvidenceProfile, ValidationFailureEvidence, }; #[cfg(any(feature = "session-evidence", test))] pub use evidence::{ diff --git a/crates/webcodex-validation/src/profile_tests.rs b/crates/webcodex-validation/src/profile_tests.rs index f66d25f0a..32cb354e9 100644 --- a/crates/webcodex-validation/src/profile_tests.rs +++ b/crates/webcodex-validation/src/profile_tests.rs @@ -890,3 +890,58 @@ fn project_validation_python_pytest_owns_filter_plan_and_parser() { assert!(!is_known_tool_name("pytest")); assert!(!is_known_tool_name("python:pytest:test")); } + +#[test] +fn evidence_profiles_share_direct_adapter_registry_and_recipe_mapping() { + use crate::{validation_evidence_profile_for_recipe, validation_evidence_profile_for_tool}; + for (backend, action, tool) in [ + ("rust", SemanticCheck::Format, "cargo_fmt"), + ("rust", SemanticCheck::Check, "cargo_check"), + ("rust", SemanticCheck::Test, "cargo_test"), + ("go", SemanticCheck::Check, "go_vet"), + ("go", SemanticCheck::Test, "go_test"), + ("python", SemanticCheck::Test, "python:pytest:test"), + ] { + let adapter = validation_adapter_for_tool(tool).unwrap(); + let profile = validation_evidence_profile_for_tool(tool).unwrap(); + let recipe = validation_evidence_profile_for_recipe(backend, action).unwrap(); + assert_eq!(profile.tool_identity(), adapter.tool_identity()); + assert_eq!(recipe.tool_identity(), tool); + assert_eq!(profile.validation_kind(), adapter.validation_kind()); + assert_eq!( + profile.reports_test_run_metadata(), + adapter.reports_test_run_metadata() + ); + for truncated in [false, true] { + let stdout = "test result: ok. 2 passed; 0 failed; 0 ignored\n2 passed in 0.01s\n"; + let stderr = "error: example failure"; + let diagnostics = profile.parse(stdout, stderr, truncated); + assert_eq!( + serde_json::to_value(&diagnostics).unwrap(), + serde_json::to_value(adapter.parse(stdout, stderr, truncated)).unwrap() + ); + for success in [false, true] { + let evidence = || ValidationFailureEvidence { + success, + reported_failure_kind: None, + exit_code: Some(if success { 0 } else { 1 }), + diagnostics: Some(&diagnostics), + stdout_excerpt: stdout, + stderr_excerpt: stderr, + }; + assert_eq!( + profile.map_failure_kind(evidence()), + adapter.map_failure_kind(evidence()) + ); + } + } + assert!(adapter + .build_readonly_plan(ValidationCommandOptions { + check: tool == "cargo_fmt", + ..Default::default() + }) + .is_ok()); + } + assert!(validation_evidence_profile_for_tool("unknown_test").is_none()); + assert!(validation_evidence_profile_for_recipe("go", SemanticCheck::Format).is_none()); +} diff --git a/src/tool_runtime/jobs.rs b/src/tool_runtime/jobs.rs index 35127975f..bdbdb7469 100644 --- a/src/tool_runtime/jobs.rs +++ b/src/tool_runtime/jobs.rs @@ -427,10 +427,11 @@ pub(crate) fn structured_validation_evidence( truncated: bool, ) -> StructuredValidationEvidence { let combined = format!("{stdout}\n{stderr}"); - let diagnostics = matches!(kind, "check" | "test") - .then(|| super::validation_profile::validation_adapter_for_tool(tool)) + let profile = matches!(kind, "check" | "test") + .then(|| super::validation_profile::validation_evidence_profile_for_tool(tool)) .flatten() - .map(|adapter| adapter.parse(stdout, stderr, truncated)); + .filter(|adapter| adapter.validation_kind() == kind); + let diagnostics = profile.map(|adapter| adapter.parse(stdout, stderr, truncated)); let mut evidence = StructuredValidationEvidence { diagnostics, tests_detected: None, @@ -442,8 +443,8 @@ pub(crate) fn structured_validation_evidence( warnings_count: None, errors_count: None, }; - match kind { - "test" if tool == "python:pytest:test" => { + match profile.map(|adapter| adapter.tool_identity()) { + Some("python:pytest:test") => { let summary = evidence .diagnostics .as_ref() @@ -463,7 +464,7 @@ pub(crate) fn structured_validation_evidence( evidence.zero_tests_run = evidence.tests_run_count.map(|count| count == 0); } } - "test" if tool == "go_test" => { + Some("go_test") => { let test_summary = evidence .diagnostics .as_ref() @@ -482,7 +483,7 @@ pub(crate) fn structured_validation_evidence( evidence.zero_tests_run = evidence.tests_run_count.map(|count| count == 0); } } - "test" => { + Some("cargo_test") => { let metadata = super::cargo::parse_cargo_test_run_metadata(&combined); evidence.tests_detected = Some(metadata.tests_detected); evidence.test_count_evidence_reason = Some(if truncated { @@ -497,7 +498,9 @@ pub(crate) fn structured_validation_evidence( evidence.zero_tests_run = metadata.zero_tests_run; } } - "check" if !truncated => { + // Preserve the existing check counter projection for known Cargo/Go + // adapters; unknown check identities must not inherit Rustc parsing. + Some("cargo_check" | "go_vet") if !truncated => { evidence.warnings_count = Some(super::cargo::count_rustc_diagnostics(&combined, "warning:") as u64); evidence.errors_count = @@ -548,11 +551,8 @@ pub(crate) fn validation_job_projection_with_policy( no_run: Option, ) -> Option { let tool = tool?; - let kind = kind.unwrap_or(match tool { - "cargo_test" | "go_test" | "python:pytest:test" => "test", - "cargo_fmt" => "format", - _ => "check", - }); + let profile = super::validation_profile::validation_evidence_profile_for_tool(tool); + let kind = kind.unwrap_or_else(|| profile.map_or("check", |profile| profile.validation_kind())); let lifecycle = RunnerJobLifecycle::from_wire(status).ok(); if !lifecycle.is_some_and(RunnerJobLifecycle::is_terminal) { let mut value = json!({ @@ -610,7 +610,7 @@ pub(crate) fn validation_job_projection_with_policy( } let process_passed = lifecycle == Some(RunnerJobLifecycle::Completed) && exit_code == Some(0); let mut evidence = structured_validation_evidence(tool, kind, stdout, stderr, truncated); - if tool == "cargo_test" { + if tool == "cargo_test" && kind == "test" { if let Some(authoritative) = authoritative_test_count.filter(|evidence| evidence.is_valid()) { evidence.tests_detected = Some(authoritative.tests_detected); @@ -650,7 +650,8 @@ pub(crate) fn validation_job_projection_with_policy( diagnostics.reason = Some("pytest summary contradicts process outcome"); } } - let mut passed = process_passed && !pytest_contradictory; + let profile_mismatch = profile.is_some_and(|profile| profile.validation_kind() != kind); + let mut passed = process_passed && !pytest_contradictory && !profile_mismatch; let mut value = json!({ "tool": tool, "kind": kind, @@ -668,31 +669,6 @@ pub(crate) fn validation_job_projection_with_policy( value["tests_passed"] = json!(evidence.tests_passed); value["tests_failed"] = json!(evidence.tests_failed); value["zero_tests_run"] = json!(evidence.zero_tests_run); - if matches!(tool, "cargo_test" | "go_test" | "python:pytest:test") && process_passed { - if let Some(minimum_tests) = minimum_tests { - let (status, reason_code) = match evidence.tests_run_count { - Some(actual) if actual >= minimum_tests => ("passed", "minimum_satisfied"), - Some(_) => { - passed = false; - ("failed", "minimum_not_met") - } - None => { - passed = false; - ("unproven", "test_count_unproven") - } - }; - value["passed"] = json!(passed); - value["test_count_assertion"] = json!({ - "minimum_tests": minimum_tests, - "actual_tests_run": evidence.tests_run_count, - "status": status, - "reason_code": reason_code, - "evidence_reason_code": evidence - .test_count_evidence_reason - .unwrap_or("no_complete_summary"), - }); - } - } } "check" => { value["warnings_count"] = json!(evidence.warnings_count); @@ -700,6 +676,33 @@ pub(crate) fn validation_job_projection_with_policy( } _ => {} } + // A requested minimum requires proven counts, including for unknown tools + // or mismatched kinds. Unknown tools without a minimum retain process success. + if process_passed { + if let Some(minimum_tests) = minimum_tests { + let (status, reason_code) = match evidence.tests_run_count { + Some(actual) if actual >= minimum_tests => ("passed", "minimum_satisfied"), + Some(_) => { + passed = false; + ("failed", "minimum_not_met") + } + None => { + passed = false; + ("unproven", "test_count_unproven") + } + }; + value["passed"] = json!(passed); + value["test_count_assertion"] = json!({ + "minimum_tests": minimum_tests, + "actual_tests_run": evidence.tests_run_count, + "status": status, + "reason_code": reason_code, + "evidence_reason_code": evidence + .test_count_evidence_reason + .unwrap_or("no_complete_summary"), + }); + } + } apply_cargo_test_execution_policy(&mut value, tool, require_tests, no_run); Some(value) } @@ -932,7 +935,7 @@ impl ToolRuntime { let kind = metadata .map(|metadata| metadata.kind.as_str()) .or_else(|| { - super::validation_profile::validation_adapter_for_tool(tool) + super::validation_profile::validation_evidence_profile_for_tool(tool) .map(|adapter| adapter.validation_kind()) }) .or(job.purpose.as_deref()); diff --git a/src/tool_runtime/tests/jobs.rs b/src/tool_runtime/tests/jobs.rs index 7dd07b4fc..444dd9336 100644 --- a/src/tool_runtime/tests/jobs.rs +++ b/src/tool_runtime/tests/jobs.rs @@ -11,6 +11,183 @@ use crate::runner_protocol::{ }; use serde_json::json; +#[test] +fn validation_evidence_unknown_profiles_preserve_only_process_success() { + assert_unproven_validation_profiles( + &[("unknown_test", "test"), ("unknown_check", "check")], + true, + ); +} + +#[test] +fn validation_evidence_known_mismatched_profiles_fail_even_without_minimum() { + assert_unproven_validation_profiles( + &[ + ("cargo_check", "test"), + ("cargo_fmt", "test"), + ("cargo_test", "check"), + ("go_vet", "test"), + ("go_test", "check"), + ("python:pytest:test", "check"), + ("cargo_test", "format"), + ], + false, + ); +} + +fn assert_unproven_validation_profiles(profiles: &[(&str, &str)], generic_success: bool) { + use crate::tool_runtime::jobs::{structured_validation_evidence, validation_job_projection}; + + let stdout = "test result: ok. 20 passed; 0 failed; 0 ignored\n"; + let stderr = "warning: unused import\nerror[E0308]: mismatched types\n"; + for &(tool, kind) in profiles { + for truncated in [false, true] { + let evidence = structured_validation_evidence(tool, kind, stdout, stderr, truncated); + assert!(evidence.diagnostics.is_none(), "{tool}/{kind}"); + assert!(evidence.tests_detected.is_none()); + assert!(evidence.tests_run_count.is_none()); + assert!(evidence.tests_passed.is_none()); + assert!(evidence.tests_failed.is_none()); + assert!(evidence.zero_tests_run.is_none()); + assert!(evidence.warnings_count.is_none()); + assert!(evidence.errors_count.is_none()); + for minimum in [None, Some(0), Some(1)] { + let value = validation_job_projection( + Some(tool), + Some(kind), + "completed", + Some(0), + stdout, + stderr, + truncated, + minimum, + ) + .unwrap(); + assert_eq!( + value["passed"], + generic_success && minimum.is_none(), + "{tool}/{kind}: {value}" + ); + if let Some(minimum) = minimum { + assert_eq!(value["test_count_assertion"]["status"], "unproven"); + assert_eq!(value["test_count_assertion"]["minimum_tests"], minimum); + assert_eq!( + value["test_count_assertion"]["reason_code"], + "test_count_unproven" + ); + assert!(value["test_count_assertion"]["actual_tests_run"].is_null()); + } else { + assert!(value.get("test_count_assertion").is_none()); + } + } + } + } +} + +#[test] +fn validation_evidence_go_vet_preserves_historic_counters() { + use crate::tool_runtime::jobs::structured_validation_evidence; + let stderr = "warning: unused import\nerror[E0308]: mismatched types\n"; + // Preserve known Go check counter compatibility while unknown tools + // cannot borrow the old generic Rustc counter fallback. + let go = structured_validation_evidence("go_vet", "check", "", stderr, false); + assert!(go.diagnostics.is_some()); + assert_eq!(go.warnings_count, Some(1)); + assert_eq!(go.errors_count, Some(1)); +} + +#[test] +fn validation_evidence_authoritative_cargo_count_requires_matching_kind_and_tool() { + use crate::runner_protocol::ShellJobTestCountEvidence; + use webcodex_core::validation_evidence::CargoTestCountEvidenceStatus; + + let count = ShellJobTestCountEvidence { + tests_detected: true, + tests_run_count: Some(20), + status: CargoTestCountEvidenceStatus::CompleteSummary, + }; + for (tool, kind) in [ + ("cargo_test", "check"), + ("unknown_test", "test"), + ("go_test", "test"), + ] { + let value = crate::tool_runtime::jobs::validation_job_projection_with_policy( + Some(tool), + Some(kind), + "completed", + Some(0), + "", + "", + true, + Some(&count), + Some(1), + None, + None, + ) + .unwrap(); + assert_eq!(value["passed"], false, "{value}"); + assert_eq!(value["test_count_assertion"]["status"], "unproven"); + assert!(value["test_count_assertion"]["actual_tests_run"].is_null()); + } +} + +#[test] +fn validation_evidence_known_profiles_preserve_counts_failures_zero_and_truncation() { + use crate::tool_runtime::jobs::validation_job_projection; + + for (tool, stdout, passed, failed, status, exit) in [ + ("cargo_test", "test result: ok. 2 passed; 0 failed; 0 ignored\n", 2, 0, "completed", 0), + ("cargo_test", "test result: FAILED. 1 passed; 1 failed; 0 ignored\n", 1, 1, "failed", 101), + ("cargo_test", "test result: ok. 0 passed; 0 failed; 0 ignored\n", 0, 0, "completed", 0), + ("go_test", "{\"Action\":\"pass\",\"Package\":\"example\",\"Test\":\"TestOne\"}\n{\"Action\":\"pass\",\"Package\":\"example\"}\n", 1, 0, "completed", 0), + ("go_test", "{\"Action\":\"fail\",\"Package\":\"example\",\"Test\":\"TestOne\"}\n{\"Action\":\"fail\",\"Package\":\"example\"}\n", 0, 1, "failed", 1), + ("python:pytest:test", "2 passed in 0.01s\n", 2, 0, "completed", 0), + ("python:pytest:test", "1 failed, 1 passed in 0.01s\n", 1, 1, "failed", 1), + ("python:pytest:test", "no tests ran in 0.01s\n", 0, 0, "failed", 5), + ] { + for truncated in [false, true] { + let value = validation_job_projection( + Some(tool), Some("test"), status, Some(exit), stdout, "", truncated, Some(1), + ).unwrap(); + if truncated { + assert!(value["tests_run_count"].is_null(), "{value}"); + assert!(value["tests_passed"].is_null()); + assert!(value["tests_failed"].is_null()); + assert!(value["zero_tests_run"].is_null()); + } else { + assert_eq!(value["tests_run_count"], passed + failed, "{value}"); + assert_eq!(value["tests_passed"], passed, "{value}"); + assert_eq!(value["tests_failed"], failed, "{value}"); + assert_eq!(value["zero_tests_run"], passed + failed == 0); + assert_eq!(value["diagnostics"]["available"], true); + } + assert_eq!(value["passed"], exit == 0 && passed > 0 && !truncated, "{value}"); + if exit != 0 { + assert!(value.get("test_count_assertion").is_none()); + } else { + assert_eq!(value["test_count_assertion"]["status"], + if truncated { "unproven" } else if passed > 0 { "passed" } else { "failed" }); + } + } + } + // Package-only Go output cannot prove zero; preserve that existing contract. + let go_no_tests = validation_job_projection( + Some("go_test"), + Some("test"), + "completed", + Some(0), + "{\"Action\":\"pass\",\"Package\":\"example\"}\n", + "", + false, + Some(1), + ) + .unwrap(); + assert!(go_no_tests["tests_run_count"].is_null()); + assert!(go_no_tests["zero_tests_run"].is_null()); + assert_eq!(go_no_tests["passed"], false); + assert_eq!(go_no_tests["test_count_assertion"]["status"], "unproven"); +} + fn run_jobs_test_in_large_stack_thread(test: F) where F: FnOnce() -> Fut + Send + 'static, diff --git a/src/tool_runtime/tests/validation_handoff/project_validation.rs b/src/tool_runtime/tests/validation_handoff/project_validation.rs index 22417ae9a..a3eb3f06a 100644 --- a/src/tool_runtime/tests/validation_handoff/project_validation.rs +++ b/src/tool_runtime/tests/validation_handoff/project_validation.rs @@ -939,3 +939,68 @@ async fn project_validation_all_packages_dispatches_and_completes_same_wire_job( .is_none()); } } + +#[tokio::test] +async fn project_validation_mismatched_evidence_profile_starts_nothing() { + for (backend, identity) in [ + ("rust", "cargo_test"), + ("rust", "go_vet"), + ("go", "cargo_check"), + ("python", "cargo_test"), + ("rust", "unknown_check"), + ] { + let runtime = setup(1).await; + let task = tokio::spawn({ + let runtime = runtime.clone(); + async move { + runtime + .dispatch_with_auth( + call(ProjectValidationAction::Check, None), + Some(&auth_context(None, true)), + ) + .await + } + }); + let request = wait_for_runner_request(&runtime, "project-validation").await; + assert_eq!(request.kind, "plan_project_validation"); + let semantic: ProjectValidationRequest = + serde_json::from_str(request.content.as_deref().unwrap()).unwrap(); + let operation = webcodex_validation::project_validation_operation( + "rust", + webcodex_validation::SemanticCheck::Check, + None, + false, + ) + .unwrap(); + let plan = ProjectValidationPlan { + provenance: ProjectValidationProvenance { + request: semantic, + backend: backend.into(), + recipe_root: ".".into(), + root_digest: "a".repeat(64), + manifest_digest: "b".repeat(64), + invocation_digest: "c".repeat(64), + }, + adapter: identity.into(), + step: operation.build_readonly_plan().unwrap().structured_step, + validation_target_id: operation.validation_target_id(Some(".")).unwrap(), + }; + complete_sync_shell_lifecycle( + &runtime, + "project-validation", + request.request_id, + ShellCommandExecutionState::Completed, + Some(0), + &serde_json::to_string(&ProjectValidationPlanningResult::Ready { plan }).unwrap(), + "", + None, + ) + .await; + let result = task.await.unwrap(); + assert!(!result.success, "{backend}/{identity}"); + assert!(runtime.runner_registry.list_jobs(Some(10)).await.is_empty()); + assert!(probe_patch_agent_request(&runtime, "project-validation") + .await + .is_none()); + } +} diff --git a/src/tool_runtime/validation.rs b/src/tool_runtime/validation.rs index 754867080..a729e3c19 100644 --- a/src/tool_runtime/validation.rs +++ b/src/tool_runtime/validation.rs @@ -15,13 +15,13 @@ use super::structured_execution::{ StructuredExecutionBudget, StructuredJobHandoffFailure, STRUCTURED_EXECUTION_SYNC_WAIT_SECS, }; use super::tool_result::ToolResult; -#[cfg(test)] -use super::validation_profile::ValidationCommandOptions; use super::validation_profile::{ - requires_multi_package_cargo_check, runtime_profile, validation_adapter_for_tool, - CargoReadOnlyValidationOperation, ReadOnlyValidationOperation, ValidationAdapter, + requires_multi_package_cargo_check, runtime_profile, validation_evidence_profile_for_recipe, + CargoReadOnlyValidationOperation, ReadOnlyValidationOperation, ValidationEvidenceProfile, ValidationFailureEvidence, }; +#[cfg(test)] +use super::validation_profile::{validation_adapter_for_tool, ValidationCommandOptions}; use super::ToolRuntime; use crate::auth::AuthContext; use crate::runner_http::{RunnerFeature, ShellJobStartMetadata}; @@ -63,7 +63,7 @@ pub(super) fn terminal_test_count_assertion_failure_message( } pub(super) fn cargo_fmt_check_is_stable_diff( - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, output: &ProjectCommandOutput, ) -> bool { if output.execution_state != ShellCommandExecutionState::Completed @@ -533,11 +533,21 @@ impl ToolRuntime { if !plan.provenance.is_valid() || plan.provenance.request != request || !canonical_step { return ToolResult::err("invalid Runner project validation plan"); } - let Some(adapter) = validation_adapter_for_tool(&plan.adapter) else { + let semantic_action = match action { + ProjectValidationAction::FormatCheck => webcodex_validation::SemanticCheck::Format, + ProjectValidationAction::Check => webcodex_validation::SemanticCheck::Check, + ProjectValidationAction::Test => webcodex_validation::SemanticCheck::Test, + }; + let Some(adapter) = + validation_evidence_profile_for_recipe(&plan.provenance.backend, semantic_action) + else { return ToolResult::err( "validation adapter unavailable; upgrade Server and Runner together", ); }; + if adapter.tool_identity() != plan.adapter || adapter.validation_kind() != action.kind() { + return ToolResult::err("invalid Runner project validation evidence profile"); + } let source_fence = self.validation_sources.capture(&resolved.resolved_id); let provenance = plan.provenance; let relative_root = provenance.recipe_root.clone(); @@ -760,7 +770,7 @@ impl ToolRuntime { config: &crate::projects::ProjectConfig, cwd: Option<&str>, command: &str, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, step: ShellJobValidationStep, purpose: ExecutionPurpose, timeout_secs: u64, @@ -898,7 +908,7 @@ impl ToolRuntime { &self, job_id: String, sync_wait_secs: u64, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, handoff: ValidationHandoff, ) -> ToolResult { let mut guard = ValidationCleanupGuard::new( @@ -1050,7 +1060,7 @@ impl ToolRuntime { async fn recover_validation_handoff( &self, job_id: String, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, handoff: ValidationHandoff, guard: &mut ValidationCleanupGuard, ) -> ToolResult { @@ -1078,7 +1088,7 @@ impl ToolRuntime { async fn validation_terminal_result( &self, job_id: String, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, job_status: &str, handoff: ValidationHandoff, ) -> ToolResult { @@ -1231,7 +1241,7 @@ impl ToolRuntime { command: &str, cwd: Option<&str>, config: &crate::projects::ProjectConfig, - adapter: &'static dyn ValidationAdapter, + adapter: &'static dyn ValidationEvidenceProfile, output: ProjectCommandOutput, timeout_secs: u64, sync_wait_secs: u64, diff --git a/src/tool_runtime/validation_profile/mod.rs b/src/tool_runtime/validation_profile/mod.rs index ba810503d..02103a8b1 100644 --- a/src/tool_runtime/validation_profile/mod.rs +++ b/src/tool_runtime/validation_profile/mod.rs @@ -4,9 +4,10 @@ use super::helpers::{ }; pub(crate) use webcodex_validation::{ - validation_adapter_for_tool, CargoCheckOptions, CargoReadOnlyValidationOperation, + validation_adapter_for_tool, validation_evidence_profile_for_recipe, + validation_evidence_profile_for_tool, CargoCheckOptions, CargoReadOnlyValidationOperation, CargoTestOptions, GoReadOnlyValidationOperation, GoTestOptions, ReadOnlyValidationOperation, - ValidationAdapter, ValidationCommandOptions, ValidationFailureEvidence, + ValidationCommandOptions, ValidationEvidenceProfile, ValidationFailureEvidence, }; pub(crate) struct ValidationRuntimeProfile { From 6f6dd077cda7d0b84e44ab3c77bd7dc5c6e8e7a4 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Thu, 8 Oct 2026 21:50:27 +0800 Subject: [PATCH 2/5] test(validation): align passive evidence profile fixture --- src/tool_runtime/tests/jobs.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/tool_runtime/tests/jobs.rs b/src/tool_runtime/tests/jobs.rs index 444dd9336..59d2da9f8 100644 --- a/src/tool_runtime/tests/jobs.rs +++ b/src/tool_runtime/tests/jobs.rs @@ -376,6 +376,17 @@ fn passive_validation_projection_separates_execution_result_from_source_freshnes metadata.kind = "test".to_string(); job.exit_code = Some(0); job.test_count_evidence = None; + let mismatched = runtime + .passive_job_validation_projection(&job, None) + .expect("mismatched validation profile"); + assert_eq!(mismatched["passed"], false); + + // A consistent Cargo test identity without count evidence remains + // inconclusive, independently of source freshness. + let metadata = job.validation.as_mut().unwrap(); + metadata.adapter = "cargo_test".to_string(); + metadata.steps[0].name = "test".to_string(); + metadata.steps[0].args = vec!["test".to_string()]; let inconclusive = runtime .passive_job_validation_projection(&job, None) .expect("test validation projection"); From 87241f2885a6b9201f720345017c5e79b3a5a367 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:56:39 +0800 Subject: [PATCH 3/5] feat(validation): add bounded Ruff project checks --- .../src/project_validation_tests.rs | 95 ++++++ crates/webcodex-core/src/runner_protocol.rs | 8 + .../webcodex-core/src/runner_protocol/job.rs | 68 ++++- .../webcodex-core/src/validation_identity.rs | 8 + crates/webcodex-runner-config/src/lib.rs | 1 + .../src/job_updates.rs | 18 ++ .../webcodex-runner-registry/src/requests.rs | 26 +- .../src/tests/project_dependency_policy.rs | 71 +++++ .../src/tests/reconciliation.rs | 33 +++ .../job_manager/local_shell.rs | 9 + .../job_manager_tests/project_validation.rs | 116 ++++++-- .../src/webcodex_runner/shell.rs | 4 +- .../src/webcodex_runner/shell/commands.rs | 57 +++- .../src/webcodex_runner/shell/scripts.rs | 19 ++ .../src/webcodex_runner/shell_tests.rs | 272 ++++++++++++++++++ .../webcodex_runner/transport/registration.rs | 1 + .../validation/project_tests.rs | 44 ++- .../src/registry/output_schemas/testing.rs | 2 +- .../src/tool_call/canonical_enum.rs | 2 +- .../src/tool_definition/testing.rs | 2 +- crates/webcodex-validation/src/adapters/go.rs | 8 - .../webcodex-validation/src/adapters/mod.rs | 96 ++++--- .../src/adapters/python.rs | 137 +++++++++ .../webcodex-validation/src/adapters/rust.rs | 12 - .../webcodex-validation/src/profile_tests.rs | 92 +++++- crates/webcodex-validation/src/recipe.rs | 107 ++++++- .../webcodex-validation/src/recipe_tests.rs | 118 ++++++++ .../webcodex-workflow-session/src/events.rs | 26 ++ .../python-pytest-project-validation.md | 5 +- .../python-ruff-project-validation.md | 79 +++++ src/runner_quic.rs | 1 + src/runner_ws.rs | 1 + src/tool_runtime/jobs.rs | 14 +- src/tool_runtime/tests/jobs.rs | 64 +++++ src/tool_runtime/tests/metadata.rs | 1 + src/tool_runtime/tests/validation_handoff.rs | 28 +- .../validation_handoff/project_validation.rs | 3 + src/tool_runtime/validation.rs | 4 +- src/tool_runtime/validation_profile/mod.rs | 5 + 39 files changed, 1533 insertions(+), 124 deletions(-) create mode 100644 docs/implementation/python-ruff-project-validation.md diff --git a/crates/webcodex-core/src/project_validation_tests.rs b/crates/webcodex-core/src/project_validation_tests.rs index d2f2a172c..8168e0f37 100644 --- a/crates/webcodex-core/src/project_validation_tests.rs +++ b/crates/webcodex-core/src/project_validation_tests.rs @@ -318,3 +318,98 @@ fn project_validation_python_argv_identity_and_metadata_are_closed() { metadata.steps[0].args.push("--collect-only".into()); assert!(!metadata.is_valid()); } + +#[test] +fn project_ruff_step_identity_metadata_and_capability_are_independent() { + use crate::runner_protocol::{ + RunnerCapabilities, ShellJobValidationMetadata, ShellJobValidationStep, + }; + use crate::validation_identity::{ + structured_validation_target_identity, ToolValidationIdentityKind, + }; + let capabilities: RunnerCapabilities = + serde_json::from_value(serde_json::json!({"project_validation_python_pytest_v1":true})) + .unwrap(); + assert!(!capabilities.project_validation_python_ruff_v1); + let mut identities = std::collections::HashSet::new(); + for (action, kind, identity_kind) in [ + ( + ProjectValidationAction::Check, + "check", + ToolValidationIdentityKind::PythonRuffCheck, + ), + ( + ProjectValidationAction::FormatCheck, + "format", + ToolValidationIdentityKind::PythonRuffFormat, + ), + ] { + let step = ShellJobValidationStep::python_ruff(kind).unwrap(); + assert!(step.is_canonical() && step.is_structured_ruff()); + assert_eq!(&step.args[..4], ["-I", "-B", "-m", "ruff"]); + for removed in 0..step.args.len() { + let mut bad = step.clone(); + bad.args.remove(removed); + assert!(!bad.is_structured_ruff()); + } + for extra in [ + "--fix", + "--fix-only", + "--output-file=report.json", + "--config=../ruff.toml", + "--isolated", + "src", + ] { + let mut bad = step.clone(); + bad.args.push(extra.into()); + assert!(!bad.is_canonical()); + } + let identity = + structured_validation_target_identity(identity_kind, &serde_json::json!({"cwd":"."})) + .unwrap(); + assert!(identities.insert(identity.clone())); + let mut metadata = ShellJobValidationMetadata { + tool: "project_validate".into(), + kind: kind.into(), + steps: vec![step], + effective_timeout_secs: 60, + sync_wait_secs: 1, + adapter: identity_kind.tool_name().unwrap().into(), + validation_target_id: Some(identity), + source_fence: None, + minimum_tests: None, + require_tests: None, + no_run: None, + project_validation: Some(ProjectValidationProvenance { + request: ProjectValidationRequest { + project_id: "demo".into(), + cwd: None, + action, + adapter: ProjectValidationAdapter::Python, + scope: None, + dependency_policy: None, + test: None, + }, + backend: "python".into(), + recipe_root: ".".into(), + root_digest: "a".repeat(64), + manifest_digest: "b".repeat(64), + invocation_digest: "c".repeat(64), + }), + }; + assert!(metadata.is_valid()); + metadata.minimum_tests = Some(1); + assert!(!metadata.is_valid()); + metadata.minimum_tests = None; + metadata.steps[0] + .env + .push(("CARGO_TARGET_DIR".into(), "/tmp/cache".into())); + assert!(!metadata.is_valid()); + metadata.steps[0].env.clear(); + metadata.adapter = "python:pytest:test".into(); + assert!(!metadata.is_valid()); + metadata.adapter = identity_kind.tool_name().unwrap().into(); + metadata.project_validation = None; + assert!(!metadata.is_valid()); + } +} diff --git a/crates/webcodex-core/src/runner_protocol.rs b/crates/webcodex-core/src/runner_protocol.rs index 81cc7f1d6..eaa41b1c7 100644 --- a/crates/webcodex-core/src/runner_protocol.rs +++ b/crates/webcodex-core/src/runner_protocol.rs @@ -609,6 +609,13 @@ runner_capabilities! { #[serde(default, skip_serializing_if = "is_false")] pub project_validation_python_pytest_v1: bool = false; } + /// Read-only project Ruff check/format with pinned local configuration, + /// exact Python preflight and independent evidence identities. + ProjectValidationPythonRuff => RUNNER_CAPABILITY_PROJECT_VALIDATION_PYTHON_RUFF("project_validation_python_ruff_v1"), + v2_baseline = false { + #[serde(default, skip_serializing_if = "is_false")] + pub project_validation_python_ruff_v1: bool = false; + } /// The Runner understands the first-class model-facing `go_test` tool identity /// and its durable `ShellJobValidationMetadata` contract. This is deliberately /// separate from Go JSON parsing support: older Runners may advertise @@ -2832,6 +2839,7 @@ mod envelope_tests { project_all_packages_v1: false, project_validation_test_options_v1: false, project_validation_python_pytest_v1: false, + project_validation_python_ruff_v1: false, structured_go_test_tool: true, structured_go_test_packages: true, structured_process_argv: true, diff --git a/crates/webcodex-core/src/runner_protocol/job.rs b/crates/webcodex-core/src/runner_protocol/job.rs index d62b27d76..ff59da6d6 100644 --- a/crates/webcodex-core/src/runner_protocol/job.rs +++ b/crates/webcodex-core/src/runner_protocol/job.rs @@ -246,9 +246,15 @@ impl ShellJobValidationStep { ("check", "go") => is_canonical_go_vet_args(&args), ("test", "go") => args == ["test", "./..."] || self.is_structured_go_test_json(), ("format", "python") => { - args == ["-m", "ruff", "format", "--check"] || args == ["-m", "black", "--check"] + args == ["-m", "ruff", "format", "--check"] + || args == ["-m", "black", "--check"] + || self.is_structured_ruff() + } + ("check", "python") => { + args == ["-m", "ruff", "check"] + || args == ["-m", "mypy"] + || self.is_structured_ruff() } - ("check", "python") => args == ["-m", "ruff", "check"] || args == ["-m", "mypy"], ("test", "python") => { args == ["-m", "pytest"] || args == ["-B", "-m", "unittest", "discover", "-v"] @@ -264,6 +270,53 @@ impl ShellJobValidationStep { } } + /// Exact Runner-owned Ruff argv. Callers must additionally prove the local + /// manifest and project provenance; this constructor grants no authority. + pub fn python_ruff(kind: &str) -> Option { + let args: &[&str] = match kind { + "check" => &[ + "-I", + "-B", + "-m", + "ruff", + "check", + "--no-fix", + "--no-fix-only", + "--no-cache", + "--no-respect-gitignore", + "--output-format", + "json-lines", + "--config", + "pyproject.toml", + ".", + ], + "format" => &[ + "-I", + "-B", + "-m", + "ruff", + "format", + "--check", + "--no-cache", + "--no-respect-gitignore", + "--config", + "pyproject.toml", + ".", + ], + _ => return None, + }; + Some(Self { + name: kind.into(), + program: "python".into(), + args: args.iter().map(|arg| (*arg).into()).collect(), + env: Vec::new(), + }) + } + + pub fn is_structured_ruff(&self) -> bool { + Self::python_ruff(&self.name).as_ref() == Some(self) + } + /// Exact project pytest argv; no executable, free flags or environment input. pub fn is_structured_pytest(&self) -> bool { if self.name != "test" || self.program != "python" || !self.env.is_empty() { @@ -872,7 +925,10 @@ impl ShellJobValidationMetadata { (provenance.backend.as_str(), self.adapter.as_str()), ("rust", "cargo_fmt" | "cargo_check" | "cargo_test") | ("go", "go_vet" | "go_test") - | ("python", "python:pytest:test") + | ( + "python", + "python:pytest:test" | "python:ruff:check" | "python:ruff:format" + ) ) || (self.require_tests, self.minimum_tests) != provenance.request.test_requirements() @@ -898,6 +954,12 @@ impl ShellJobValidationMetadata { && self.kind == "test" && step.is_structured_pytest() } + "python:ruff:check" | "python:ruff:format" => { + self.tool == "project_validate" + && self.adapter == format!("python:ruff:{}", self.kind) + && self.kind == step.name + && step.is_structured_ruff() + } "go_vet" => self.kind == "check" && step.name == "check" && step.program == "go", _ => false, } diff --git a/crates/webcodex-core/src/validation_identity.rs b/crates/webcodex-core/src/validation_identity.rs index 75eb65aa5..7e451cf0c 100644 --- a/crates/webcodex-core/src/validation_identity.rs +++ b/crates/webcodex-core/src/validation_identity.rs @@ -23,6 +23,8 @@ pub enum ToolValidationIdentityKind { GoTest, GoVet, PythonPytest, + PythonRuffCheck, + PythonRuffFormat, Project, } @@ -36,6 +38,8 @@ impl ToolValidationIdentityKind { Self::GoTest => Some("go_test"), Self::GoVet => Some("go_vet"), Self::PythonPytest => Some("python:pytest:test"), + Self::PythonRuffCheck => Some("python:ruff:check"), + Self::PythonRuffFormat => Some("python:ruff:format"), } } } @@ -206,6 +210,10 @@ pub fn structured_validation_target_identity( } semantic } + ToolValidationIdentityKind::PythonRuffCheck + | ToolValidationIdentityKind::PythonRuffFormat => { + serde_json::json!({"tool":tool_name,"kind": if kind == ToolValidationIdentityKind::PythonRuffCheck { "check" } else { "format" },"cwd":cwd}) + } ToolValidationIdentityKind::PythonPytest => { let filter = match obj.get("filter") { None | Some(Value::Null) => None, diff --git a/crates/webcodex-runner-config/src/lib.rs b/crates/webcodex-runner-config/src/lib.rs index e67c7172c..3c08cb72d 100644 --- a/crates/webcodex-runner-config/src/lib.rs +++ b/crates/webcodex-runner-config/src/lib.rs @@ -297,6 +297,7 @@ pub fn generated_runner_config_toml(opts: &RunnerInitOptions) -> Result, ) -> Result<(String, oneshot::Receiver), String> { payload.validate()?; - let requires_python = - payload.adapter == webcodex_core::project_validation::ProjectValidationAdapter::Python; + let python_capability = (payload.adapter + == webcodex_core::project_validation::ProjectValidationAdapter::Python) + .then_some( + if payload.action + == webcodex_core::project_validation::ProjectValidationAction::Test + { + RunnerFeature::ProjectValidationPythonPytest + } else { + RunnerFeature::ProjectValidationPythonRuff + }, + ); let requires_package_scope = payload .scope .as_ref() @@ -2713,15 +2722,10 @@ impl RunnerRegistry { RunnerFeature::ProjectValidation, )); } - if requires_python - && !runner - .runner_features - .supports(RunnerFeature::ProjectValidationPythonPytest) - { - return Err(capability_upgrade_error( - &client_id, - RunnerFeature::ProjectValidationPythonPytest, - )); + if let Some(capability) = python_capability { + if !runner.runner_features.supports(capability) { + return Err(capability_upgrade_error(&client_id, capability)); + } } if requires_all_packages && !runner diff --git a/crates/webcodex-runner-registry/src/tests/project_dependency_policy.rs b/crates/webcodex-runner-registry/src/tests/project_dependency_policy.rs index e50827a0c..26d0eb3a6 100644 --- a/crates/webcodex-runner-registry/src/tests/project_dependency_policy.rs +++ b/crates/webcodex-runner-registry/src/tests/project_dependency_policy.rs @@ -225,6 +225,24 @@ async fn project_validation_base_capability_error_identifies_runner_without_job_ #[tokio::test] async fn project_validation_optional_capability_errors_identify_runner_without_dispatch() { let cases = [ + ( + RunnerCapabilityId::ProjectValidationPythonRuff, + ProjectValidationRequest { + action: ProjectValidationAction::Check, + adapter: ProjectValidationAdapter::Python, + dependency_policy: None, + ..locked_validation_request() + }, + ), + ( + RunnerCapabilityId::ProjectValidationPythonRuff, + ProjectValidationRequest { + action: ProjectValidationAction::FormatCheck, + adapter: ProjectValidationAdapter::Python, + dependency_policy: None, + ..locked_validation_request() + }, + ), ( RunnerCapabilityId::ProjectValidationPythonPytest, ProjectValidationRequest { @@ -436,3 +454,56 @@ async fn dependency_policy_is_fenced_again_at_project_validation_job_admission() } } } + +#[tokio::test] +async fn project_validation_python_preplan_selects_capability_by_action() { + for action in [ + ProjectValidationAction::Test, + ProjectValidationAction::Check, + ProjectValidationAction::FormatCheck, + ] { + for supported in [false, true] { + let registry = RunnerRegistry::default(); + let mut runner = registration(true); + runner.capabilities.project_validation_python_pytest_v1 = + if action == ProjectValidationAction::Test { + supported + } else { + !supported + }; + runner.capabilities.project_validation_python_ruff_v1 = + if action == ProjectValidationAction::Test { + !supported + } else { + supported + }; + registry.register(runner).await.unwrap(); + let result = registry + .enqueue_project_validation_plan( + "policy-runner".into(), + ProjectValidationRequest { + action, + adapter: ProjectValidationAdapter::Python, + dependency_policy: None, + ..locked_validation_request() + }, + Some(&auth_context(None, true)), + ) + .await; + assert_eq!( + result.is_ok(), + supported, + "{action:?}, supported={supported}" + ); + let queued = registry + .poll(RunnerPollRequest { + client_id: "policy-runner".into(), + runner_instance_id: "inst".into(), + }) + .await + .unwrap(); + assert_eq!(queued.is_some(), supported); + assert!(registry.list_jobs(Some(10)).await.is_empty()); + } + } +} diff --git a/crates/webcodex-runner-registry/src/tests/reconciliation.rs b/crates/webcodex-runner-registry/src/tests/reconciliation.rs index f17117a7a..5b6d4ab0a 100644 --- a/crates/webcodex-runner-registry/src/tests/reconciliation.rs +++ b/crates/webcodex-runner-registry/src/tests/reconciliation.rs @@ -3956,3 +3956,36 @@ async fn project_validation_python_pytest_capability_rechecked_at_job_admission( } } } + +#[tokio::test] +async fn project_validation_ruff_admission_requires_ruff_capability_even_with_pytest() { + use webcodex_core::project_validation::*; + for action in [ProjectValidationAction::Check, ProjectValidationAction::FormatCheck] { + for supported in [false, true] { + let registry = RunnerRegistry::default(); + let mut registration = register_request(INSTANCE_A, empty_inventory()); + registration.capabilities.project_validation_v1 = true; + registration.capabilities.project_validation_python_pytest_v1 = !supported; + registration.capabilities.project_validation_python_ruff_v1 = supported; + registry.register(registration).await.unwrap(); + let mut metadata = cargo_validation_start_metadata(None, None, None); + let validation = metadata.validation.as_mut().unwrap(); + validation.tool = "project_validate".into(); + validation.adapter = format!("python:ruff:{}", action.kind()); + validation.kind = action.kind().into(); + validation.steps = vec![ShellJobValidationStep::python_ruff(action.kind()).unwrap()]; + validation.project_validation = Some(ProjectValidationProvenance { + request: ProjectValidationRequest { project_id: "demo".into(), cwd: None, action, adapter: ProjectValidationAdapter::Auto, scope: None, dependency_policy: None, test: None }, + backend: "python".into(), recipe_root: ".".into(), root_digest: "a".repeat(64), manifest_digest: "b".repeat(64), invocation_digest: "c".repeat(64), + }); + assert!(validation.is_valid()); + metadata.validation_steps = validation.steps.clone(); + let result = registry.start_job_with_metadata(start_request("validation"), "tester".into(), metadata).await; + if supported { assert!(result.is_ok(), "{result:?}"); } + else { + assert!(result.unwrap_err().contains("project_validation_python_ruff_v1")); + assert!(registry.list_jobs(Some(10)).await.is_empty()); + } + } + } +} diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager/local_shell.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager/local_shell.rs index 469673395..a68cca6af 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager/local_shell.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager/local_shell.rs @@ -119,6 +119,7 @@ impl JobManager { if validation && steps.iter().any(|step| { !step.is_structured_pytest() + && !step.is_structured_ruff() && !validation_module_available( &shell, prepared_profile.as_deref(), @@ -150,6 +151,14 @@ impl JobManager { &cwd_path, Some(self.shutting_down.as_ref()), ) + } else if validation && steps[index].is_structured_ruff() { + crate::webcodex_runner::shell::configured_ruff_job_command( + &shell, + prepared_profile.as_deref(), + &steps[index], + &cwd_path, + Some(self.shutting_down.as_ref()), + ) } else if validation { configured_validation_job_command( &shell, diff --git a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/project_validation.rs b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/project_validation.rs index 32b8dc480..a35c6d69c 100644 --- a/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/project_validation.rs +++ b/crates/webcodex-runner/src/webcodex_runner/job_manager_tests/project_validation.rs @@ -9,15 +9,29 @@ use webcodex_core::runner_protocol::{ShellJobContext, ShellJobValidationMetadata #[test] fn project_validation_rechecks_manifest_and_dependency_state_after_queueing() { - for (marker, dependency, change_dependency, member_cwd) in [ - ("Cargo.toml", "Cargo.lock", false, false), - ("Cargo.toml", "Cargo.lock", true, false), - ("go.mod", "go.sum", false, false), - ("go.mod", "go.sum", true, false), - ("pyproject.toml", "pytest.ini", false, false), - ("pyproject.toml", "pytest.ini", true, false), - ("Cargo.toml", "Cargo.lock", false, true), - ("Cargo.toml", "Cargo.lock", true, true), + for (marker, dependency, change_dependency, member_cwd, ruff_action) in [ + ("Cargo.toml", "Cargo.lock", false, false, None), + ("Cargo.toml", "Cargo.lock", true, false, None), + ("go.mod", "go.sum", false, false, None), + ("go.mod", "go.sum", true, false, None), + ("pyproject.toml", "pytest.ini", false, false, None), + ("pyproject.toml", "pytest.ini", true, false, None), + ("Cargo.toml", "Cargo.lock", false, true, None), + ("Cargo.toml", "Cargo.lock", true, true, None), + ( + "pyproject.toml", + "unused", + false, + false, + Some(ProjectValidationAction::Check), + ), + ( + "pyproject.toml", + "unused", + false, + false, + Some(ProjectValidationAction::FormatCheck), + ), ] { let temp = tempfile::tempdir().unwrap(); let root = temp.path().join("project"); @@ -44,7 +58,15 @@ edition = "2021" ) .unwrap(); } else { - std::fs::write(root.join(marker), "").unwrap(); + std::fs::write( + root.join(marker), + if ruff_action.is_some() { + "[tool.ruff]\ntarget-version='py311'\n" + } else { + "" + }, + ) + .unwrap(); } std::fs::write(root.join(dependency), "").unwrap(); std::fs::write( @@ -63,7 +85,9 @@ edition = "2021" let request = ProjectValidationRequest { project_id: "demo".into(), cwd: member_cwd.then(|| "member".into()), - action: if marker == "pyproject.toml" { + action: if let Some(action) = ruff_action { + action + } else if marker == "pyproject.toml" { ProjectValidationAction::Test } else { ProjectValidationAction::Check @@ -358,6 +382,15 @@ fn enqueue_python_project_fixture( shell: ShellConfig, body: &str, filter: Option<&str>, +) -> (tempfile::TempDir, JobManager) { + enqueue_python_validation_fixture(shell, body, filter, ProjectValidationAction::Test) +} + +fn enqueue_python_validation_fixture( + shell: ShellConfig, + body: &str, + filter: Option<&str>, + action: ProjectValidationAction, ) -> (tempfile::TempDir, JobManager) { use webcodex_core::project_validation::ProjectValidationTestOptions; let temp = tempfile::tempdir().unwrap(); @@ -365,7 +398,11 @@ fn enqueue_python_project_fixture( let registry = temp.path().join("registry"); std::fs::create_dir_all(&root).unwrap(); std::fs::create_dir_all(®istry).unwrap(); - std::fs::write(root.join("pyproject.toml"), "[tool.pytest.ini_options]\n").unwrap(); + std::fs::write( + root.join("pyproject.toml"), + "[tool.pytest.ini_options]\n[tool.ruff]\ntarget-version='py311'\n", + ) + .unwrap(); std::fs::write(root.join("test_example.py"), body).unwrap(); std::fs::write( registry.join("demo.toml"), @@ -382,7 +419,7 @@ fn enqueue_python_project_fixture( let request = ProjectValidationRequest { project_id: "demo".into(), cwd: None, - action: ProjectValidationAction::Test, + action, adapter: ProjectValidationAdapter::Python, scope: None, dependency_policy: None, @@ -396,15 +433,15 @@ fn enqueue_python_project_fixture( let metadata = ShellJobValidationMetadata { project_validation: Some(plan.provenance), tool: "project_validate".into(), - kind: "test".into(), + kind: action.kind().into(), steps: vec![plan.step.clone()], effective_timeout_secs: 60, sync_wait_secs: 1, adapter: plan.adapter, validation_target_id: Some(plan.validation_target_id), source_fence: None, - minimum_tests: Some(1), - require_tests: Some(true), + minimum_tests: request.test_requirements().1, + require_tests: request.test_requirements().0, no_run: None, }; assert!(metadata.is_valid()); @@ -415,10 +452,14 @@ fn enqueue_python_project_fixture( ssh_resource: None, project_cwd: Some(".".into()), cwd: Some(cwd.to_string_lossy().into_owned()), - purpose: Some("test".into()), + purpose: Some( + webcodex_validation::execution_purpose_for_validation_kind(action.kind()) + .as_str() + .into(), + ), shell: None, - command_preview: "pytest validation".into(), - validation_steps: vec!["test".into()], + command_preview: "Python validation".into(), + validation_steps: vec![plan.step.name.clone()], structured_execution: None, }; let operation = RunnerJobOperation::StartValidation(RunnerJobValidationOperation { @@ -550,3 +591,40 @@ fn runner_real_process_project_validation_python_pytest_fixtures() { assert_eq!(snapshot.exit_code, Some(0), "{snapshot:?}"); assert_eq!(snapshot.stdout.tail.trim(), "1 passed in 0.01s"); } + +#[test] +fn project_validation_ruff_missing_interpreter_is_definitely_not_started() { + for action in [ + ProjectValidationAction::Check, + ProjectValidationAction::FormatCheck, + ] { + let bin = tempfile::tempdir().unwrap(); + let mut shell = ShellConfig::default(); + shell + .env + .insert("PATH".into(), bin.path().to_string_lossy().into_owned()); + let (_temp, manager) = enqueue_python_validation_fixture( + shell, + "raise RuntimeError('must not run')\n", + None, + action, + ); + assert!(manager.wait_for_workers(Instant::now() + Duration::from_secs(10))); + let snapshot = lock_unpoison(&manager.jobs)["pytest-fixture-job"] + .snapshot + .clone(); + manager.stop_all(); + assert_eq!( + snapshot.command_execution_state, + Some(ShellCommandExecutionState::NotStarted) + ); + assert_eq!( + snapshot.error.as_deref(), + Some(VALIDATION_TOOL_UNAVAILABLE_CODE) + ); + assert_eq!( + snapshot.context.validation.unwrap().adapter, + format!("python:ruff:{}", action.kind()) + ); + } +} diff --git a/crates/webcodex-runner/src/webcodex_runner/shell.rs b/crates/webcodex-runner/src/webcodex_runner/shell.rs index 9f227cb94..1b3f071ba 100644 --- a/crates/webcodex-runner/src/webcodex_runner/shell.rs +++ b/crates/webcodex-runner/src/webcodex_runner/shell.rs @@ -127,8 +127,8 @@ mod search; pub(crate) use commands::shell_quote_powershell; pub(crate) use commands::{ configured_explicit_shell_command, configured_prepared_shell_job_command, - configured_pytest_job_command, configured_shell_job_command, configured_validation_job_command, - explicit_shell_available, shell_quote, + configured_pytest_job_command, configured_ruff_job_command, configured_shell_job_command, + configured_validation_job_command, explicit_shell_available, shell_quote, }; use commands::{ configured_prepared_shell_command, configured_process_command, configured_process_path, diff --git a/crates/webcodex-runner/src/webcodex_runner/shell/commands.rs b/crates/webcodex-runner/src/webcodex_runner/shell/commands.rs index 054fdcae5..da82a33e3 100644 --- a/crates/webcodex-runner/src/webcodex_runner/shell/commands.rs +++ b/crates/webcodex-runner/src/webcodex_runner/shell/commands.rs @@ -1,6 +1,7 @@ //! Shell/process command construction and executable resolution. use super::*; +use webcodex_core::runner_protocol::ShellJobValidationStep; /// POSIX sh single-quote escaping. pub(crate) fn shell_quote(value: &str) -> String { @@ -195,17 +196,49 @@ pub(crate) fn configured_pytest_job_command( args: &[String], cwd: &Path, stop_requested: Option<&AtomicBool>, +) -> Result { + configured_python_module_job_command(shell, profile, args, cwd, stop_requested, false) +} + +pub(crate) fn configured_ruff_job_command( + shell: &ShellConfig, + profile: Option<&PreparedShellProfile>, + step: &ShellJobValidationStep, + cwd: &Path, + stop_requested: Option<&AtomicBool>, +) -> Result { + if !step.is_structured_ruff() { + return Err("invalid Ruff validation step".into()); + } + configured_python_module_job_command(shell, profile, &step.args, cwd, stop_requested, true) +} + +fn configured_python_module_job_command( + shell: &ShellConfig, + profile: Option<&PreparedShellProfile>, + args: &[String], + cwd: &Path, + stop_requested: Option<&AtomicBool>, + ruff: bool, ) -> Result { let unavailable = || webcodex_core::runner_protocol::VALIDATION_TOOL_UNAVAILABLE_CODE.to_string(); - let program = configured_script_interpreter(shell, profile, ShellScriptLanguage::Python) - .map_err(|_| unavailable())?; + let program = if ruff { + super::scripts::configured_validation_python_interpreter(shell, profile) + } else { + super::scripts::configured_script_interpreter(shell, profile, ShellScriptLanguage::Python) + } + .map_err(|_| unavailable())?; let mut probe = Command::new(&program); const PROBE: &str = "import sys,importlib.util;sys.exit(0 if sys.version_info.major == 3 and importlib.util.find_spec('pytest') else 42)"; - // Keep the same profile PYTHONPATH/module search environment as the actual - // pytest process. Isolation here would reject explicitly configured tooling. + const RUFF_PROBE: &str = "import sys,importlib.util;sys.exit(0 if sys.version_info.major == 3 and importlib.util.find_spec('ruff') else 42)"; + if ruff { + probe.args(["-I", "-B"]); + } + // Ruff isolates both probe and spawn from project/PYTHONPATH impersonation. + // Pytest preserves the configured module search environment for both. probe - .args(["-c", PROBE]) + .args(["-c", if ruff { RUFF_PROBE } else { PROBE }]) .current_dir(cwd) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -215,7 +248,12 @@ pub(crate) fn configured_pytest_job_command( // PYTEST_ADDOPTS is parsed as additional command-line argv before pytest // resolves rootdir/config. Structured validation owns the complete argv, so // ambient profile/shell values must not widen selection or inject flags. - probe.env_remove("PYTEST_ADDOPTS"); + if ruff { + probe.env_remove("RUFF_OUTPUT_FILE"); + probe.env("PYTHONDONTWRITEBYTECODE", "1"); + } else { + probe.env_remove("PYTEST_ADDOPTS"); + } if stop_requested.is_some_and(|flag| flag.load(Ordering::SeqCst)) { return Err(unavailable()); } @@ -241,7 +279,12 @@ pub(crate) fn configured_pytest_job_command( command.args(args); super::scripts::apply_script_environment(&mut command, shell, profile) .map_err(|_| unavailable())?; - command.env_remove("PYTEST_ADDOPTS"); + if ruff { + command.env_remove("RUFF_OUTPUT_FILE"); + command.env("PYTHONDONTWRITEBYTECODE", "1"); + } else { + command.env_remove("PYTEST_ADDOPTS"); + } Ok(command) } diff --git a/crates/webcodex-runner/src/webcodex_runner/shell/scripts.rs b/crates/webcodex-runner/src/webcodex_runner/shell/scripts.rs index f381aab2f..faaa216a1 100644 --- a/crates/webcodex-runner/src/webcodex_runner/shell/scripts.rs +++ b/crates/webcodex-runner/src/webcodex_runner/shell/scripts.rs @@ -6,6 +6,22 @@ pub(super) fn configured_script_interpreter( shell: &ShellConfig, profile: Option<&PreparedShellProfile>, language: ShellScriptLanguage, +) -> Result { + select_script_interpreter(shell, profile, language, false) +} + +pub(super) fn configured_validation_python_interpreter( + shell: &ShellConfig, + profile: Option<&PreparedShellProfile>, +) -> Result { + select_script_interpreter(shell, profile, ShellScriptLanguage::Python, true) +} + +fn select_script_interpreter( + shell: &ShellConfig, + profile: Option<&PreparedShellProfile>, + language: ShellScriptLanguage, + pin_configured: bool, ) -> Result { let configured_program = profile .map(|profile| profile.program.as_str()) @@ -60,6 +76,9 @@ pub(super) fn configured_script_interpreter( candidates.push("node".to_string()) } } + if pin_configured && configured_matches { + candidates.truncate(1); + } candidates.dedup_by(|left, right| { if cfg!(windows) { left.eq_ignore_ascii_case(right) diff --git a/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs b/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs index 8df5f2c1b..877239730 100644 --- a/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs @@ -2450,6 +2450,51 @@ fn project_validation_pytest_probe_uses_profile_python3_and_pins_spawn_program() assert_eq!(std::fs::read_to_string(capture).unwrap(), "probespawn"); } +#[cfg(unix)] +#[test] +fn project_validation_pytest_missing_configured_python3_uses_profile_path() { + use std::os::unix::fs::PermissionsExt; + let temp = crate::tests::executable_tempdir(); + let python = temp.path().join("python3"); + let capture = temp.path().join("probe"); + std::fs::write(&python,"#!/bin/sh\nif [ \"$1\" = '-c' ]; then printf 'probe' > \"$CAPTURE\"; exit 0; fi\nif [ -n \"${PYTEST_ADDOPTS+x}\" ]; then printf 'leaked' >> \"$CAPTURE\"; else printf 'spawn' >> \"$CAPTURE\"; fi\n").unwrap(); + std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o700)).unwrap(); + let profile = PreparedShellProfile { + profile_name: "python-profile".into(), + program: temp + .path() + .join("missing/python3") + .to_string_lossy() + .into_owned(), + args: vec![], + dialect: ShellDialect::Posix, + env_snapshot: std::collections::HashMap::from([ + ("PATH".into(), temp.path().to_string_lossy().into_owned()), + ("CAPTURE".into(), capture.to_string_lossy().into_owned()), + ( + "PYTEST_ADDOPTS".into(), + "-c ../../outside/pytest.ini".into(), + ), + ]), + }; + let mut command = configured_pytest_job_command( + &ShellConfig::default(), + Some(&profile), + &["-m", "pytest", "--color=no", "-rA"].map(str::to_string), + temp.path(), + None, + ) + .unwrap(); + assert_eq!(command.get_program(), python.as_os_str()); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut child = ManagedChild::spawn(&mut command).unwrap(); + assert!(child.wait().unwrap().success()); + assert_eq!(std::fs::read_to_string(capture).unwrap(), "probespawn"); +} + #[cfg(unix)] #[test] fn project_validation_pytest_missing_module_never_falls_back_or_spawns_tests() { @@ -2483,3 +2528,230 @@ fn project_validation_pytest_missing_module_never_falls_back_or_spawns_tests() { ); assert_eq!(std::fs::read_to_string(capture).unwrap(), "probe"); } + +// These are interpreter/argv fixtures implemented by a shell stub, not Ruff CLI proof. +#[cfg(unix)] +#[test] +fn project_validation_ruff_probe_and_spawn_pin_profile_python_and_environment() { + use std::os::unix::fs::PermissionsExt; + use webcodex_core::runner_protocol::ShellJobValidationStep; + let temp = crate::tests::executable_tempdir(); + let python = temp.path().join("python3"); + let capture = temp.path().join("probe"); + std::fs::write( + &python, + r#"#!/bin/sh +[ "$1" = '-I' ] || exit 40 +[ "$2" = '-B' ] || exit 41 +[ "$PYTHONDONTWRITEBYTECODE" = '1' ] || exit 42 +[ "$PYTHONPATH" = 'configured-module-path' ] || exit 43 +[ -z "${RUFF_OUTPUT_FILE+x}" ] || exit 44 +if [ "$3" = '-c' ]; then + case "$4" in *"find_spec('ruff')"*) printf 'probe' > "$CAPTURE";; *) exit 45;; esac +else + printf 'spawn' >> "$CAPTURE" +fi +"#, + ) + .unwrap(); + std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o700)).unwrap(); + let profile = PreparedShellProfile { + profile_name: "ruff-profile".into(), + program: "bash".into(), + args: vec![], + dialect: ShellDialect::Posix, + env_snapshot: std::collections::HashMap::from([ + ("PATH".into(), temp.path().to_string_lossy().into_owned()), + ("CAPTURE".into(), capture.to_string_lossy().into_owned()), + ("PYTHONPATH".into(), "configured-module-path".into()), + ("RUFF_OUTPUT_FILE".into(), "must-not-write".into()), + ("PYTHONDONTWRITEBYTECODE".into(), "0".into()), + ]), + }; + for kind in ["check", "format"] { + let step = ShellJobValidationStep::python_ruff(kind).unwrap(); + let mut command = configured_ruff_job_command( + &ShellConfig::default(), + Some(&profile), + &step, + temp.path(), + None, + ) + .unwrap(); + assert_eq!(command.get_program(), python.as_os_str()); + assert_eq!( + command.get_args().collect::>(), + step.args + .iter() + .map(std::ffi::OsStr::new) + .collect::>() + ); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut child = ManagedChild::spawn(&mut command).unwrap(); + assert!(child.wait().unwrap().success()); + assert_eq!(std::fs::read_to_string(&capture).unwrap(), "probespawn"); + } +} + +#[cfg(unix)] +#[test] +fn project_validation_python_invalid_configured_interpreter_and_missing_ruff_never_fall_back() { + use std::os::unix::fs::PermissionsExt; + use webcodex_core::runner_protocol::ShellJobValidationStep; + let temp = crate::tests::executable_tempdir(); + let capture = temp.path().join("fallback"); + for (name, script) in [ + ( + "python3", + "#!/bin/sh\nprintf probe > \"$CAPTURE\"\nexit 42\n", + ), + ( + "python", + "#!/bin/sh\nprintf fallback > \"$CAPTURE\"\nexit 0\n", + ), + ] { + let path = temp.path().join(name); + std::fs::write(&path, script).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700)).unwrap(); + } + let mut profile = PreparedShellProfile { + profile_name: "ruff-profile".into(), + program: temp + .path() + .join("missing/python3") + .to_string_lossy() + .into_owned(), + args: vec![], + dialect: ShellDialect::Posix, + env_snapshot: std::collections::HashMap::from([ + ("PATH".into(), temp.path().to_string_lossy().into_owned()), + ("CAPTURE".into(), capture.to_string_lossy().into_owned()), + ]), + }; + for configured in [true, false] { + if !configured { + profile.program = "bash".into(); + } + let result = configured_ruff_job_command( + &ShellConfig::default(), + Some(&profile), + &ShellJobValidationStep::python_ruff("check").unwrap(), + temp.path(), + None, + ); + assert_eq!( + result.unwrap_err(), + webcodex_core::runner_protocol::VALIDATION_TOOL_UNAVAILABLE_CODE + ); + if configured { + assert!(!capture.exists()); + } else { + assert_eq!(std::fs::read_to_string(&capture).unwrap(), "probe"); + std::fs::remove_file(&capture).unwrap(); + } + } + // Ruff keeps configured Python authoritative even with a PATH alternative. + profile.program = temp + .path() + .join("missing/python") + .to_string_lossy() + .into_owned(); + assert!(configured_ruff_job_command( + &ShellConfig::default(), + Some(&profile), + &ShellJobValidationStep::python_ruff("check").unwrap(), + temp.path(), + None + ) + .is_err()); + assert!(!capture.exists()); +} + +#[cfg(unix)] +#[test] +fn project_validation_ruff_system_python_ignores_project_and_pythonpath_impostors() { + use webcodex_core::runner_protocol::ShellJobValidationStep; + // Use a real system interpreter, never an interpreter supplied by the fixture. + let python = ["/usr/bin/python3", "/usr/local/bin/python3"] + .into_iter() + .find(|path| Path::new(path).is_file()) + .expect("system Python 3 is required for the Ruff isolation regression"); + let installed_ruff = Command::new(python) + .args(["-I", "-B", "-c", "import importlib.util; raise SystemExit(0 if importlib.util.find_spec('ruff') else 42)"]) + .env_clear() + .output().unwrap().status.success(); + for (on_pythonpath, package) in [(false, false), (false, true), (true, false), (true, true)] { + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let injected = temp.path().join("injected"); + std::fs::create_dir(&project).unwrap(); + std::fs::create_dir(&injected).unwrap(); + let marker = project.join("impostor-executed"); + let root = if on_pythonpath { &injected } else { &project }; + let malicious = + "from pathlib import Path\nPath('impostor-executed').touch()\nraise SystemExit(0)\n"; + if package { + std::fs::create_dir(root.join("ruff")).unwrap(); + std::fs::write(root.join("ruff/__init__.py"), malicious).unwrap(); + std::fs::write(root.join("ruff/__main__.py"), malicious).unwrap(); + } else { + std::fs::write(root.join("ruff.py"), malicious).unwrap(); + } + // An installed genuine Ruff must fail too; an impostor would falsely succeed. + std::fs::write(project.join("pyproject.toml"), "[invalid TOML").unwrap(); + let profile = PreparedShellProfile { + profile_name: "system-python".into(), + program: python.into(), + args: vec![], + dialect: ShellDialect::Posix, + env_snapshot: std::collections::HashMap::from([( + "PYTHONPATH".into(), + injected.to_string_lossy().into_owned(), + )]), + }; + for kind in ["check", "format"] { + let step = ShellJobValidationStep::python_ruff(kind).unwrap(); + match configured_ruff_job_command( + &ShellConfig::default(), + Some(&profile), + &step, + &project, + None, + ) { + Ok(mut command) => { + assert!(installed_ruff, "impostor caused false probe success"); + assert!(!command + .current_dir(&project) + .output() + .unwrap() + .status + .success()); + } + Err(error) => { + assert!(!installed_ruff); + assert_eq!( + error, + webcodex_core::runner_protocol::VALIDATION_TOOL_UNAVAILABLE_CODE + ); + } + } + // Exercise canonical spawn argv even when this system has no installed Ruff. + assert!(!Command::new(python) + .args(&step.args) + .env_clear() + .env("PYTHONPATH", &injected) + .current_dir(&project) + .output() + .unwrap() + .status + .success()); + assert!( + !marker.exists(), + "project/PYTHONPATH Ruff impostor executed" + ); + } + } +} diff --git a/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs b/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs index 00183c83e..cc7f856b4 100644 --- a/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs +++ b/crates/webcodex-runner/src/webcodex_runner/transport/registration.rs @@ -256,6 +256,7 @@ pub(crate) fn runner_register_capabilities(cfg: &RunnerConfig) -> RunnerCapabili capabilities.set(RunnerCapabilityId::ProjectAllPackages, true); capabilities.set(RunnerCapabilityId::ProjectValidationTestOptions, true); capabilities.set(RunnerCapabilityId::ProjectValidationPythonPytest, true); + capabilities.set(RunnerCapabilityId::ProjectValidationPythonRuff, true); // This binary also understands the first-class go_test durable metadata // identity. Keep this independent from JSON parsing so an old Runner that // supported Connector Go evidence cannot be mistaken for a first-class diff --git a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs index d0ee2df90..75df778dc 100644 --- a/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/validation/project_tests.rs @@ -627,7 +627,7 @@ fn project_validation_python_pytest_detects_explicit_and_auto_and_fences_config( req.action = action; req.test = None; assert!( - matches!(project::plan(&policy, ®istry, &req), Err(ProjectValidationPlanningResult::Unavailable {code,..}) if code == "validation_action_unsupported") + matches!(project::plan(&policy, ®istry, &req), Err(ProjectValidationPlanningResult::Unavailable {code,..}) if code == "validation_check_unavailable") ); } fs::remove_file(root.join("pyproject.toml")).unwrap(); @@ -862,3 +862,45 @@ fn project_validation_all_packages_rejects_excluded_recipe_cwd() { assert!(project::plan(&policy, ®istry, &req).is_ok()); } } + +#[test] +fn project_validation_ruff_auto_and_explicit_plans_share_target_and_fence_configuration() { + let (_temp, root, registry, policy) = fixture("pyproject.toml"); + fs::write( + root.join("pyproject.toml"), + "[tool.ruff]\ntarget-version='py311'\n", + ) + .unwrap(); + for action in [ + ProjectValidationAction::Check, + ProjectValidationAction::FormatCheck, + ] { + let auto = request(action); + let (baseline, _) = project::plan(&policy, ®istry, &auto).unwrap(); + let mut explicit = auto.clone(); + explicit.adapter = ProjectValidationAdapter::Python; + let (explicit_plan, _) = project::plan(&policy, ®istry, &explicit).unwrap(); + assert_eq!(baseline.step, explicit_plan.step); + assert!(baseline.step.is_structured_ruff()); + assert_eq!( + baseline.validation_target_id, + explicit_plan.validation_target_id + ); + fs::write( + root.join("pyproject.toml"), + "[tool.ruff]\ntarget-version='py311'\nline-length=99\n", + ) + .unwrap(); + let (changed, _) = project::plan(&policy, ®istry, &auto).unwrap(); + assert_ne!( + baseline.provenance.manifest_digest, + changed.provenance.manifest_digest + ); + assert_eq!(baseline.validation_target_id, changed.validation_target_id); + fs::write( + root.join("pyproject.toml"), + "[tool.ruff]\ntarget-version='py311'\n", + ) + .unwrap(); + } +} diff --git a/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs b/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs index f97e9407d..cecf4ab09 100644 --- a/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs +++ b/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs @@ -191,7 +191,7 @@ fn cargo_output_schema(tool_name: &str) -> Value { fields.extend([ ("backend", json!({"type":"string", "enum":["rust","go","python"]})), ("action", json!({"type":"string", "enum":["format_check","check","test"]})), - ("adapter", json!({"type":"string", "enum":["cargo_fmt","cargo_check","cargo_test","go_vet","go_test","python:pytest:test"]})), + ("adapter", json!({"type":"string", "enum":["cargo_fmt","cargo_check","cargo_test","go_vet","go_test","python:pytest:test","python:ruff:check","python:ruff:format"]})), ("validation_target_id", schema_type("string", "Canonical resolved validation target, independent of source freshness.")), ("detected_backend", json!({"type":["string","null"], "enum":["rust","go","node","python",null]})), ]); diff --git a/crates/webcodex-tool-contracts/src/tool_call/canonical_enum.rs b/crates/webcodex-tool-contracts/src/tool_call/canonical_enum.rs index 51cfbc5e2..a2580bfbf 100644 --- a/crates/webcodex-tool-contracts/src/tool_call/canonical_enum.rs +++ b/crates/webcodex-tool-contracts/src/tool_call/canonical_enum.rs @@ -1552,7 +1552,7 @@ pub enum ToolCall { #[serde(default)] cwd: Option, action: webcodex_core::project_validation::ProjectValidationAction, - /// Omission means auto. Rust and Go are supported; Python supports test through pytest; Node returns unavailable. + /// Omission means auto. Rust and Go are supported; Python supports test through pytest and check/format_check through project-local Ruff with explicit target-version; Node returns unavailable. #[serde(default)] adapter: Option, /// Optional portable scope. Explicit packages narrow Rust/Go selection; all_packages=true selects the diff --git a/crates/webcodex-tool-contracts/src/tool_definition/testing.rs b/crates/webcodex-tool-contracts/src/tool_definition/testing.rs index ce159552b..5262f027c 100644 --- a/crates/webcodex-tool-contracts/src/tool_definition/testing.rs +++ b/crates/webcodex-tool-contracts/src/tool_definition/testing.rs @@ -131,7 +131,7 @@ pub(super) const DEFINITIONS: &[ToolDefinition] = &[ ) .with_composition_policy(super::ToolCompositionPolicy::Sequential) .with_host_orchestration_hint(super::ToolHostOrchestrationHint::sequential()), - "Portable project validation; Runner owns recipe and argv. Rust/Go: format_check/check/test as supported; Python: test via pytest only; Node unavailable. scope selects packages (1..8) or all_packages=true for Rust/Go check/test; scoped formatting fails closed. Rust all-packages requires a Runner-proven Project-root Cargo workspace; Go keeps ./... single-module scope. Python rejects scope. test.filter is a bounded Rust substring, Go -run regexp or pytest -k expression; require_tests/min_tests govern proven counts. Python uses existing configured/profile/PATH Python 3 and pytest; never installs or falls back. Missing environment is unavailable. dependency_policy.mode=locked maps to Cargo --locked/Go -mod=readonly, not offline; Python and formatting reject it. Additive semantics require Runner support. Go forces GO111MODULE=on/GOWORK=off. No arbitrary executable, argv or shell. Project tests may have effects. Long work keeps the same Job; observe pending, never retry. Advanced options remain on cargo_* / go_test.", ) + "Portable project validation; Runner owns recipe and argv. Rust/Go: format_check/check/test as supported; Python: pytest test, Ruff check/format_check; Node unavailable. scope selects packages (1..8) or all_packages=true for Rust/Go check/test; scoped formatting fails closed. Rust all-packages requires a Runner-proven Project-root Cargo workspace; Go keeps ./... single-module scope. Python rejects scope. test.filter is a bounded Rust substring, Go -run regexp or pytest -k expression; require_tests/min_tests govern proven counts. Python uses existing configured/profile/PATH Python 3 and pytest/Ruff; never installs or falls back. Ruff needs local pyproject.toml [tool.ruff], explicit target-version, no extend; read-only with no fixes, cache, bytecode or output files. Missing environment is unavailable. dependency_policy.mode=locked maps to Cargo --locked/Go -mod=readonly, not offline; Python and formatting reject it. Additive semantics require Runner support. Go forces GO111MODULE=on/GOWORK=off. No arbitrary executable, argv or shell. Project tests may have effects. Long work keeps the same Job; observe pending, never retry. Advanced options remain on cargo_* / go_test.", ) .with_execution(super::ToolExecutionContract::new( super::ToolExecutionForm::StructuredValidation, super::ToolExecutionLifetime::Runner, diff --git a/crates/webcodex-validation/src/adapters/go.rs b/crates/webcodex-validation/src/adapters/go.rs index 6960d7700..75d6691a1 100644 --- a/crates/webcodex-validation/src/adapters/go.rs +++ b/crates/webcodex-validation/src/adapters/go.rs @@ -18,14 +18,6 @@ pub(super) fn validation_adapter(tool_identity: &str) -> Option<&'static dyn Val } } -pub(super) fn check_adapter() -> &'static dyn ValidationAdapter { - &GO_VET_ADAPTER -} - -pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { - &GO_TEST_ADAPTER -} - impl ValidationEvidenceProfile for GoTestValidationAdapter { fn validation_kind(&self) -> &'static str { "test" diff --git a/crates/webcodex-validation/src/adapters/mod.rs b/crates/webcodex-validation/src/adapters/mod.rs index d4922874a..25bd05206 100644 --- a/crates/webcodex-validation/src/adapters/mod.rs +++ b/crates/webcodex-validation/src/adapters/mod.rs @@ -106,6 +106,8 @@ pub enum ReadOnlyValidationOperation { Cargo(CargoReadOnlyValidationOperation), Go(GoReadOnlyValidationOperation), Python(PythonTestOptions), + PythonRuffCheck, + PythonRuffFormat, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -147,7 +149,10 @@ impl ReadOnlyValidationOperation { return Ok(self); }; match &mut self { - Self::Python(_) | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => { + Self::Python(_) + | Self::PythonRuffCheck + | Self::PythonRuffFormat + | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => { return Err("dependency_policy_unsupported") } Self::Cargo(CargoReadOnlyValidationOperation::Check(options)) => { @@ -168,7 +173,10 @@ impl ReadOnlyValidationOperation { fn dependency_mode(&self) -> Option { match self { - Self::Python(_) | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => None, + Self::Python(_) + | Self::PythonRuffCheck + | Self::PythonRuffFormat + | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => None, Self::Cargo(CargoReadOnlyValidationOperation::Check(options)) => { options.dependency_mode } @@ -184,12 +192,23 @@ impl ReadOnlyValidationOperation { Self::Cargo(CargoReadOnlyValidationOperation::Test(options)) => options.all_packages, Self::Go(GoReadOnlyValidationOperation::Check(options)) => options.all_packages, Self::Go(GoReadOnlyValidationOperation::Test(options)) => options.all_packages, - Self::Python(_) | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => false, + Self::Python(_) + | Self::PythonRuffCheck + | Self::PythonRuffFormat + | Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => false, } } pub fn compatibility_profile(&self) -> ValidationCompatibilityProfile { match self { + Self::PythonRuffCheck => ValidationCompatibilityProfile { + tool_identity: "python:ruff:check", + validation_identity: ToolValidationIdentityKind::PythonRuffCheck, + }, + Self::PythonRuffFormat => ValidationCompatibilityProfile { + tool_identity: "python:ruff:format", + validation_identity: ToolValidationIdentityKind::PythonRuffFormat, + }, Self::Python(_) => ValidationCompatibilityProfile { tool_identity: "python:pytest:test", validation_identity: ToolValidationIdentityKind::PythonPytest, @@ -223,43 +242,39 @@ impl ReadOnlyValidationOperation { } } - pub fn adapter(&self) -> &'static dyn ValidationAdapter { - match self { - Self::Python(_) => python::test_adapter(), - Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => rust::format_adapter(), - Self::Cargo(CargoReadOnlyValidationOperation::Check(_)) => rust::check_adapter(), - Self::Cargo(CargoReadOnlyValidationOperation::Test(_)) => rust::test_adapter(), - Self::Go(GoReadOnlyValidationOperation::Check(_)) => go::check_adapter(), - Self::Go(GoReadOnlyValidationOperation::Test(_)) => go::test_adapter(), - } + pub fn evidence_profile(&self) -> &'static dyn ValidationEvidenceProfile { + validation_evidence_profile_for_tool(self.compatibility_profile().tool_identity) + .expect("operation has a registered evidence profile") } pub fn build_readonly_plan(&self) -> Result { + let adapter = validation_adapter_for_tool(self.compatibility_profile().tool_identity) + .ok_or("Ruff requires Runner-owned project manifest resolution")?; match self { - Self::Python(options) => self - .adapter() - .build_readonly_plan(ValidationCommandOptions { - filter: options.filter.clone(), - ..Default::default() - }), - Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => self - .adapter() + Self::PythonRuffCheck | Self::PythonRuffFormat => { + unreachable!("Ruff has no static command adapter") + } + Self::Python(options) => adapter.build_readonly_plan(ValidationCommandOptions { + filter: options.filter.clone(), + ..Default::default() + }), + Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => adapter .build_readonly_plan(ValidationCommandOptions { check: true, ..ValidationCommandOptions::default() }), - Self::Cargo(CargoReadOnlyValidationOperation::Check(options)) => self - .adapter() - .build_readonly_plan(ValidationCommandOptions::from(options.clone())), - Self::Cargo(CargoReadOnlyValidationOperation::Test(options)) => self - .adapter() - .build_readonly_plan(ValidationCommandOptions::from(options.clone())), - Self::Go(GoReadOnlyValidationOperation::Check(options)) => self - .adapter() - .build_readonly_plan(ValidationCommandOptions::from(options.clone())), - Self::Go(GoReadOnlyValidationOperation::Test(options)) => self - .adapter() - .build_readonly_plan(ValidationCommandOptions::from(options.clone())), + Self::Cargo(CargoReadOnlyValidationOperation::Check(options)) => { + adapter.build_readonly_plan(ValidationCommandOptions::from(options.clone())) + } + Self::Cargo(CargoReadOnlyValidationOperation::Test(options)) => { + adapter.build_readonly_plan(ValidationCommandOptions::from(options.clone())) + } + Self::Go(GoReadOnlyValidationOperation::Check(options)) => { + adapter.build_readonly_plan(ValidationCommandOptions::from(options.clone())) + } + Self::Go(GoReadOnlyValidationOperation::Test(options)) => { + adapter.build_readonly_plan(ValidationCommandOptions::from(options.clone())) + } } } @@ -268,6 +283,7 @@ impl ReadOnlyValidationOperation { pub fn validation_target_id(&self, cwd: Option<&str>) -> Option { let profile = self.compatibility_profile(); let arguments = match self { + Self::PythonRuffCheck | Self::PythonRuffFormat => serde_json::json!({"cwd":cwd}), Self::Python(options) => serde_json::json!({"cwd":cwd,"filter":options.filter}), Self::Cargo(CargoReadOnlyValidationOperation::FormatCheck) => serde_json::json!({ "cwd": cwd, @@ -378,7 +394,8 @@ pub fn project_validation_operation( ("python", Test) => Ok(ReadOnlyValidationOperation::Python(PythonTestOptions { filter: None, })), - ("python", _) => Err("validation_action_unsupported"), + ("python", Check) => Ok(ReadOnlyValidationOperation::PythonRuffCheck), + ("python", Format) => Ok(ReadOnlyValidationOperation::PythonRuffFormat), _ => Err("validation_adapter_unavailable"), } } @@ -582,6 +599,7 @@ pub fn validation_evidence_profile_for_tool( ) -> Option<&'static dyn ValidationEvidenceProfile> { validation_adapter_for_tool(tool_identity) .map(|adapter| adapter as &dyn ValidationEvidenceProfile) + .or_else(|| python::ruff_evidence_profile(tool_identity)) } /// Evidence-only view of the canonical backend/action mapping. @@ -589,6 +607,14 @@ pub fn validation_evidence_profile_for_recipe( backend: &str, action: crate::SemanticCheck, ) -> Option<&'static dyn ValidationEvidenceProfile> { - validation_adapter_for_recipe(backend, action) - .map(|adapter| adapter as &dyn ValidationEvidenceProfile) + match (backend, action) { + ("python", crate::SemanticCheck::Check) => { + validation_evidence_profile_for_tool("python:ruff:check") + } + ("python", crate::SemanticCheck::Format) => { + validation_evidence_profile_for_tool("python:ruff:format") + } + _ => validation_adapter_for_recipe(backend, action) + .map(|adapter| adapter as &dyn ValidationEvidenceProfile), + } } diff --git a/crates/webcodex-validation/src/adapters/python.rs b/crates/webcodex-validation/src/adapters/python.rs index 93d405d74..ac5da9177 100644 --- a/crates/webcodex-validation/src/adapters/python.rs +++ b/crates/webcodex-validation/src/adapters/python.rs @@ -4,6 +4,143 @@ use super::{ }; use webcodex_core::validation_evidence::{parse_pytest_diagnostics, ValidationDiagnostics}; +struct RuffEvidenceProfile { + kind: &'static str, + identity: &'static str, +} +static RUFF_CHECK: RuffEvidenceProfile = RuffEvidenceProfile { + kind: "check", + identity: "python:ruff:check", +}; +static RUFF_FORMAT: RuffEvidenceProfile = RuffEvidenceProfile { + kind: "format", + identity: "python:ruff:format", +}; + +pub(super) fn ruff_evidence_profile( + identity: &str, +) -> Option<&'static dyn ValidationEvidenceProfile> { + match identity { + "python:ruff:check" => Some(&RUFF_CHECK), + "python:ruff:format" => Some(&RUFF_FORMAT), + _ => None, + } +} + +impl ValidationEvidenceProfile for RuffEvidenceProfile { + fn validation_kind(&self) -> &'static str { + self.kind + } + fn tool_identity(&self) -> &'static str { + self.identity + } + fn parse(&self, stdout: &str, _stderr: &str, truncated: bool) -> ValidationDiagnostics { + use sha2::{Digest, Sha256}; + // Do not copy filenames (Ruff commonly emits absolute paths), source, + // messages, fixes or arbitrary JSON fields into safe diagnostic metadata. + let truncated = truncated || stdout.len() > 64 * 1024; + let mut result = ValidationDiagnostics { + available: false, + parser: "ruff_validation_parser_v1", + reason: Some("no complete Ruff diagnostics"), + diagnostic_count: None, + diagnostics: Vec::new(), + returned_diagnostic_count: 0, + diagnostics_truncated: truncated, + invalid_diagnostics_omitted: 0, + test_summary: None, + failed_test_details: Vec::new(), + failed_test_details_truncated: false, + truncated: Some(truncated), + }; + if truncated { + return result; + } + if self.kind == "format" { + // Native exit status owns format success. Human output is version + // dependent and never proves diagnostic or test counts. + result.reason = Some("Ruff format file counts are unproven; use native exit status"); + return result; + } + let mut count = 0usize; + let mut seen = std::collections::HashSet::new(); + for line in stdout.lines().filter(|line| !line.trim().is_empty()) { + let code = serde_json::from_str::(line) + .ok() + .and_then(|value| { + let code = value.get("code")?.as_str()?; + let location = value.get("location")?; + let row = location.get("row")?.as_u64()?; + let column = location.get("column")?.as_u64()?; + let filename = value.get("filename")?.as_str()?; + (code.len() <= 32 + && !code.is_empty() + && code + .bytes() + .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit()) + && row > 0 + && column > 0 + && !filename.is_empty()) + .then(|| { + let mut fingerprint = Sha256::new(); + fingerprint.update((code.len() as u64).to_be_bytes()); + fingerprint.update(code.as_bytes()); + fingerprint.update(row.to_be_bytes()); + fingerprint.update(column.to_be_bytes()); + fingerprint.update(filename.as_bytes()); + (code.to_string(), <[u8; 32]>::from(fingerprint.finalize())) + }) + }); + let Some((code, fingerprint)) = code else { + result.invalid_diagnostics_omitted += 1; + continue; + }; + if !seen.insert(fingerprint) { + continue; + } + count += 1; + if result.diagnostics.len() < webcodex_core::validation_evidence::MAX_DIAGNOSTICS { + result + .diagnostics + .push(webcodex_core::validation_evidence::CargoDiagnostic { + severity: "error", + code: Some(code), + file: None, + line: None, + column: None, + message: "Ruff reported a lint violation".into(), + }); + } + } + result.available = count > 0; + result.reason = (!result.available).then_some("no complete Ruff diagnostics"); + if result.invalid_diagnostics_omitted == 0 && count > 0 { + result.diagnostic_count = Some(count); + } + result.returned_diagnostic_count = result.diagnostics.len(); + result.diagnostics_truncated = count > result.diagnostics.len(); + result + } + fn map_failure_kind(&self, evidence: ValidationFailureEvidence<'_>) -> &'static str { + if evidence.success { + "unknown" + } else if matches!( + evidence.reported_failure_kind, + Some("timeout" | "timed_out" | "command_timeout") + ) { + "timeout" + } else if evidence.exit_code == Some(1) { + if self.kind == "format" { + "format_diff" + } else { + "validation_failed" + } + } else { + "process_exit" + } + } +} + struct PytestValidationAdapter; static PYTEST_ADAPTER: PytestValidationAdapter = PytestValidationAdapter; pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { diff --git a/crates/webcodex-validation/src/adapters/rust.rs b/crates/webcodex-validation/src/adapters/rust.rs index aa1bbf11f..ffd10d0db 100644 --- a/crates/webcodex-validation/src/adapters/rust.rs +++ b/crates/webcodex-validation/src/adapters/rust.rs @@ -44,18 +44,6 @@ pub(super) fn validation_adapters() -> &'static [&'static dyn ValidationAdapter] &RUST_ADAPTERS } -pub(super) fn format_adapter() -> &'static dyn ValidationAdapter { - &CARGO_FMT_ADAPTER -} - -pub(super) fn check_adapter() -> &'static dyn ValidationAdapter { - &CARGO_CHECK_ADAPTER -} - -pub(super) fn test_adapter() -> &'static dyn ValidationAdapter { - &CARGO_TEST_ADAPTER -} - impl ValidationEvidenceProfile for RustValidationAdapter { fn validation_kind(&self) -> &'static str { match self.kind { diff --git a/crates/webcodex-validation/src/profile_tests.rs b/crates/webcodex-validation/src/profile_tests.rs index 32cb354e9..e6347bae3 100644 --- a/crates/webcodex-validation/src/profile_tests.rs +++ b/crates/webcodex-validation/src/profile_tests.rs @@ -162,7 +162,7 @@ fn semantic_read_only_operations_preserve_legacy_leaf_plans_and_identity_profile .build_readonly_plan(legacy_options) .unwrap(); assert_eq!(semantic_plan, legacy_plan, "{tool_name}"); - assert_eq!(operation.adapter().tool_identity(), tool_name); + assert_eq!(operation.evidence_profile().tool_identity(), tool_name); } } @@ -311,7 +311,7 @@ fn go_check_semantic_operation_uses_canonical_vet_adapter() { dependency_mode: None, })); assert_eq!(operation.compatibility_profile().tool_identity, "go_vet"); - assert_eq!(operation.adapter().tool_identity(), "go_vet"); + assert_eq!(operation.evidence_profile().tool_identity(), "go_vet"); assert_eq!( operation .build_readonly_plan() @@ -839,7 +839,7 @@ fn project_validation_python_pytest_owns_filter_plan_and_parser() { selected.validation_target_id(Some(".")), operation.validation_target_id(Some(".")) ); - let adapter = selected.adapter(); + let adapter = selected.evidence_profile(); assert_eq!(adapter.tool_identity(), "python:pytest:test"); assert!(adapter.reports_test_run_metadata()); assert_eq!( @@ -856,8 +856,11 @@ fn project_validation_python_pytest_owns_filter_plan_and_parser() { .is_none()); for action in [SemanticCheck::Format, SemanticCheck::Check] { assert_eq!( - project_validation_operation("python", action, None, false).unwrap_err(), - "validation_action_unsupported" + project_validation_operation("python", action, None, false) + .unwrap() + .build_readonly_plan() + .unwrap_err(), + "Ruff requires Runner-owned project manifest resolution" ); } assert_eq!( @@ -877,7 +880,8 @@ fn project_validation_python_pytest_owns_filter_plan_and_parser() { } )) .is_err()); - assert!(adapter + assert!(validation_adapter_for_tool("python:pytest:test") + .unwrap() .build_readonly_plan(ValidationCommandOptions { all_packages: true, ..Default::default() @@ -945,3 +949,79 @@ fn evidence_profiles_share_direct_adapter_registry_and_recipe_mapping() { assert!(validation_evidence_profile_for_tool("unknown_test").is_none()); assert!(validation_evidence_profile_for_recipe("go", SemanticCheck::Format).is_none()); } + +#[test] +fn ruff_evidence_profiles_have_no_static_command_authority_or_borrowed_counts() { + use crate::{validation_evidence_profile_for_recipe, validation_evidence_profile_for_tool}; + let mut identities = std::collections::HashSet::new(); + for (action, tool) in [ + (SemanticCheck::Check, "python:ruff:check"), + (SemanticCheck::Format, "python:ruff:format"), + ] { + let op = project_validation_operation("python", action, None, false).unwrap(); + assert!(op.build_readonly_plan().is_err()); + assert!(validation_adapter_for_tool(tool).is_none()); + assert!(identities.insert(op.validation_target_id(Some(".")).unwrap())); + let retained = webcodex_workflow_session::execution_output_summary_for_tool_result( + "project_validate", + &serde_json::json!({"adapter": tool, "stdout_tail": "", "stderr_tail": ""}), + ) + .unwrap(); + assert_eq!( + validation_evidence_profile_for_tool(retained["adapter"].as_str().unwrap()) + .unwrap() + .tool_identity(), + tool, + ); + let profile = validation_evidence_profile_for_tool(tool).unwrap(); + assert_eq!( + profile.tool_identity(), + validation_evidence_profile_for_recipe("python", action) + .unwrap() + .tool_identity() + ); + assert!(!profile.reports_test_run_metadata()); + for output in [ + "test result: ok. 100 passed; 0 failed; 0 ignored", + "2 passed in 0.01s", + "error[E0308]: secret", + "Would reformat: /private/token.py\n1 file would be reformatted\n", + ] { + let evidence = profile.parse(output, output, false); + assert!(evidence.test_summary.is_none()); + assert!(evidence.diagnostic_count.is_none()); + assert!(!serde_json::to_string(&evidence) + .unwrap() + .contains("/private")); + } + } + let profile = validation_evidence_profile_for_tool("python:ruff:check").unwrap(); + let line = r#"{"code":"F401","location":{"row":1,"column":8},"filename":"/private/token.py","message":"secret_value","fix":{"content":"secret_value"}}"#; + let parsed = profile.parse(line, "", false); + assert_eq!(parsed.diagnostic_count, Some(1)); + assert_eq!(parsed.diagnostics[0].code.as_deref(), Some("F401")); + let safe = serde_json::to_string(&parsed).unwrap(); + assert!(!safe.contains("secret_value") && !safe.contains("/private")); + assert!(profile.parse(line, "", true).diagnostic_count.is_none()); + assert!(profile + .parse(&format!("{line}\n{{partial"), "", false) + .diagnostic_count + .is_none()); + assert!(profile.parse("", "", false).diagnostic_count.is_none()); + assert_eq!( + profile + .parse(&format!("{line}\n{line}"), "", false) + .diagnostic_count, + Some(1) + ); + let lines = (1..=21) + .map(|row| line.replace("\"row\":1", &format!("\"row\":{row}"))) + .collect::>(); + let bounded = profile.parse(&lines.join("\n"), "", false); + assert_eq!(bounded.diagnostic_count, Some(21)); + assert_eq!(bounded.returned_diagnostic_count, 20); + assert!(bounded.diagnostics_truncated); + let oversized = profile.parse(&"x".repeat(64 * 1024 + 1), "", false); + assert_eq!(oversized.truncated, Some(true)); + assert!(oversized.diagnostic_count.is_none()); +} diff --git a/crates/webcodex-validation/src/recipe.rs b/crates/webcodex-validation/src/recipe.rs index 70564e1db..6c007464f 100644 --- a/crates/webcodex-validation/src/recipe.rs +++ b/crates/webcodex-validation/src/recipe.rs @@ -162,7 +162,8 @@ pub fn resolve_validation_recipe_with_packages( } /// Canonical project gateway planning. Ordinary recipe workflows retain their -/// existing Python Ruff/mypy/unittest choices; this gateway supports pytest only. +/// existing Python Ruff/mypy/unittest choices. Project Ruff requires a pinned +/// local manifest; evidence profiles cannot manufacture this command authority. pub fn resolve_project_validation_recipe( execution_root: &Path, cwd: Option<&str>, @@ -187,6 +188,14 @@ pub fn resolve_project_validation_recipe( dependency_policy, ); } + let ruff_digest = if checks.iter().any(|check| *check != SemanticCheck::Test) { + Some(project_ruff_manifest_digest( + &resolved.execution_root, + &resolved.absolute_root, + )?) + } else { + None + }; let mut steps = Vec::with_capacity(checks.len()); for check in checks { let operation = crate::project_validation_operation( @@ -198,12 +207,14 @@ pub fn resolve_project_validation_recipe( .and_then(|operation| operation.with_dependency_policy(dependency_policy)) .and_then(|operation| operation.with_test_filter(test_filter)) .map_err(RecipeError::new)?; - steps.push( + steps.push(if *check == SemanticCheck::Test { operation .build_readonly_plan() .map_err(|_| check_unavailable())? - .structured_step, - ); + .structured_step + } else { + ShellJobValidationStep::python_ruff(check.as_str()).ok_or_else(check_unavailable)? + }); } let test_filter = test_filter .map(webcodex_core::runner_protocol::normalize_pytest_filter) @@ -213,11 +224,23 @@ pub fn resolve_project_validation_recipe( // pytest searches configuration from the invocation directory through its // ancestors. Bind every candidate inside the registered Project, and fail // closed if that search could escape to an ambient parent configuration. - let manifest_digest = digest_project_recipe_files( - &resolved.execution_root, - pytest_project_provenance_files(&resolved.execution_root, &resolved.absolute_root)?, - ) - .map_err(map_project_recipe_error)?; + let pytest_digest = if checks.contains(&SemanticCheck::Test) { + Some( + digest_project_recipe_files( + &resolved.execution_root, + pytest_project_provenance_files(&resolved.execution_root, &resolved.absolute_root)?, + ) + .map_err(map_project_recipe_error)?, + ) + } else { + None + }; + let manifest_digest = match (pytest_digest, ruff_digest) { + (Some(pytest), None) => pytest, + (None, Some(ruff)) => ruff, + (Some(pytest), Some(ruff)) => format!("{:x}", Sha256::digest(format!("{pytest}\0{ruff}"))), + (None, None) => return Err(check_unavailable()), + }; let invocation_digest = format!( "{:x}", Sha256::digest(serde_json::to_vec(&steps).map_err(|_| manifest_invalid())?) @@ -232,6 +255,72 @@ pub fn resolve_project_validation_recipe( }) } +fn project_ruff_manifest_digest( + execution_root: &Path, + recipe_root: &Path, +) -> Result { + use std::io::Read; + let path = recipe_root + .join("pyproject.toml") + .canonicalize() + .map_err(|_| check_unavailable())?; + if !path.starts_with(execution_root) { + return Err(manifest_invalid()); + } + if !fs::metadata(&path) + .map_err(|_| manifest_invalid())? + .is_file() + { + return Err(manifest_invalid()); + } + // Bound both the parse and digest to the same bytes. Never hash a second + // read that could differ from the config whose authority was just checked. + let mut bytes = Vec::new(); + const MAX_BYTES: u64 = 1024 * 1024; + let file = fs::File::open(&path).map_err(|_| manifest_invalid())?; + if !file.metadata().map_err(|_| manifest_invalid())?.is_file() { + return Err(manifest_invalid()); + } + file.take(MAX_BYTES + 1) + .read_to_end(&mut bytes) + .map_err(|_| manifest_invalid())?; + if bytes.len() as u64 > MAX_BYTES { + return Err(manifest_invalid()); + } + let value: toml::Value = + toml::from_str(std::str::from_utf8(&bytes).map_err(|_| manifest_invalid())?) + .map_err(|_| manifest_invalid())?; + let ruff = value + .get("tool") + .and_then(|tool| tool.get("ruff")) + .and_then(toml::Value::as_table) + .ok_or_else(check_unavailable)?; + if ruff.contains_key("extend") { + return Err(manifest_invalid()); + } + let target = ruff + .get("target-version") + .and_then(toml::Value::as_str) + .ok_or_else(manifest_invalid)?; + if !matches!( + target, + "py37" | "py38" | "py39" | "py310" | "py311" | "py312" | "py313" | "py314" + ) { + return Err(manifest_invalid()); + } + let mut digest = Sha256::new(); + digest.update(b"webcodex-project-ruff-manifest-v1\0"); + digest.update( + path.strip_prefix(execution_root) + .map_err(|_| manifest_invalid())? + .as_os_str() + .as_encoded_bytes(), + ); + digest.update((bytes.len() as u64).to_be_bytes()); + digest.update(bytes); + Ok(format!("{:x}", digest.finalize())) +} + fn pytest_project_provenance_files( execution_root: &Path, recipe_root: &Path, diff --git a/crates/webcodex-validation/src/recipe_tests.rs b/crates/webcodex-validation/src/recipe_tests.rs index d512cdd45..ce8e50e6d 100644 --- a/crates/webcodex-validation/src/recipe_tests.rs +++ b/crates/webcodex-validation/src/recipe_tests.rs @@ -984,3 +984,121 @@ fn manifestless_explicit_python_uses_stable_unittest_plan() { assert_eq!(pytest.steps[0].args, ["-m", "pytest"]); assert_ne!(pytest.manifest_digest, seed); } + +#[test] +fn project_ruff_requires_local_explicit_config_and_fences_exact_bytes() { + let temp = tempfile::tempdir().unwrap(); + let plan = |action| { + resolve_project_validation_recipe( + temp.path(), + None, + Some(RecipeId::Python), + &[action], + None, + None, + false, + None, + ) + }; + for action in [SemanticCheck::Check, SemanticCheck::Format] { + assert!(plan(action).is_err(), "manifestless Ruff must fail closed"); + for manifest in [ + "", + "[tool.black]\n", + "[tool.ruff]\n", + "[tool.ruff]\ntarget-version=311\n", + "[tool.ruff]\ntarget-version='py311'\nextend='../ruff.toml'\n", + "[tool.ruff]\ntarget-version='py311'\nextend=''\n", + "[tool.ruff]\ntarget-version='ambient'\n", + ] { + write(temp.path(), "pyproject.toml", manifest); + assert!(plan(action).is_err(), "{manifest}"); + } + write( + temp.path(), + "pyproject.toml", + "[tool.ruff]\ntarget-version='py311'\nfix=true\nfix-only=true\n", + ); + let first = plan(action).unwrap(); + assert!(first.steps[0].is_structured_ruff()); + assert_eq!( + first.steps[0], + webcodex_core::runner_protocol::ShellJobValidationStep::python_ruff(action.as_str()) + .unwrap() + ); + write( + temp.path(), + "pyproject.toml", + "[tool.ruff]\ntarget-version='py312'\n", + ); + let changed = plan(action).unwrap(); + assert_ne!(first.manifest_digest, changed.manifest_digest); + assert_eq!(first.invocation_digest, changed.invocation_digest); + for (packages, all, filter, policy) in [ + (Some(vec!["src".into()]), false, None, None), + (None, true, None, None), + (None, false, Some("name"), None), + ( + None, + false, + None, + Some(webcodex_core::project_validation::ProjectDependencyPolicy { + mode: webcodex_core::project_validation::ProjectDependencyMode::Locked, + }), + ), + ] { + assert!(resolve_project_validation_recipe( + temp.path(), + None, + Some(RecipeId::Python), + &[action], + filter, + packages.as_deref(), + all, + policy + ) + .is_err()); + } + fs::remove_file(temp.path().join("pyproject.toml")).unwrap(); + } + write( + temp.path(), + "pyproject.toml", + &format!( + "[tool.ruff]\ntarget-version='py311'\n#{}", + "x".repeat(1024 * 1024) + ), + ); + assert_eq!( + plan(SemanticCheck::Check).unwrap_err().code, + "validation_manifest_invalid" + ); +} + +#[cfg(unix)] +#[test] +fn project_ruff_rejects_manifest_symlink_escape() { + let temp = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + write( + outside.path(), + "pyproject.toml", + "[tool.ruff]\ntarget-version='py311'\n", + ); + std::os::unix::fs::symlink( + outside.path().join("pyproject.toml"), + temp.path().join("pyproject.toml"), + ) + .unwrap(); + assert!(resolve_project_validation_recipe( + temp.path(), + None, + Some(RecipeId::Python), + &[SemanticCheck::Check], + None, + None, + false, + None + ) + .is_err()); +} diff --git a/crates/webcodex-workflow-session/src/events.rs b/crates/webcodex-workflow-session/src/events.rs index 4c63e6cd1..c41de8070 100644 --- a/crates/webcodex-workflow-session/src/events.rs +++ b/crates/webcodex-workflow-session/src/events.rs @@ -1120,6 +1120,30 @@ pub fn validation_output_summary_for_tool_result(tool_name: &str, output: &Value mod handoff_evidence_tests { use super::*; + #[test] + fn project_validation_ruff_evidence_survives_summary_and_resanitization() { + for (adapter, action) in [ + ("python:ruff:check", "check"), + ("python:ruff:format", "format_check"), + ] { + let output = json!({ + "adapter": adapter, "action": action, "backend": "python", + "validation_target_id": "target:0123456789abcdef01234567", + "stdout_tail": "", "stderr_tail": "", "unretained": "payload", + }); + let summary = + validation_output_summary_for_tool_result("project_validate", &output).unwrap(); + let retained = + sanitize_persisted_validation_output_summary("project_validate", &summary).unwrap(); + for value in [&summary, &retained] { + for field in ["adapter", "action", "backend", "validation_target_id"] { + assert_eq!(value[field], output[field]); + } + assert!(value.get("unretained").is_none()); + } + } + } + #[test] fn sparse_job_handoff_unknown_retains_incomplete_evidence_without_payload() { for tool in ["run_shell", "cargo_check", "cargo_test"] { @@ -1634,6 +1658,8 @@ fn copy_project_validation_evidence(summary: &mut Value, output: &Value) { "go_vet", "go_test", "python:pytest:test", + "python:ruff:check", + "python:ruff:format", ][..], ), ("action", &["format_check", "check", "test"][..]), diff --git a/docs/implementation/python-pytest-project-validation.md b/docs/implementation/python-pytest-project-validation.md index 50fd3000b..41d446338 100644 --- a/docs/implementation/python-pytest-project-validation.md +++ b/docs/implementation/python-pytest-project-validation.md @@ -3,7 +3,8 @@ `project_validate(action=test, adapter=python)` plans a bounded pytest run on the owning Runner. `auto` uses the existing nearest `pyproject.toml` recipe root; explicit Python also works without a manifest. Ambiguous recipes, escaping paths, -check/format actions, package scope (including `scope.all_packages`) and dependency policy fail closed. `cwd` +package scope (including `scope.all_packages`) and dependency policy fail closed. +Python check/format actions use the separate [Ruff project contract](python-ruff-project-validation.md). `cwd` selects a recipe root, not a free pytest positional argument. Ordinary recipe workflows retain their existing Python unittest/Ruff/mypy choices. @@ -50,7 +51,7 @@ target and policy; existing Cargo/Go sparse receipts are unchanged. `project_validation_python_pytest_v1` is additive and false when omitted. The running binary advertises it independently of local Python/pytest installation. -Explicit Python planning and every Python Job admission check that capability; +Explicit Python test planning and every pytest Job admission check that capability; Auto may discover Python on the Runner but cannot enqueue it without current support. Admission and post-queue replan fences cover pytest configuration from the selected Python cwd through the registered Project root. If no recognized diff --git a/docs/implementation/python-ruff-project-validation.md b/docs/implementation/python-ruff-project-validation.md new file mode 100644 index 000000000..bf71e2d8f --- /dev/null +++ b/docs/implementation/python-ruff-project-validation.md @@ -0,0 +1,79 @@ +# Python Ruff project validation (#962 P1) + +`project_validate(action=check|format_check, adapter=python|auto)` resolves Ruff on +its owning Runner. Python `test` remains pytest. Rust/Go and ordinary recipe +workflows keep their existing semantics. This slice adds no public Ruff tool, +installer, environment creation, package scope, dependency policy or free flags. + +Planning requires a Project-local `pyproject.toml` with a `[tool.ruff]` table and +an explicit `target-version` (`py37` through `py314`). The installed Ruff must +support that target. Any `extend` entry is rejected, including an empty one. +The manifest must resolve inside the registered Project and be at most 1 MiB. +The planner parses and hashes the same bounded bytes, including the resolved +relative manifest path. The existing admission and post-queue replans compare +manifest/invocation/root digests, exact step, cwd and target identity. Ruff uses +an explicit config, so ancestor and nested Ruff configs are not selected. + +The protocol owns one exact argv per operation: + +```text +python -I -B -m ruff check --no-fix --no-fix-only --no-cache --no-respect-gitignore --output-format json-lines --config pyproject.toml . +python -I -B -m ruff format --check --no-cache --no-respect-gitignore --config pyproject.toml . +``` + +Check overrides config-driven fixes and fix-only behavior. Format only checks; +there is no write/diff/output-file mode. Both disable Ruff cache and Python +bytecode writes, and disable gitignore discovery. `RUFF_OUTPUT_FILE` is removed +from probe and spawn environments. There are no model-supplied argv/env values. + +Runner selects the existing configured/profile/PATH Python through its script +interpreter resolver. A configured Python is authoritative: invalid configured +paths fail rather than choosing a PATH alternative. Without a configured Python, +the existing platform candidate order applies. A five-second managed probe checks +Python 3 and the Ruff module in the exact selected interpreter, cwd and profile +environment. Ruff probe and spawn use `-I -B`: Python isolated mode intentionally +ignores cwd, `PYTHONPATH`, and user site packages, while preserving installed +interpreter site packages (including the selected venv). Pytest preserves its +configured `PYTHONPATH` and historical interpreter fallback candidates without +`-I`. Ruff also sets `PYTHONDONTWRITEBYTECODE=1`. Missing interpreter/module returns definite +not-started unavailability. No fallback follows a failed probe. Probe descendants +are cleaned up through the existing managed process-tree path. + +`project_validation_python_ruff_v1` is separate from pytest and defaults false +when omitted. The Runner binary advertises protocol support independently of +installed tools. Explicit Python test planning requires the pytest capability; +check/format planning requires Ruff. Job admission checks the actual planned +adapter and step again, including auto selection and Runner replacement. + +`python:ruff:check` and `python:ruff:format` have separate target identities and +evidence profiles in the existing registry. They have no static command adapter: +command planning requires Runner manifest resolution. Check diagnostics accept +bounded JSON lines and retain only validated rule codes with a fixed message; +paths, source, messages and fix payloads are not copied into safe diagnostics. +Malformed/truncated excerpts do not prove complete counts. Empty output does not +fabricate a zero diagnostic count. Format uses native process status and leaves +human-output file counts unproven. Neither profile borrows Rust, Go or pytest +counts. Existing source fences, timeout, cancellation, same-Job observation, +reconciliation and no-replay behavior apply. + +## Validation limits + +Ruff 0.16.9 was exercised on Linux aarch64 using a SHA-256-verified +official PyPI wheel unpacked into an isolated temporary Python environment. +Both exact canonical command forms were verified: lint and format returned +exit code 1 for deliberately failing source, JSON Lines carried lint rule codes, +and project settings `fix=true` and `fix-only=true` did not modify source. +No Ruff cache, Python bytecode, or project-local `ruff.py` execution was observed, +even with `PYTHONPATH` pointing at the project. The temporary wheel, environment, +and test files were removed; the system Python was not modified. + +That evidence is for Ruff **0.16.9**, not a promise that every older or future +version supports these flags and targets. Unsupported versions or configuration +must fail without dropping safety flags, installation, or fallback. + +Local Rust tests separately cover configuration authority, canonical argv, +identity/evidence, capability selection and admission, queue-time replanning, +missing tooling, and workflow-session evidence retention. Stub interpreter +tests verify process selection and environment wiring; they are not substitutes +for executing an actual Ruff binary. The contract is Ruff's requested read-only +check/format behavior in a trusted interpreter, not an OS-level filesystem sandbox. diff --git a/src/runner_quic.rs b/src/runner_quic.rs index af74794ac..654ab1943 100644 --- a/src/runner_quic.rs +++ b/src/runner_quic.rs @@ -618,6 +618,7 @@ mod tests { project_all_packages_v1: false, project_validation_test_options_v1: false, project_validation_python_pytest_v1: false, + project_validation_python_ruff_v1: false, structured_go_test_tool: true, structured_go_test_packages: true, structured_process_argv: true, diff --git a/src/runner_ws.rs b/src/runner_ws.rs index 9844ea865..45ef472c3 100644 --- a/src/runner_ws.rs +++ b/src/runner_ws.rs @@ -429,6 +429,7 @@ mod tests { project_all_packages_v1: false, project_validation_test_options_v1: false, project_validation_python_pytest_v1: false, + project_validation_python_ruff_v1: false, structured_go_test_tool: true, structured_go_test_packages: true, structured_process_argv: true, diff --git a/src/tool_runtime/jobs.rs b/src/tool_runtime/jobs.rs index 344a1c84c..a05bcd19a 100644 --- a/src/tool_runtime/jobs.rs +++ b/src/tool_runtime/jobs.rs @@ -427,7 +427,8 @@ pub(crate) fn structured_validation_evidence( truncated: bool, ) -> StructuredValidationEvidence { let combined = format!("{stdout}\n{stderr}"); - let profile = matches!(kind, "check" | "test") + let profile = (matches!(kind, "check" | "test") + || (kind == "format" && tool == "python:ruff:format")) .then(|| super::validation_profile::validation_evidence_profile_for_tool(tool)) .flatten() .filter(|adapter| adapter.validation_kind() == kind); @@ -651,7 +652,16 @@ pub(crate) fn validation_job_projection_with_policy( } } let profile_mismatch = profile.is_some_and(|profile| profile.validation_kind() != kind); - let mut passed = process_passed && !pytest_contradictory && !profile_mismatch; + // Canonical Ruff check disables fixes and exit-code overrides: a reported + // lint violation cannot agree with a successful native check. + let ruff_contradictory = tool == "python:ruff:check" + && process_passed + && evidence + .diagnostics + .as_ref() + .is_some_and(|diagnostics| !diagnostics.diagnostics.is_empty()); + let mut passed = + process_passed && !pytest_contradictory && !ruff_contradictory && !profile_mismatch; let mut value = json!({ "tool": tool, "kind": kind, diff --git a/src/tool_runtime/tests/jobs.rs b/src/tool_runtime/tests/jobs.rs index 59d2da9f8..f533b350a 100644 --- a/src/tool_runtime/tests/jobs.rs +++ b/src/tool_runtime/tests/jobs.rs @@ -3248,3 +3248,67 @@ fn run_shell_recovered_terminal_snapshot_stays_rich_after_projection() { assert_eq!(result.output["recovered_after_server_restart"], true); assert_run_shell_result_matches_schema(&result); } + +#[test] +fn validation_ruff_profiles_keep_native_outcomes_and_do_not_borrow_counts() { + use crate::tool_runtime::jobs::{structured_validation_evidence, validation_job_projection}; + for (tool, kind) in [ + ("python:ruff:check", "check"), + ("python:ruff:format", "format"), + ] { + let stdout = "test result: ok. 20 passed; 0 failed; 0 ignored\n2 passed in 0.01s\n"; + let evidence = + structured_validation_evidence(tool, kind, stdout, "error[E0308]: private", false); + assert_eq!( + evidence.diagnostics.as_ref().unwrap().parser, + "ruff_validation_parser_v1" + ); + assert!(evidence.tests_run_count.is_none()); + assert!(evidence.errors_count.is_none()); + assert!(evidence.warnings_count.is_none()); + for (status, exit, passed) in [ + ("completed", 0, true), + ("failed", 1, false), + ("failed", 2, false), + ] { + let value = validation_job_projection( + Some(tool), + Some(kind), + status, + Some(exit), + stdout, + "", + false, + None, + ) + .unwrap(); + assert_eq!(value["passed"], passed); + } + for status in ["timed_out", "cancelled", "lost"] { + let value = validation_job_projection( + Some(tool), + Some(kind), + status, + None, + stdout, + "", + false, + None, + ) + .unwrap(); + assert!(value["passed"].is_null()); + } + } + let contradiction = validation_job_projection( + Some("python:ruff:check"), + Some("check"), + "completed", + Some(0), + r#"{"code":"F401","location":{"row":1,"column":1},"filename":"private.py"}"#, + "", + false, + None, + ) + .unwrap(); + assert_eq!(contradiction["passed"], false); +} diff --git a/src/tool_runtime/tests/metadata.rs b/src/tool_runtime/tests/metadata.rs index 0c0945db6..5212f6306 100644 --- a/src/tool_runtime/tests/metadata.rs +++ b/src/tool_runtime/tests/metadata.rs @@ -519,6 +519,7 @@ async fn register_agent_projects_for_auth( project_all_packages_v1: false, project_validation_test_options_v1: false, project_validation_python_pytest_v1: false, + project_validation_python_ruff_v1: false, structured_go_test_tool: true, structured_go_test_packages: true, structured_process_argv: true, diff --git a/src/tool_runtime/tests/validation_handoff.rs b/src/tool_runtime/tests/validation_handoff.rs index ecbb1fb35..f67445090 100644 --- a/src/tool_runtime/tests/validation_handoff.rs +++ b/src/tool_runtime/tests/validation_handoff.rs @@ -108,8 +108,32 @@ pub(super) async fn complete_project_validation_plan_request( .and_then(|test| test.filter.as_deref()), ) .unwrap(); - let adapter = operation.adapter(); - let step = operation.build_readonly_plan().unwrap().structured_step; + let adapter = operation.evidence_profile(); + let step = if backend == "python" && check != webcodex_validation::SemanticCheck::Test { + // Mock Runner planning still resolves a real local manifest. The Server + // fixture is not proof of a real Ruff execution. + let root = tempfile::tempdir().unwrap(); + std::fs::write( + root.path().join("pyproject.toml"), + "[tool.ruff]\ntarget-version='py311'\n", + ) + .unwrap(); + webcodex_validation::resolve_project_validation_recipe( + root.path(), + None, + Some(webcodex_validation::RecipeId::Python), + &[check], + None, + None, + false, + None, + ) + .unwrap() + .steps + .remove(0) + } else { + operation.build_readonly_plan().unwrap().structured_step + }; let validation_target_id = operation.validation_target_id(Some(".")).unwrap(); let plan = ProjectValidationPlan { provenance: ProjectValidationProvenance { diff --git a/src/tool_runtime/tests/validation_handoff/project_validation.rs b/src/tool_runtime/tests/validation_handoff/project_validation.rs index a3eb3f06a..72dc68367 100644 --- a/src/tool_runtime/tests/validation_handoff/project_validation.rs +++ b/src/tool_runtime/tests/validation_handoff/project_validation.rs @@ -19,6 +19,7 @@ async fn setup(grace_ms: u64) -> ToolRuntime { project_all_packages_v1: true, project_validation_test_options_v1: true, project_validation_python_pytest_v1: true, + project_validation_python_ruff_v1: true, structured_go_test_json: true, structured_go_test_tool: true, structured_go_test_packages: true, @@ -604,6 +605,8 @@ async fn project_validation_readonly_adapters_preserve_source_fence_through_hand ("go", ProjectValidationAction::Check, "go_vet", ""), ("go", ProjectValidationAction::Test, "go_test", "{\"Action\":\"run\",\"Package\":\"example/pkg\",\"Test\":\"TestOne\"}\n{\"Action\":\"pass\",\"Package\":\"example/pkg\",\"Test\":\"TestOne\"}\n{\"Action\":\"pass\",\"Package\":\"example/pkg\"}\n"), ("python", ProjectValidationAction::Test, "python:pytest:test", "1 passed in 0.01s\n"), + ("python", ProjectValidationAction::Check, "python:ruff:check", ""), + ("python", ProjectValidationAction::FormatCheck, "python:ruff:format", "1 file already formatted\n"), ] { for external_writer in [false, true] { let runtime = setup(1).await; diff --git a/src/tool_runtime/validation.rs b/src/tool_runtime/validation.rs index 393b2f442..af28eab75 100644 --- a/src/tool_runtime/validation.rs +++ b/src/tool_runtime/validation.rs @@ -517,7 +517,7 @@ impl ToolRuntime { let plan = match serde_json::from_str::(response.stdout.as_deref().unwrap_or("")) { Ok(ProjectValidationPlanningResult::Ready { plan }) => plan, Ok(ProjectValidationPlanningResult::Unavailable { code, detected_backend }) => return ToolResult::err_with_output( - "Project validation unavailable; use a supported Rust/Go action or Python pytest tests with an existing environment.", + "Project validation unavailable; use a supported Rust/Go action or Python pytest/Ruff with existing tooling and supported project configuration.", json!({"execution_source":"project_validate", "execution_state":"not_started", "command_started":false, "command_completed":false, "failure_kind":code, "detected_backend":detected_backend})), Err(_) => return ToolResult::err("invalid Runner validation plan; upgrade Server and Runner together"), }; @@ -614,7 +614,7 @@ impl ToolRuntime { )) } }; - let adapter = operation.adapter(); + let adapter = operation.evidence_profile(); let no_run = validation_no_run(&operation); let validation_target_id = operation.validation_target_id(cwd.as_deref()); let plan = match operation.build_readonly_plan() { diff --git a/src/tool_runtime/validation_profile/mod.rs b/src/tool_runtime/validation_profile/mod.rs index 02103a8b1..4ec872560 100644 --- a/src/tool_runtime/validation_profile/mod.rs +++ b/src/tool_runtime/validation_profile/mod.rs @@ -18,6 +18,11 @@ pub(crate) struct ValidationRuntimeProfile { pub(crate) fn runtime_profile(operation: &ReadOnlyValidationOperation) -> ValidationRuntimeProfile { match operation { + ReadOnlyValidationOperation::PythonRuffCheck | ReadOnlyValidationOperation::PythonRuffFormat => ValidationRuntimeProfile { + default_timeout_secs: DEFAULT_CARGO_CHECK_TIMEOUT_SECS, + force_agent_handoff: false, + invalid_argument_guidance: "use project-local pyproject.toml [tool.ruff] with target-version and no extend; no flags or install.", + }, ReadOnlyValidationOperation::Python(_) => ValidationRuntimeProfile { default_timeout_secs: DEFAULT_CARGO_TEST_TIMEOUT_SECS, force_agent_handoff: true, From b3ebc67663a60a10a21bcc8e65326d1015d6a581 Mon Sep 17 00:00:00 2001 From: ITSGJY <133723203+ITSGJY@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:11:40 +0800 Subject: [PATCH 4/5] test(runner): make Ruff isolation fixture opt-in --- .../src/webcodex_runner/shell_tests.rs | 23 +++++++++++-------- .../python-ruff-project-validation.md | 5 +++- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs b/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs index 877239730..a785a2375 100644 --- a/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs +++ b/crates/webcodex-runner/src/webcodex_runner/shell_tests.rs @@ -2670,16 +2670,19 @@ fn project_validation_python_invalid_configured_interpreter_and_missing_ruff_nev assert!(!capture.exists()); } -#[cfg(unix)] +#[cfg(all(unix, feature = "runner-real-process-tests"))] #[test] -fn project_validation_ruff_system_python_ignores_project_and_pythonpath_impostors() { +#[ignore = "requires explicitly supplied existing Python 3 environment; never installs tooling"] +fn runner_real_process_project_validation_ruff_module_isolation() { use webcodex_core::runner_protocol::ShellJobValidationStep; - // Use a real system interpreter, never an interpreter supplied by the fixture. - let python = ["/usr/bin/python3", "/usr/local/bin/python3"] - .into_iter() - .find(|path| Path::new(path).is_file()) - .expect("system Python 3 is required for the Ruff isolation regression"); - let installed_ruff = Command::new(python) + // Optional real-process fixture. Keep ordinary unit tests toolchain-neutral. + let python = std::env::var("WEBCODEX_TEST_RUFF_PYTHON") + .expect("set WEBCODEX_TEST_RUFF_PYTHON to an existing Python 3 interpreter"); + assert!( + Path::new(&python).is_file(), + "configured Python interpreter must exist" + ); + let installed_ruff = Command::new(&python) .args(["-I", "-B", "-c", "import importlib.util; raise SystemExit(0 if importlib.util.find_spec('ruff') else 42)"]) .env_clear() .output().unwrap().status.success(); @@ -2704,7 +2707,7 @@ fn project_validation_ruff_system_python_ignores_project_and_pythonpath_impostor std::fs::write(project.join("pyproject.toml"), "[invalid TOML").unwrap(); let profile = PreparedShellProfile { profile_name: "system-python".into(), - program: python.into(), + program: python.clone(), args: vec![], dialect: ShellDialect::Posix, env_snapshot: std::collections::HashMap::from([( @@ -2739,7 +2742,7 @@ fn project_validation_ruff_system_python_ignores_project_and_pythonpath_impostor } } // Exercise canonical spawn argv even when this system has no installed Ruff. - assert!(!Command::new(python) + assert!(!Command::new(&python) .args(&step.args) .env_clear() .env("PYTHONPATH", &injected) diff --git a/docs/implementation/python-ruff-project-validation.md b/docs/implementation/python-ruff-project-validation.md index bf71e2d8f..b9c78f691 100644 --- a/docs/implementation/python-ruff-project-validation.md +++ b/docs/implementation/python-ruff-project-validation.md @@ -75,5 +75,8 @@ Local Rust tests separately cover configuration authority, canonical argv, identity/evidence, capability selection and admission, queue-time replanning, missing tooling, and workflow-session evidence retention. Stub interpreter tests verify process selection and environment wiring; they are not substitutes -for executing an actual Ruff binary. The contract is Ruff's requested read-only +for executing an actual Ruff binary. The opt-in real-process module-isolation +regression requires an existing Python 3 interpreter via +`WEBCODEX_TEST_RUFF_PYTHON` and the `runner-real-process-tests` feature; run +it explicitly with `--ignored`. The contract is Ruff's requested read-only check/format behavior in a trusted interpreter, not an OS-level filesystem sandbox. From 3685e5b0ad3d703a5c23a1676af619768781279f Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Fri, 9 Oct 2026 08:36:17 +0800 Subject: [PATCH 5/5] fix(tooling): align project validation failure schema --- .../src/registry/output_schemas/testing.rs | 58 +++++++++++++++- .../src/tests/output_schemas.rs | 69 +++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs b/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs index cecf4ab09..f78ee2ec1 100644 --- a/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs +++ b/crates/webcodex-tool-contracts/src/registry/output_schemas/testing.rs @@ -259,6 +259,60 @@ fn cargo_output_schema(tool_name: &str) -> Value { .into_iter() .map(Value::from) .collect::>(); + let terminal_not_started_failure_kinds = if tool_name == "project_validate" { + json!([ + "permission_denied", + "project_not_found", + "cwd_invalid", + "sandbox_unavailable", + "executor_unavailable", + "validation_plan_stale", + "validation_unavailable" + ]) + } else { + json!([ + "permission_denied", + "project_not_found", + "cwd_invalid", + "sandbox_unavailable", + "executor_unavailable" + ]) + }; + let prestart_failure_kinds = if tool_name == "project_validate" { + json!([ + "invalid_arguments", + "capability_unavailable", + "permission_denied", + "project_not_found", + "cwd_invalid", + "sandbox_unavailable", + "executor_unavailable", + "unknown_project", + "invalid_project_path", + "validation_recipe_not_found", + "validation_recipe_mismatch", + "validation_recipe_ambiguous", + "validation_manifest_invalid", + "validation_adapter_unavailable", + "validation_scope_invalid", + "validation_scope_unsupported", + "validation_scope_unavailable", + "validation_action_unsupported", + "dependency_policy_unsupported", + "test_filter_unsupported", + "validation_check_unavailable" + ]) + } else { + json!([ + "invalid_arguments", + "capability_unavailable", + "permission_denied", + "project_not_found", + "cwd_invalid", + "sandbox_unavailable", + "executor_unavailable" + ]) + }; let output = json!({ "type": "object", "properties": properties, @@ -431,7 +485,7 @@ fn cargo_output_schema(tool_name: &str) -> Value { "command_completed": {"const": false}, "execution_state": {"const": "not_started"}, "passed": {"const": false}, - "failure_kind": {"enum": ["permission_denied", "project_not_found", "cwd_invalid", "sandbox_unavailable", "executor_unavailable"]}, + "failure_kind": {"enum": terminal_not_started_failure_kinds}, "job_id": {"enum": []}, "job_status": {"enum": []}, "continuation": {"enum": []}, @@ -471,7 +525,7 @@ fn cargo_output_schema(tool_name: &str) -> Value { "properties": { "command_started": {"const": false}, "command_completed": {"const": false}, - "failure_kind": {"enum": ["invalid_arguments", "capability_unavailable", "permission_denied", "project_not_found", "cwd_invalid", "sandbox_unavailable", "executor_unavailable"]}, + "failure_kind": {"enum": prestart_failure_kinds}, "job_id": {"enum": []}, "job_status": {"enum": []}, "continuation": {"enum": []}, diff --git a/crates/webcodex-tool-contracts/src/tests/output_schemas.rs b/crates/webcodex-tool-contracts/src/tests/output_schemas.rs index 132794345..bccc8ac06 100644 --- a/crates/webcodex-tool-contracts/src/tests/output_schemas.rs +++ b/crates/webcodex-tool-contracts/src/tests/output_schemas.rs @@ -3374,6 +3374,75 @@ fn run_skill_resource_success_requires_provenance_and_keeps_lifecycle_constraint ); } +#[test] +fn project_validate_schema_admits_canonical_planning_and_replan_failures() { + let specs = registered_tool_specs(); + let schema = &spec_named(&specs, "project_validate").output_schema; + for failure_kind in [ + "unknown_project", + "invalid_project_path", + "validation_recipe_not_found", + "validation_recipe_mismatch", + "validation_recipe_ambiguous", + "validation_manifest_invalid", + "validation_adapter_unavailable", + "validation_scope_invalid", + "validation_scope_unsupported", + "validation_scope_unavailable", + "validation_action_unsupported", + "dependency_policy_unsupported", + "test_filter_unsupported", + "validation_check_unavailable", + ] { + let result = json!({ + "success": false, + "output": { + "execution_state": "not_started", + "command_started": false, + "command_completed": false, + "failure_kind": failure_kind, + "detected_backend": "python" + }, + "error": "project validation planning unavailable" + }); + test_support::validate_schema_instance(&result, schema).unwrap_or_else(|error| { + panic!("project_validate schema must admit planning failure {failure_kind}: {error}") + }); + } + + for failure_kind in ["validation_plan_stale", "validation_unavailable"] { + let result = json!({ + "success": false, + "output": { + "project": "agent:special:demo", + "command_summary": "python -I -B -m ruff check", + "cwd": ".", + "execution_state": "not_started", + "exit_code": null, + "duration_ms": null, + "stdout_tail": "", + "stderr_tail": "", + "stdout_lines": 0, + "stderr_lines": 0, + "stdout_truncated": false, + "stderr_truncated": false, + "promoted_to_job": false, + "terminal": true, + "command_started": false, + "command_completed": false, + "passed": false, + "effective_timeout_secs": 60, + "sync_wait_secs": 5, + "failure_kind": failure_kind + }, + "error": "project validation did not start" + }); + test_support::validate_schema_instance(&result, schema).unwrap_or_else(|error| { + panic!("project_validate schema must admit fenced failure {failure_kind}: {error}") + }); + } +} + #[test] fn browser_observation_schema_accepts_canonical_runner_output_and_rejects_private_ids() { let observe = crate::output_schema_for_tool("observe_browser");