Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,41 @@ use webcodex_core::runner_operation::{

#[test]
fn project_build_rechecks_manifest_and_dependency_state_after_queueing() {
for (marker, dependency, change_dependency) in [
("Cargo.toml", "Cargo.lock", false),
("Cargo.toml", "Cargo.lock", true),
("go.mod", "go.sum", false),
("go.mod", "go.sum", true),
for (marker, dependency, change_dependency, member_cwd) in [
("Cargo.toml", "Cargo.lock", false, false),
("Cargo.toml", "Cargo.lock", true, false),
("go.mod", "go.sum", false, false),
("go.mod", "go.sum", true, false),
("Cargo.toml", "Cargo.lock", false, true),
("Cargo.toml", "Cargo.lock", true, true),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("project");
let registry = temp.path().join("registry");
std::fs::create_dir_all(&root).unwrap();
std::fs::create_dir_all(&registry).unwrap();
std::fs::write(root.join(marker), "").unwrap();
if member_cwd {
std::fs::create_dir_all(root.join("member")).unwrap();
std::fs::write(
root.join("Cargo.toml"),
r#"[workspace]
members = ["member"]
resolver = "2"
"#,
)
.unwrap();
std::fs::write(
root.join("member/Cargo.toml"),
r#"[package]
name = "member"
version = "0.1.0"
edition = "2021"
"#,
)
.unwrap();
} else {
std::fs::write(root.join(marker), "").unwrap();
}
std::fs::write(root.join(dependency), "").unwrap();
std::fs::write(
registry.join("demo.toml"),
Expand All @@ -33,15 +56,15 @@ fn project_build_rechecks_manifest_and_dependency_state_after_queueing() {
};
let request = ProjectBuildRequest {
project_id: "demo".into(),
cwd: None,
cwd: member_cwd.then(|| "member".into()),
adapter: ProjectBuildAdapter::Auto,
scope: None,
};
let (plan, cwd) =
crate::webcodex_runner::project_build::plan(&policy, &registry, &request).unwrap();
let mut context = structured_process_context(&cwd, plan.process.args.len(), false);
context.runtime_project_id = Some("agent:structured-agent:demo".into());
context.project_cwd = Some(".".into());
context.project_cwd = Some(if member_cwd { "member" } else { "." }.into());
context.purpose = Some("build".into());
context
.structured_execution
Expand Down Expand Up @@ -111,18 +134,21 @@ fn project_build_rechecks_manifest_and_dependency_state_after_queueing() {
.slot_reserved;
manager.stop_all();
assert!(finished, "queued build worker did not terminate");
assert_eq!(result.status, "failed", "{marker}/{changed}: {result:?}");
assert_eq!(
result.status, "failed",
"{marker}/{changed}/member={member_cwd}: {result:?}"
);
assert_eq!(
result.command_execution_state,
Some(ShellCommandExecutionState::NotStarted),
"{marker}/{changed}: {result:?}"
"{marker}/{changed}/member={member_cwd}: {result:?}"
);
assert!(
result
.error
.as_deref()
.is_some_and(|error| error.contains("build_plan_stale")),
"{marker}/{changed}: {result:?}"
"{marker}/{changed}/member={member_cwd}: {result:?}"
);
assert!(!reserved, "rejected build retained the execution slot");
assert!(!root.join("target").exists());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,41 @@ use webcodex_core::runner_protocol::{ShellJobContext, ShellJobValidationMetadata

#[test]
fn project_validation_rechecks_manifest_and_dependency_state_after_queueing() {
for (marker, dependency, change_dependency) in [
("Cargo.toml", "Cargo.lock", false),
("Cargo.toml", "Cargo.lock", true),
("go.mod", "go.sum", false),
("go.mod", "go.sum", true),
for (marker, dependency, change_dependency, member_cwd) in [
("Cargo.toml", "Cargo.lock", false, false),
("Cargo.toml", "Cargo.lock", true, false),
("go.mod", "go.sum", false, false),
("go.mod", "go.sum", true, false),
("Cargo.toml", "Cargo.lock", false, true),
("Cargo.toml", "Cargo.lock", true, true),
] {
let temp = tempfile::tempdir().unwrap();
let root = temp.path().join("project");
let registry = temp.path().join("registry");
std::fs::create_dir_all(&root).unwrap();
std::fs::create_dir_all(&registry).unwrap();
std::fs::write(root.join(marker), "").unwrap();
if member_cwd {
std::fs::create_dir_all(root.join("member")).unwrap();
std::fs::write(
root.join("Cargo.toml"),
r#"[workspace]
members = ["member"]
resolver = "2"
"#,
)
.unwrap();
std::fs::write(
root.join("member/Cargo.toml"),
r#"[package]
name = "member"
version = "0.1.0"
edition = "2021"
"#,
)
.unwrap();
} else {
std::fs::write(root.join(marker), "").unwrap();
}
std::fs::write(root.join(dependency), "").unwrap();
std::fs::write(
registry.join("demo.toml"),
Expand All @@ -37,7 +60,7 @@ fn project_validation_rechecks_manifest_and_dependency_state_after_queueing() {
};
let request = ProjectValidationRequest {
project_id: "demo".into(),
cwd: None,
cwd: member_cwd.then(|| "member".into()),
action: ProjectValidationAction::Check,
adapter: ProjectValidationAdapter::Auto,
scope: None,
Expand Down Expand Up @@ -68,7 +91,7 @@ fn project_validation_rechecks_manifest_and_dependency_state_after_queueing() {
validation: Some(metadata),
workflow_session_id: None,
ssh_resource: None,
project_cwd: Some(".".into()),
project_cwd: Some(if member_cwd { "member" } else { "." }.into()),
cwd: Some(cwd.to_string_lossy().into_owned()),
purpose: Some("validation".into()),
shell: None,
Expand Down Expand Up @@ -143,18 +166,21 @@ fn project_validation_rechecks_manifest_and_dependency_state_after_queueing() {
.slot_reserved;
manager.stop_all();

assert_eq!(result.status, "failed", "{marker}/{changed}: {result:?}");
assert_eq!(
result.status, "failed",
"{marker}/{changed}/member={member_cwd}: {result:?}"
);
assert_eq!(
result.command_execution_state,
Some(ShellCommandExecutionState::NotStarted),
"{marker}/{changed}: {result:?}"
"{marker}/{changed}/member={member_cwd}: {result:?}"
);
assert!(
result
.error
.as_deref()
.is_some_and(|error| error.contains("validation_plan_stale")),
"{marker}/{changed}: {result:?}"
"{marker}/{changed}/member={member_cwd}: {result:?}"
);
assert!(!reserved, "rejected validation retained the execution slot");
assert!(lock_unpoison(&manager.queued).is_empty());
Expand Down
20 changes: 8 additions & 12 deletions crates/webcodex-runner/src/webcodex_runner/project_build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ use webcodex_core::project_build::{
};
use webcodex_core::runner_operation::RunnerJobOperation;
use webcodex_workspace::project_recipe::{
digest_project_recipe_files, project_recipe_dependency_state_files,
resolve_project_recipe_root, ProjectRecipeId, ProjectRecipeResolutionError,
digest_project_recipe_files, project_recipe_provenance_files, resolve_project_recipe_root,
ProjectRecipeId, ProjectRecipeResolutionError,
};

fn unavailable(code: &str, backend: Option<&str>) -> ProjectBuildPlanningResult {
Expand Down Expand Up @@ -79,16 +79,12 @@ pub(crate) fn plan(

let process = canonical_project_build_process(backend.as_str(), request)
.map_err(|_| unavailable("build_scope_invalid", Some(backend.as_str())))?;
let manifest_digest = digest_project_recipe_files(
&resolved.execution_root,
std::iter::once(resolved.marker_path()).chain(
project_recipe_dependency_state_files(backend)
.expect("project build supports only Rust/Go")
.iter()
.map(|file| resolved.absolute_root.join(file)),
),
)
.map_err(|error| unavailable(map_recipe_error(error), Some(backend.as_str())))?;
let provenance_files = project_recipe_provenance_files(&resolved)
.map_err(|error| unavailable(map_recipe_error(error), Some(backend.as_str())))?
.expect("project build supports only Rust/Go");
let manifest_digest =
digest_project_recipe_files(&resolved.execution_root, provenance_files)
.map_err(|error| unavailable(map_recipe_error(error), Some(backend.as_str())))?;

let plan = ProjectBuildPlan {
provenance: ProjectBuildProvenance {
Expand Down
53 changes: 34 additions & 19 deletions crates/webcodex-validation/src/recipe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use std::fs;
use std::path::Path;
use webcodex_core::runner_protocol::{normalize_rust_test_filter, ShellJobValidationStep};
use webcodex_workspace::project_recipe::{
digest_project_recipe_files, project_recipe_dependency_state_files, read_project_recipe_file,
digest_project_recipe_files, project_recipe_provenance_files, read_project_recipe_file,
resolve_project_recipe_root, ProjectRecipeResolutionError,
};

Expand Down Expand Up @@ -166,22 +166,40 @@ pub fn resolve_validation_recipe_with_packages(
} else {
let manifest =
read_project_recipe_file(root, &marker_path).map_err(map_project_recipe_error)?;
let (steps, extra_digest_files) = match recipe {
let (steps, provenance_files) = match recipe {
RecipeId::Rust | RecipeId::Go => {
canonical_adapter_steps(recipe, checks, test_filter.as_deref(), package_scope)?
let steps =
canonical_adapter_steps(recipe, checks, test_filter.as_deref(), package_scope)?;
let provenance_files = project_recipe_provenance_files(&resolved_root)
.map_err(map_project_recipe_error)?
.expect("canonical project validation adapters are Rust/Go");
(steps, provenance_files)
}
RecipeId::Node => {
let (steps, extra_digest_files) = node_steps(&recipe_root, &manifest, checks)?;
let provenance_files = std::iter::once(marker_path.clone())
.chain(
extra_digest_files
.into_iter()
.map(|file| recipe_root.join(file)),
)
.collect();
(steps, provenance_files)
}
RecipeId::Python => {
let (steps, extra_digest_files) = python_steps(&manifest, checks)?;
let provenance_files = std::iter::once(marker_path.clone())
.chain(
extra_digest_files
.into_iter()
.map(|file| recipe_root.join(file)),
)
.collect();
(steps, provenance_files)
}
RecipeId::Node => node_steps(&recipe_root, &manifest, checks)?,
RecipeId::Python => python_steps(&manifest, checks)?,
};
let manifest_digest = digest_project_recipe_files(
root,
std::iter::once(marker_path).chain(
extra_digest_files
.into_iter()
.map(|file| recipe_root.join(file)),
),
)
.map_err(map_project_recipe_error)?;
let manifest_digest = digest_project_recipe_files(root, provenance_files)
.map_err(map_project_recipe_error)?;
(steps, manifest_digest)
};
let invocation_digest = format!(
Expand Down Expand Up @@ -209,7 +227,7 @@ fn canonical_adapter_steps(
checks: &[SemanticCheck],
test_filter: Option<&str>,
package_scope: Option<&[String]>,
) -> Result<(Vec<ShellJobValidationStep>, Vec<&'static str>), RecipeError> {
) -> Result<Vec<ShellJobValidationStep>, RecipeError> {
let mut steps = Vec::with_capacity(checks.len());
for check in checks {
let operation = crate::project_validation_operation(
Expand Down Expand Up @@ -240,10 +258,7 @@ fn canonical_adapter_steps(
})?;
steps.push(plan.structured_step);
}
let extra_digest_files = project_recipe_dependency_state_files(recipe)
.expect("canonical project validation adapters are Rust/Go")
.to_vec();
Ok((steps, extra_digest_files))
Ok(steps)
}

fn node_steps(
Expand Down
60 changes: 60 additions & 0 deletions crates/webcodex-validation/src/recipe_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,66 @@ fn project_recipe_filtered_execution_is_deterministic_without_repeating_arg_buil
}
}

#[test]
fn rust_member_manifest_digest_tracks_workspace_root_inputs() {
let temp = tempfile::tempdir().unwrap();
write(
temp.path(),
"Cargo.toml",
"[workspace]\nmembers=['member']\nresolver='2'\n",
);
write(temp.path(), "Cargo.lock", "version = 4\n");
write(
temp.path(),
"member/Cargo.toml",
"[package]\nname='member'\nversion='0.1.0'\nedition='2021'\n",
);

let resolve_member = || {
resolve(
temp.path(),
Some("member"),
Some(RecipeId::Rust),
&[SemanticCheck::Check],
None,
)
.unwrap()
};
let original = resolve_member();
assert_eq!(original.recipe_root_relative, "member");
assert_eq!(original.steps[0].args, ["check", "--all-targets"]);

write(
temp.path(),
"member/src/lib.rs",
"pub fn value() -> u8 { 1 }\n",
);
let source_only = resolve_member();
assert_eq!(source_only.manifest_digest, original.manifest_digest);
assert_eq!(source_only.invocation_digest, original.invocation_digest);

write(temp.path(), "Cargo.lock", "version = 3\n");
let lock_changed = resolve_member();
assert_ne!(lock_changed.manifest_digest, original.manifest_digest);
assert_eq!(lock_changed.invocation_digest, original.invocation_digest);

write(temp.path(), "Cargo.lock", "version = 4\n");
write(
temp.path(),
"Cargo.toml",
"[workspace]\nmembers=['member']\nresolver='2'\n[profile.dev]\nopt-level=1\n",
);
let workspace_manifest_changed = resolve_member();
assert_ne!(
workspace_manifest_changed.manifest_digest,
original.manifest_digest
);
assert_eq!(
workspace_manifest_changed.invocation_digest,
original.invocation_digest
);
}

#[test]
fn unavailable_checks_and_unsupported_filters_fail_before_execution() {
let go = tempfile::tempdir().unwrap();
Expand Down
1 change: 1 addition & 0 deletions crates/webcodex-workspace/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ webcodex-core = { path = "../webcodex-core" }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.10"
toml = "0.8"

[dev-dependencies]
tempfile = "3"
Loading
Loading