diff --git a/Cargo.lock b/Cargo.lock index f6c0206ef7..10661d8886 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2142,6 +2142,7 @@ dependencies = [ "serde_norway", "sha2", "shell-words", + "signal-hook", "sigpipe", "thiserror 2.0.20", "tokio", @@ -2167,6 +2168,7 @@ dependencies = [ "serde_norway", "sha2", "thiserror 2.0.20", + "tokio-util", "url", "uuid", ] diff --git a/README.md b/README.md index a74344c630..e5318652be 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,20 @@ Set `build.x-recipe` to `dockerfile`, `railpack`, or `auto` (the default) to con selection. A failed recipe never falls back to another. Railpack uses matching pinned 0.39.0 preparation/frontend tooling with BuildKit -0.26.2, provisioned through Docker, and builds one native Linux AMD64 or ARM64 -image. Service variables default build variables; Compose `build.args` and then +0.26.2, provisioned through Docker. Builds default to the native Linux AMD64 or +ARM64 platform. Set `build.platforms: [linux/amd64, linux/arm64]` to build both; +the execution host must provide native or emulated support for each platform. +Separate solves are assembled locally with pinned regctl 0.11.6 into one immutable +image in Docker’s containerd store, with every platform’s content verified. +Multi-platform builds currently require local output and reject requested +`build.provenance` or `build.sbom`; Dockerfiles remain limited to one platform. Deploy platform +inference is separate work. + +The [prototype findings](https://github.com/getployz/ployz2/blob/c3ca5519a4607256ffb28052d77a1c7d89f1bbe1/prototypes/railpack-transfer/FINDINGS.md) +preserve the evidence for this assembly approach. They used shipped beta binaries +and emulated ARM64; they do not qualify this implementation or native ARM64. + +Service variables default build variables; Compose `build.args` and then `--build-arg` override them without changing runtime values. Values travel as private secret mounts. Docker ignore patterns and Railpack's configured exclusions apply before source transfer. Recipes can still print or embed values. diff --git a/crates/ployz-build/Cargo.toml b/crates/ployz-build/Cargo.toml index 32fae3a235..9b2f8044ac 100644 --- a/crates/ployz-build/Cargo.toml +++ b/crates/ployz-build/Cargo.toml @@ -15,6 +15,7 @@ rustix = { version = "1.1.4", features = ["fs", "process"] } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" sha2 = "0.11.0" +tokio-util = "0.7.19" thiserror = "2.0.20" [lints] diff --git a/crates/ployz-build/src/builder.rs b/crates/ployz-build/src/builder.rs index ddf7e3662e..122dd98bcd 100644 --- a/crates/ployz-build/src/builder.rs +++ b/crates/ployz-build/src/builder.rs @@ -51,6 +51,9 @@ impl<'a> Builder<'a> { } /// Read the running worker, not the Machine's advertised architecture. + /// + /// # Errors + /// Refuses unsupported image stores, hosts, resource limits, or requested worker platforms. pub(crate) fn native_platform( &self, targets: &[crate::Target], @@ -118,8 +121,13 @@ impl<'a> Builder<'a> { "the selected BuildKit worker reported no capability".into(), ) })?; - for target in targets { - let requested = target.platform.as_deref().unwrap_or(&native); + for requested in targets.iter().flat_map(|target| { + target + .platforms + .iter() + .map(String::as_str) + .chain(target.platforms.is_empty().then_some(native.as_str())) + }) { if !nodes.iter().any(|node| { node.get("Status").and_then(serde_json::Value::as_str) == Some("running") && node @@ -175,7 +183,17 @@ impl<'a> Builder<'a> { }); } match cleanup { - Ok(()) => result, + Ok(()) => result.and_then(|output| { + if self + .docker + .cancellation + .is_some_and(crate::Cancellation::is_cancelled) + { + Err(BuildError::Cancelled) + } else { + Ok(output) + } + }), Err(error) => Err(BuildError::UncertainTermination(match result { Ok(_) => format!("output handling completed; cleanup failed: {error}"), Err(cause) => format!("{cause}; cleanup failed: {error}"), @@ -261,13 +279,19 @@ impl Lock { /// /// # Errors /// Fails when the lock file cannot be opened or locked. - pub(crate) fn acquire() -> Result { - Self::acquire_in(&directory()) + pub(crate) fn acquire(cancellation: &crate::Cancellation) -> Result { + Self::acquire_in(&directory(), cancellation) } - fn acquire_in(directory: &std::path::Path) -> Result { + fn acquire_in( + directory: &std::path::Path, + cancellation: &crate::Cancellation, + ) -> Result { let deadline = std::time::Instant::now() + QUEUE_TIMEOUT; loop { + if cancellation.is_cancelled() { + return Err(BuildError::Cancelled); + } match Self::try_acquire_in(directory) { Err(BuildError::Busy) if std::time::Instant::now() < deadline => { std::thread::sleep(std::time::Duration::from_millis(200)); @@ -444,7 +468,7 @@ esac let result = builder.native_platform( &[crate::Target { name: "api".into(), - platform: Some(platform.into()), + platforms: vec![platform.into()], }], &resources, ); @@ -564,7 +588,7 @@ esac progress: None, }; - let lock = Lock::acquire_in(&directory).unwrap(); + let lock = Lock::acquire_in(&directory, &crate::Cancellation::default()).unwrap(); let builder = Builder::acquire(&docker, lock, &crate::policy::Resources::default()).unwrap(); let name = builder_name(); diff --git a/crates/ployz-build/src/execution.rs b/crates/ployz-build/src/execution.rs index f8119c42a5..1d645d0b09 100644 --- a/crates/ployz-build/src/execution.rs +++ b/crates/ployz-build/src/execution.rs @@ -2,14 +2,7 @@ use crate::{BuildError, Deadline, EXECUTION_TIMEOUT, builder::Lock}; use serde::{Deserialize, Serialize}; -use std::{ - path::PathBuf, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Duration, -}; +use std::{path::PathBuf, time::Duration}; /// Earliest observed phase of one Build attempt. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -72,20 +65,9 @@ impl WorkEvidence { } } -/// A request to stop, never proof that execution stopped. -#[derive(Clone, Default)] -pub struct Cancellation(Arc); -impl Cancellation { - /// Request bounded termination. - pub fn cancel(&self) { - self.0.store(true, Ordering::Release); - } - /// Whether termination has been requested. - #[must_use] - pub fn is_cancelled(&self) -> bool { - self.0.load(Ordering::Acquire) - } -} +/// A caller-owned request to stop, never proof that execution stopped. +/// Shared by synchronous Build execution and subsequent async command phases. +pub use tokio_util::sync::CancellationToken as Cancellation; /// Execution-host settings. These are never deserialized from a Build request. #[derive(Clone)] @@ -210,14 +192,14 @@ impl Admission { cancellation: Cancellation::default(), }) } - pub(crate) fn wait() -> Result { - let lock = Lock::acquire()?; + pub(crate) fn wait(cancellation: &Cancellation) -> Result { + let lock = Lock::acquire(cancellation)?; let resources = crate::policy::Resources::load(&HostPolicy::default().configuration_file)?; Ok(Self { resources, lock, deadline: Deadline::starting_now(EXECUTION_TIMEOUT), - cancellation: Cancellation::default(), + cancellation: cancellation.clone(), }) } /// Handle for requesting this attempt to stop. diff --git a/crates/ployz-build/src/index.rs b/crates/ployz-build/src/index.rs new file mode 100644 index 0000000000..f08ae5a2f3 --- /dev/null +++ b/crates/ployz-build/src/index.rs @@ -0,0 +1,207 @@ +//! Railpack 0.39.0 needs separate solves and upstream OCI index assembly. +//! Primary evidence (shipped beta binaries, ARM64 under emulation): +//! https://github.com/getployz/ployz2/blob/c3ca5519a4607256ffb28052d77a1c7d89f1bbe1/prototypes/railpack-transfer/FINDINGS.md + +use crate::{ + BuildError, BuiltImage, Docker, Planned, Progress, Request, Stage, Streams, TargetEvidence, + TargetMetadata, bake_arguments, builder::Builder, builder_name, +}; +use std::{collections::BTreeMap, path::Path}; + +// regctl 0.11.6, matching the prototype. Containerized so remote Docker also works. +const REGCTL: &str = + "regclient/regctl@sha256:5fe7c6a6206e1d9d71af719674c9db03e74a4ae221a32089b691f5b55533d6ea"; + +/// Solve each requested platform, assemble its exact content, and verify the local image. +/// +/// # Errors +/// Reports solve, assembly, import or content failures; cleanup failure remains uncertain. +pub(crate) fn build( + docker: &Docker<'_>, + builder: &Builder<'_>, + request: &Request<'_>, + target: &Planned<'_>, + overrides: Option<&Path>, +) -> Result { + let name = format!("{}-assemble", builder_name()); + docker.with_container( + &name, + &[ + "--network", + "none", + "--entrypoint", + "/bin/sleep", + REGCTL, + "infinity", + ], + |name| { + docker + .run("start image assembly", &["start", name], Streams::Captured) + .map_err(|error| error.at(Stage::Preparation))?; + let output = |action, arguments: &[&str], streams| { + docker + .run(action, arguments, streams) + .map_err(|error| error.at(Stage::Output)) + }; + let regctl = |action, args: &[&str]| { + let mut command = vec!["exec", name, "regctl"]; + command.extend_from_slice(args); + output(action, &command, Streams::Captured) + }; + let mut refs = Vec::new(); + let mut tags = Vec::new(); + for (number, platform) in target.target.platforms.iter().enumerate() { + let archive = request.working_dir.join("private/railpack/variant.tar"); + let metadata = request.working_dir.join("private/railpack/variant.json"); + let mut args = + bake_arguments(request, std::slice::from_ref(target), &metadata, overrides); + args.retain(|arg| arg != "--load"); + for setting in [ + format!("{}.platform={platform}", target.bake), + format!("{}.output=type=oci,dest={}", target.bake, archive.display()), + format!("{}.attest=type=provenance,disabled=true", target.bake), + ] { + args.extend(["--set".into(), setting]); + } + if let Some(progress) = docker.progress { + progress(Progress::Stage(Stage::Building)); + } + builder.run(&args, || { + if let Some(progress) = docker.progress { + progress(Progress::Target { + name: target.target.name.clone(), + outcome: TargetEvidence::Unknown, + }); + } + })?; + if let Some(progress) = docker.progress { + progress(Progress::Stage(Stage::Output)); + } + let results: BTreeMap = + serde_json::from_slice(&std::fs::read(&metadata).map_err(result_error)?) + .map_err(result_error)?; + let result = results.get(&target.bake).ok_or_else(|| { + BuildError::Result("Railpack solve produced no image result".into()) + })?; + let current_tags = result.tags(); + if number == 0 { + tags = current_tags; + } else if tags != current_tags { + return Err(BuildError::Result( + "Railpack solves reported different image tags".into(), + )); + } + output( + "copy Railpack variant for assembly", + &[ + "cp", + &archive.to_string_lossy(), + &format!("{name}:/tmp/variant.tar"), + ], + Streams::Captured, + )?; + let reference = format!("ocidir:///tmp/layout@{}", result.digest); + regctl( + "import Railpack variant", + &["image", "import", &reference, "/tmp/variant.tar"], + )?; + refs.push(reference); + } + let tag = tags + .first() + .ok_or_else(|| BuildError::Result("Railpack solve tagged no image".into()))?; + let mut args = vec!["index", "create", "ocidir:///tmp/layout:complete"]; + for reference in &refs { + args.extend(["--ref", reference]); + } + regctl("assemble Railpack image index", &args)?; + let digest = regctl( + "read assembled image identity", + &["image", "digest", "ocidir:///tmp/layout:complete"], + )?; + let digest = digest.trim(); + if !digest + .strip_prefix("sha256:") + .is_some_and(|hex| hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit())) + { + return Err(BuildError::Result( + "assembly returned an invalid image digest".into(), + )); + } + regctl( + "export assembled Railpack image", + &[ + "image", + "export", + "ocidir:///tmp/layout:complete", + "/tmp/complete.tar", + "--name", + tag, + ], + )?; + let archive = request.working_dir.join("private/railpack/complete.tar"); + output( + "copy assembled Railpack image", + &[ + "cp", + &format!("{name}:/tmp/complete.tar"), + &archive.to_string_lossy(), + ], + Streams::Captured, + )?; + output( + "load assembled Railpack image", + &["image", "load", "--input", &archive.to_string_lossy()], + Streams::Captured, + )?; + let reference = digest.to_owned(); + let inspected = output( + "inspect assembled Railpack image", + &["image", "inspect", &reference, "--format", "{{json .}}"], + Streams::Captured, + )?; + let image: crate::ImageInspection = + serde_json::from_str(&inspected).map_err(result_error)?; + if !image.descriptor.is_some_and(|d| { + d.digest == digest && d.media_type == "application/vnd.oci.image.index.v1+json" + }) { + return Err(BuildError::Result( + "the local store does not hold the assembled Railpack index".into(), + )); + } + for platform in &target.target.platforms { + // Export forces Docker to read the manifest, config and every layer. + // Inspect alone can list an index variant whose content is absent. + output( + "verify Railpack platform content", + &["image", "save", "--platform", platform, &reference], + Streams::Discarded, + )?; + } + for tag in &tags { + output( + "tag completed Railpack image", + &["image", "tag", &reference, tag], + Streams::Captured, + )?; + } + let image = BuiltImage { + reference, + tags, + platforms: target.target.platforms.clone(), + location: docker.location(), + }; + if let Some(progress) = docker.progress { + progress(Progress::Target { + name: target.target.name.clone(), + outcome: TargetEvidence::Image(image.clone()), + }); + } + Ok(image) + }, + ) +} + +fn result_error(error: impl std::fmt::Display) -> BuildError { + BuildError::Result(format!("read Railpack image result: {error}")) +} diff --git a/crates/ployz-build/src/lib.rs b/crates/ployz-build/src/lib.rs index 01a62822ee..bbf628a606 100644 --- a/crates/ployz-build/src/lib.rs +++ b/crates/ployz-build/src/lib.rs @@ -12,6 +12,7 @@ mod builder; mod execution; +mod index; mod policy; pub use policy::clear_cache; mod received_recipe; @@ -87,9 +88,9 @@ pub struct Request<'a> { pub struct Target { /// Caller's name for this image, as the captured Compose file names it. pub name: String, - /// Platform the capture asks for, which the completed image must carry. - /// `None` accepts whichever platform the builder produces. - pub platform: Option, + /// Platforms the capture asks for, which the completed image must carry. + /// Empty accepts the native platform. + pub platforms: Vec, } /// What an attempt does with its result. These outcomes are exclusive: an @@ -109,13 +110,15 @@ pub enum Output { /// the content this attempt observed there. #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct BuiltImage { - /// Digest-pinned reference. Binds later use to this attempt's content, so - /// a later Build moving a shared tag cannot substitute its image. + /// Docker content digest on the execution host. Unlike repository@digest, + /// this still resolves after the repository's last tag moves to another image. pub reference: String, /// Tags this attempt applied, as the execution host recorded them. pub tags: Vec, - /// The platform actually present, not the one requested. - pub platform: String, + /// Platforms verified in the execution host’s image store. + pub platforms: Vec, + /// Docker endpoint holding this content. + pub location: String, } /// Why a Build Attempt did not produce the image it was asked for. @@ -149,7 +152,7 @@ pub enum BuildError { #[error("the build exceeded its {0}s execution timeout and was terminated")] TimedOut(u64), /// The attempt stopped, but its termination could not be observed. - #[error("the build was terminated but its builder did not stop: {0}")] + #[error("build resource cleanup could not be confirmed: {0}")] UncertainTermination(String), /// The build finished, but its result is not the image it claims. #[error("{0}")] @@ -206,7 +209,8 @@ impl BuildError { /// [`EXECUTION_TIMEOUT`], with a separate bounded budget for cleanup. /// /// Returns the completed images in the order of `targets`. Validation and -/// registry publication retain no local image, so they return none. +/// registry publication retain no local image, so they return none. Cancellation +/// is supplied by the caller; this function never installs process signal handlers. /// /// # Errors /// Returns a prerequisite error when Docker cannot serve the Build, a request @@ -214,8 +218,11 @@ impl BuildError { /// diagnosis, a timeout, an uncertain outcome when termination could not be /// observed, or a result error when the completed image is not the content it /// claims. -pub fn execute(request: &Request<'_>) -> Result, BuildError> { - execute_admitted(request, Admission::wait()?, &|event| { +pub fn execute( + request: &Request<'_>, + cancellation: &Cancellation, +) -> Result, BuildError> { + execute_admitted(request, Admission::wait(cancellation)?, &|event| { if let Progress::Output(bytes) = event { use std::io::Write as _; let _ = std::io::stderr().write_all(&bytes); @@ -235,6 +242,18 @@ pub fn execute_admitted( progress: &(dyn Fn(Progress) + Sync), ) -> Result, BuildError> { let planned = plan(request.targets)?; + for target in request.targets { + if target.platforms.len() > 1 && !request.railpack.iter().any(|r| r.name == target.name) { + return Err(BuildError::Request( + "a Dockerfile Build produces one platform".into(), + )); + } + } + if request.output != Output::Load && request.targets.iter().any(|t| t.platforms.len() > 1) { + return Err(BuildError::Request( + "multi-platform Railpack builds require local image output".into(), + )); + } if planned.is_empty() { return Ok(Vec::new()); } @@ -269,30 +288,39 @@ pub fn execute_admitted( })?; let builder = Builder::acquire(&docker, admission.lock, &admission.resources) .map_err(|error| error.at(Stage::Preparation))?; + let mut preparation = None; let result = (|| { let native = builder .native_platform(request.targets, &admission.resources) .map_err(|error| error.at(Stage::Preparation))?; - let preparation = railpack::prepare(&docker, request, &admission.resources) + preparation = railpack::prepare(&docker, request, &native, &admission.resources) .map_err(|error| error.at(Stage::Preparation))?; let overrides = preparation .as_ref() .map(railpack::Preparation::override_file); + let (multi, ordinary): (Vec<_>, Vec<_>) = planned + .into_iter() + .partition(|target| target.target.platforms.len() > 1); + let mut images = Vec::new(); // A successful per-target push is publication evidence. A failed batch // cannot tell us which of its registry exports completed. let batch_size = if request.output == Output::Registry { 1 } else { - planned.len() + ordinary.len().max(1) }; - for batch in planned.chunks(batch_size) { + for batch in ordinary.chunks(batch_size) { let mut arguments = bake_arguments(request, batch, &metadata, overrides.as_deref()); - for target in &planned { + for target in batch { arguments.push("--set".into()); arguments.push(format!( "{}.platform={}", target.bake, - target.target.platform.as_deref().unwrap_or(&native) + target + .target + .platforms + .first() + .map_or(native.as_str(), String::as_str) )); } progress(Progress::Stage(Stage::Building)); @@ -319,8 +347,10 @@ pub fn execute_admitted( progress(Progress::Stage(Stage::Output)); match request.output { Output::Load => { - return built_images(&docker, &metadata, batch, progress) - .map_err(|error| error.at(Stage::Output)); + images.extend( + built_images(&docker, &metadata, batch, progress) + .map_err(|error| error.at(Stage::Output))?, + ); } Output::Registry | Output::Validate => { for target in batch { @@ -336,7 +366,21 @@ pub fn execute_admitted( } } } - Ok(Vec::new()) + for target in &multi { + let image = index::build(&docker, &builder, request, target, overrides.as_deref())?; + images.push(image); + } + let mut by_name = ordinary + .iter() + .chain(&multi) + .map(|target| target.target.name.as_str()) + .zip(images) + .collect::>(); + Ok(request + .targets + .iter() + .filter_map(|target| by_name.remove(target.name.as_str())) + .collect()) })(); progress(Progress::Stage(Stage::Cleanup)); // An ephemeral worker may exit before periodic GC runs. Use upstream @@ -452,31 +496,23 @@ fn built_images( )) }) })?; - let tags = result - .name - .split(',') - .map(str::trim) - .filter(|tag| !tag.is_empty()) - .map(ToOwned::to_owned) - .collect::>(); - let repository = tags - .first() - .map(|tag| repository(tag)) - .ok_or_else(|| { - BuildError::Result(format!("the build tagged no image for '{name}'")) - })? - .to_owned(); - let reference = format!("{repository}@{}", result.digest); + let tags = result.tags(); + if tags.is_empty() { + return Err(BuildError::Result(format!( + "the build tagged no image for '{name}'" + ))); + } + let reference = result.digest; let platform = verify( docker, &reference, - &result.digest, - planned.target.platform.as_deref(), + planned.target.platforms.first().map(String::as_str), )?; let image = BuiltImage { reference, tags, - platform, + platforms: vec![platform], + location: docker.location(), }; progress(Progress::Target { name: name.clone(), @@ -503,7 +539,6 @@ fn built_images( fn verify( docker: &Docker<'_>, reference: &str, - digest: &str, requested: Option<&str>, ) -> Result { let inspected = docker @@ -528,7 +563,7 @@ fn verify( "the local image store reports no content descriptor for {reference}; Ployz Builds require Docker's containerd image store" ))); }; - if descriptor.digest != digest { + if descriptor.digest != reference { return Err(BuildError::Result(format!( "the local image store holds {} for {reference} rather than the completed content", descriptor.digest @@ -565,15 +600,6 @@ fn covers(observed: &str, requested: &str) -> bool { .is_some_and(|rest| rest.starts_with('/')) } -/// The repository part of a tag, keeping a registry port intact. -fn repository(tag: &str) -> &str { - let start = tag.rfind('/').map_or(0, |slash| slash + 1); - match tag.get(start..).and_then(|name| name.find([':', '@'])) { - Some(offset) => tag.get(..start + offset).unwrap_or(tag), - None => tag, - } -} - #[derive(Deserialize)] struct TargetMetadata { #[serde(rename = "containerimage.digest")] @@ -582,6 +608,17 @@ struct TargetMetadata { name: String, } +impl TargetMetadata { + fn tags(&self) -> Vec { + self.name + .split(',') + .map(str::trim) + .filter(|tag| !tag.is_empty()) + .map(ToOwned::to_owned) + .collect() + } +} + #[derive(Deserialize)] struct ImageInspection { #[serde(rename = "Os")] @@ -628,6 +665,8 @@ pub(crate) enum Streams { Inherited, /// Evidence this crate reads. Captured, + /// Read failure diagnostics while discarding binary image content. + Discarded, } /// The Docker CLI this attempt drives, with the values captured for it. @@ -644,6 +683,14 @@ pub(crate) struct Docker<'a> { } impl<'a> Docker<'a> { + fn location(&self) -> String { + self.environment + .get("DOCKER_HOST") + .filter(|host| !host.is_empty()) + .cloned() + .unwrap_or_else(|| "unix:///var/run/docker.sock".into()) + } + /// Host policy and builder ownership apply only to this Machine's Docker. fn require_local(&self) -> Result<(), BuildError> { if self @@ -696,6 +743,48 @@ impl<'a> Docker<'a> { } } + /// Run work with a private helper container, confirming its absence afterwards. + /// + /// # Errors + /// Preserves the work's failure unless container removal cannot be confirmed. + pub(crate) fn with_container( + &self, + name: &str, + arguments: &[&str], + work: impl FnOnce(&str) -> Result, + ) -> Result { + self.remove_container(name)?; + // A failed or interrupted create may still have reached Docker. Always + // confirm absence, including when no successful create was observed. + let mut create = vec!["create", "--name", name]; + create.extend_from_slice(arguments); + let result = self + .run("create build helper", &create, Streams::Captured) + .map_err(|error| error.at(Stage::Preparation)) + .and_then(|_| work(name)); + match (result, self.remove_container(name)) { + (result, Ok(())) => result, + (Ok(_), Err(cleanup)) => Err(cleanup), + (Err(error), Err(cleanup)) => Err(error.with_later_failure(cleanup)), + } + } + + fn remove_container(&self, name: &str) -> Result<(), BuildError> { + match self.releasing().run( + "remove build helper", + &["rm", "--force", name], + Streams::Captured, + ) { + Ok(_) => Ok(()), + Err(BuildError::Docker { diagnostic, .. }) + if diagnostic.contains("No such container") => + { + Ok(()) + } + Err(error) => Err(BuildError::UncertainTermination(error.to_string())), + } + } + /// Run one Docker command to completion within the deadline. /// /// # Errors @@ -734,9 +823,13 @@ impl<'a> Docker<'a> { // block until its deadline. let output = self.working_dir.join("docker-output"); let diagnosis = self.working_dir.join("docker-diagnosis"); - if streams == Streams::Captured { + if streams != Streams::Inherited { command - .stdout(self.create(action, &output)?) + .stdout(match streams { + Streams::Captured => Stdio::from(self.create(action, &output)?), + Streams::Discarded => Stdio::null(), + Streams::Inherited => unreachable!(), + }) .stderr(self.create(action, &diagnosis)?); } let mut progress_reader = if streams == Streams::Inherited && self.progress.is_some() { @@ -794,7 +887,9 @@ impl<'a> Docker<'a> { // Inherited output already reached the operator, and the file // still holds whatever the previous captured command wrote. let diagnostic = match streams { - Streams::Captured => std::fs::read_to_string(&diagnosis).unwrap_or_default(), + Streams::Captured | Streams::Discarded => { + std::fs::read_to_string(&diagnosis).unwrap_or_default() + } Streams::Inherited => String::new(), }; let diagnostic = diagnostic.trim(); @@ -814,7 +909,7 @@ impl<'a> Docker<'a> { diagnostic: error.to_string(), }) } - Streams::Inherited => Ok(String::new()), + Streams::Inherited | Streams::Discarded => Ok(String::new()), } } diff --git a/crates/ployz-build/src/railpack.rs b/crates/ployz-build/src/railpack.rs index ee33df2631..02638c5fee 100644 --- a/crates/ployz-build/src/railpack.rs +++ b/crates/ployz-build/src/railpack.rs @@ -53,6 +53,7 @@ impl Drop for Preparation { pub(crate) fn prepare( docker: &Docker<'_>, request: &Request<'_>, + native: &str, resources: &crate::policy::Resources, ) -> Result, BuildError> { if request.railpack.is_empty() { @@ -63,16 +64,24 @@ pub(crate) fn prepare( "Railpack does not support --check".into(), )); } - let native = docker.run( - "read the native build platform", - &["version", "--format", "{{.Server.Os}}/{{.Server.Arch}}"], - Streams::Captured, - )?; - let native = native.trim(); - if !matches!(native, "linux/amd64" | "linux/arm64") { - return Err(BuildError::Request( - "Railpack requires a native Linux AMD64 or ARM64 builder".into(), - )); + for recipe in request.railpack { + let target = request + .targets + .iter() + .find(|t| t.name == recipe.name) + .ok_or_else(|| BuildError::Request("Railpack recipe has no build target".into()))?; + for platform in target + .platforms + .iter() + .map(String::as_str) + .chain(target.platforms.is_empty().then_some(native)) + { + if !matches!(platform, "linux/amd64" | "linux/arm64") { + return Err(BuildError::Request( + "Railpack supports only linux/amd64 and linux/arm64".into(), + )); + } + } } let directory = request.working_dir.join("private/railpack"); fs::DirBuilder::new() @@ -84,11 +93,6 @@ pub(crate) fn prepare( if request.railpack.iter().any(|recipe| recipe.refresh_cache) { // ponytail: this frontend ignores no-cache/pull. Prune only the locked, // Ployz-owned builder; replace this cold rebuild when upstream supports them. - docker.run( - "start the builder for cache refresh", - &["buildx", "inspect", &builder_name(), "--bootstrap"], - Streams::Captured, - )?; docker.run( "refresh Railpack build cache", &[ @@ -103,17 +107,6 @@ pub(crate) fn prepare( )?; } for (index, recipe) in request.railpack.iter().enumerate() { - if request - .targets - .iter() - .find(|target| target.name == recipe.name) - .and_then(|target| target.platform.as_deref()) - .is_some_and(|platform| !crate::covers(native, platform)) - { - return Err(BuildError::Request(format!( - "Railpack requires the execution host's native platform {native}" - ))); - } let directory = preparation.directory.join(index.to_string()); fs::DirBuilder::new() .mode(0o700) @@ -166,7 +159,7 @@ pub(crate) fn prepare( recipe.name.replace('.', "_"), json!({ "dockerfile": plan, - "platforms": [native], + "platforms": [request.targets.iter().find(|t| t.name == recipe.name).and_then(|t| t.platforms.first()).map_or(native, String::as_str)], "args": {"BUILDKIT_SYNTAX": IMAGE, "secrets-hash": hash}, "secret": secrets, }), @@ -189,67 +182,45 @@ fn prepare_container( resources: &crate::policy::Resources, ) -> Result<(), BuildError> { let name = format!("{}-prepare", builder_name()); - // The builder lock also owns this name. Remove an abandoned preparation - // before copying new source; docker cp works with remote Docker hosts too. - let _ = docker.releasing().run( - "remove stale Railpack preparation", - &["rm", "--force", &name], - Streams::Captured, - ); - let result = (|| { - let mut arguments = vec![ - "create".into(), - "--name".into(), - name.clone(), - "--network".into(), - "host".into(), - "--entrypoint".into(), - "/bin/sh".into(), - ]; - arguments.extend(resources.preparation_arguments()); - arguments.extend([IMAGE.into(), "/prepare.sh".into()]); - docker.run( - "create Railpack preparation", - &arguments.iter().map(String::as_str).collect::>(), - Streams::Captured, - )?; - docker.run( - "copy captured Railpack source", - &["cp", &context.to_string_lossy(), &format!("{name}:/app")], - Streams::Captured, - )?; - docker.run( - "copy private Railpack variables", - &[ - "cp", - &script.to_string_lossy(), - &format!("{name}:/prepare.sh"), - ], - Streams::Captured, - )?; - docker.run( - "Railpack preparation", - &["start", "--attach", &name], - Streams::Inherited, - )?; - docker.run( - "read the Railpack plan", - &["cp", &format!("{name}:/plan.json"), &plan.to_string_lossy()], - Streams::Captured, - )?; - Ok(()) - })(); - match docker.releasing().run( - "remove Railpack preparation", - &["rm", "--force", &name], - Streams::Captured, - ) { - Ok(_) => result, - Err(BuildError::Docker { diagnostic, .. }) if diagnostic.contains("No such container") => { - result - } - Err(error) => Err(BuildError::UncertainTermination(error.to_string())), - } + let mut arguments = vec![ + "--network".into(), + "host".into(), + "--entrypoint".into(), + "/bin/sh".into(), + ]; + arguments.extend(resources.preparation_arguments()); + arguments.extend([IMAGE.into(), "/prepare.sh".into()]); + docker.with_container( + &name, + &arguments.iter().map(String::as_str).collect::>(), + |name| { + docker.run( + "copy captured Railpack source", + &["cp", &context.to_string_lossy(), &format!("{name}:/app")], + Streams::Captured, + )?; + docker.run( + "copy private Railpack variables", + &[ + "cp", + &script.to_string_lossy(), + &format!("{name}:/prepare.sh"), + ], + Streams::Captured, + )?; + docker.run( + "Railpack preparation", + &["start", "--attach", name], + Streams::Inherited, + )?; + docker.run( + "read the Railpack plan", + &["cp", &format!("{name}:/plan.json"), &plan.to_string_lossy()], + Streams::Captured, + )?; + Ok(()) + }, + ) } fn quote(value: &str) -> String { diff --git a/crates/ployz-build/src/received_recipe.rs b/crates/ployz-build/src/received_recipe.rs index 10ced738b3..d9187a885e 100644 --- a/crates/ployz-build/src/received_recipe.rs +++ b/crates/ployz-build/src/received_recipe.rs @@ -148,9 +148,7 @@ pub fn validate_capture( .iter() .find(|target| target.name == name) .expect("validated target"); - if platforms.len() != usize::from(target.platform.is_some()) - || platforms.first().map(text).transpose()? != target.platform.as_deref() - { + if platforms.iter().map(text).collect::, _>>()? != target.platforms { return Err("Build recipe platforms differ from the admitted request".into()); } } @@ -452,7 +450,7 @@ mod tests { let definition = Definition { targets: vec![crate::Target { name: "api".into(), - platform: None, + platforms: Vec::new(), }], output: crate::Output::Load, no_cache: false, @@ -497,7 +495,7 @@ mod tests { let definition = Definition { targets: vec![crate::Target { name: "api".into(), - platform: None, + platforms: Vec::new(), }], output: crate::Output::Load, no_cache: false, diff --git a/crates/ployz-build/src/tests.rs b/crates/ployz-build/src/tests.rs index e8d0b66201..100db50a0b 100644 --- a/crates/ployz-build/src/tests.rs +++ b/crates/ployz-build/src/tests.rs @@ -163,7 +163,7 @@ fn cancellation_does_not_wait_for_a_surviving_output_writer() { fn target(name: &str, platform: Option<&str>) -> Target { Target { name: name.to_owned(), - platform: platform.map(ToOwned::to_owned), + platforms: platform.map(ToOwned::to_owned).into_iter().collect(), } } @@ -268,23 +268,6 @@ fn an_observed_platform_covers_a_request_without_its_variant() { assert!(!covers("linux/arm", "linux/arm64")); } -#[test] -fn a_repository_survives_tags_digests_and_registry_ports() { - assert_eq!( - repository("docker.io/library/api:v1"), - "docker.io/library/api" - ); - assert_eq!(repository("127.0.0.1:5000/api:v1"), "127.0.0.1:5000/api"); - assert_eq!( - repository("registry.test:5000/team/api"), - "registry.test:5000/team/api" - ); - assert_eq!( - repository("api@sha256:0000000000000000000000000000000000000000000000000000000000000000"), - "api" - ); -} - #[test] fn requested_output_selects_exclusive_bake_behavior() { let environment = BTreeMap::new(); @@ -361,6 +344,69 @@ fn a_command_that_outlasts_its_budget_is_terminated() { assert!(waited.elapsed() < Duration::from_secs(5)); } +#[test] +fn executing_a_build_does_not_change_process_signal_handlers() { + const CHILD: &str = "PLOYZ_BUILD_SIGNAL_TEST"; + if let Ok(signal) = std::env::var(CHILD) { + let directory = + std::env::temp_dir().join(format!("ployz-build-signal-{}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let program = directory.join("docker"); + executable( + &program, + &format!( + r#"#!/bin/sh +case "$1 $2" in + 'context show') echo default ;; + 'info --format') echo '{{"DriverStatus":[["driver-type","io.containerd.snapshotter.v1"]],"Architecture":"amd64","OSType":"linux"}}' ;; + 'buildx ls') echo '{{"Name":"{}","Nodes":[{{"Status":"running","Platforms":["linux/amd64"]}}]}}' ;; +esac +"#, + builder_name() + ), + ); + let environment = BTreeMap::new(); + let targets = [target("api", None)]; + let request = Request { + compose_file: Path::new("compose.yaml"), + working_dir: &directory, + environment: &environment, + docker: Some(&program), + targets: &targets, + railpack: &[], + build_args: &[], + output: Output::Validate, + no_cache: false, + pull: false, + }; + execute(&request, &Cancellation::new()).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + Command::new("kill") + .args([&signal, &std::process::id().to_string()]) + .status() + .unwrap(); + std::thread::sleep(Duration::from_millis(200)); + panic!("the build swallowed {signal} after returning"); + } + use std::os::unix::process::ExitStatusExt as _; + for (signal, number) in [("-INT", 2), ("-TERM", 15)] { + let directory = + std::env::temp_dir().join(format!("ployz-signal-home-{}-{number}", std::process::id())); + std::fs::create_dir_all(&directory).unwrap(); + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::executing_a_build_does_not_change_process_signal_handlers", + ]) + .env(CHILD, signal) + .env("HOME", &directory) + .status() + .unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + assert_eq!(status.signal(), Some(number), "{signal}: {status}"); + } +} + #[test] fn docker_preflight_holds_quarantine_and_clears_confirmed_failures() { let directory = std::env::temp_dir().join(format!("ployz-preflight-{}", uuid::Uuid::new_v4())); diff --git a/crates/ployz-build/tests/host_policy.rs b/crates/ployz-build/tests/host_policy.rs index 59604c2e31..20c7179330 100644 --- a/crates/ployz-build/tests/host_policy.rs +++ b/crates/ployz-build/tests/host_policy.rs @@ -150,7 +150,7 @@ exit 0 host.configure("cpu_cores: 8\nmemory_bytes: 8589934592"); let targets = [Target { name: "app".into(), - platform: None, + platforms: Vec::new(), }]; let railpack = [Railpack { name: "app".into(), @@ -182,7 +182,7 @@ fn resource_launch_failure_reports_preparation_and_releases_confirmed_ownership( host.configure("cpu_cores: 0.5"); let targets = [Target { name: "app".into(), - platform: None, + platforms: Vec::new(), }]; let error = execute_admitted( &Request { @@ -302,7 +302,7 @@ echo "$*" >> "$root/mutations" ); let targets = [Target { name: "api".into(), - platform: None, + platforms: Vec::new(), }]; for (key, value) in [ ("DOCKER_HOST", "ssh://remote"), diff --git a/crates/ployz/Cargo.toml b/crates/ployz/Cargo.toml index 4725faee48..e2d7c582fc 100644 --- a/crates/ployz/Cargo.toml +++ b/crates/ployz/Cargo.toml @@ -33,6 +33,7 @@ thiserror = "2.0.20" tokio = { version = "1.53.1", features = ["io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = "0.1.19" tokio-util = "0.7.19" +signal-hook = "0.3.18" tonic = { version = "0.13.1", features = ["gzip", "transport"] } tower = { version = "0.5.3", features = ["util"] } ts-rs.workspace = true diff --git a/crates/ployz/src/cancellation.rs b/crates/ployz/src/cancellation.rs new file mode 100644 index 0000000000..929ba76bd6 --- /dev/null +++ b/crates/ployz/src/cancellation.rs @@ -0,0 +1,156 @@ +//! Process-lifetime signal ownership for commands spanning synchronous and async work. + +use tokio_util::sync::CancellationToken; + +use crate::failure::Failure as Error; + +/// Observe interrupts until this CLI process exits; every phase shares one token. +/// +/// # Errors +/// Fails before command work if signals or the listener thread cannot be installed. +pub(crate) fn listen() -> Result { + static SIGNALS: std::sync::OnceLock> = + std::sync::OnceLock::new(); + SIGNALS + .get_or_init(install) + .as_ref() + .cloned() + .map_err(|error| Error::usage(format!("listen for command cancellation: {error}"))) +} + +fn install() -> std::io::Result { + let mut signals = signal_hook::iterator::Signals::new([ + signal_hook::consts::SIGINT, + signal_hook::consts::SIGTERM, + ])?; + let cancellation = CancellationToken::new(); + let cancelled = cancellation.clone(); + std::thread::Builder::new() + .name("command-signals".into()) + .spawn(move || { + for _ in signals.forever() { + cancelled.cancel(); + } + })?; + Ok(cancellation) +} + +/// Cancel read-only work without abandoning mutation or cleanup futures. +/// +/// # Errors +/// Reports cancellation or the read's own failure. +pub(crate) async fn read( + cancellation: &CancellationToken, + work: impl std::future::Future>, +) -> Result { + tokio::select! { + biased; + () = cancellation.cancelled() => Err(Error::usage("command cancelled")), + result = work => result, + } +} + +/// Read a prompt response without holding command cancellation behind stdin. +/// +/// # Errors +/// Reports cancellation, input errors, or failure to start the input thread. +pub(crate) async fn read_line( + cancellation: &CancellationToken, + mut input: impl std::io::BufRead + Send + 'static, +) -> Result { + read(cancellation, async { + let (sender, receiver) = tokio::sync::oneshot::channel(); + // ponytail: a cancelled terminal read can linger until CLI exit; use + // cancellable OS I/O if interactive commands become reusable. + std::thread::Builder::new() + .name("confirmation-input".into()) + .spawn(move || { + let mut line = String::new(); + let result = input.read_line(&mut line).map(|_| line); + let _ = sender.send(result); + })?; + Ok(receiver + .await + .map_err(|error| std::io::Error::other(error.to_string()))??) + }) + .await +} + +/// Subscribe to Ctrl-C for async-only commands already running on Tokio. +pub(crate) fn on_ctrl_c() -> CancellationToken { + let cancellation = CancellationToken::new(); + let signal = cancellation.clone(); + tokio::spawn(async move { + tokio::select! { + () = signal.cancelled() => {} + result = tokio::signal::ctrl_c() => if result.is_ok() { + signal.cancel(); + } + } + }); + cancellation +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + + #[tokio::test] + async fn cancellation_stops_a_prompt_with_no_input() { + let (input, _writer) = std::os::unix::net::UnixStream::pair().unwrap(); + let cancellation = CancellationToken::new(); + let response = read_line(&cancellation, std::io::BufReader::new(input)); + tokio::pin!(response); + tokio::select! { + result = &mut response => panic!("input unexpectedly completed: {result:?}"), + () = tokio::time::sleep(std::time::Duration::from_millis(20)) => {} + } + cancellation.cancel(); + let result = tokio::time::timeout(std::time::Duration::from_secs(1), response) + .await + .unwrap(); + assert!(result.unwrap_err().to_string().contains("cancelled")); + } + + #[test] + fn interrupts_survive_the_handoff_to_an_async_phase() { + const CHILD: &str = "PLOYZ_COMMAND_SIGNAL_TEST"; + if let Ok(signal) = std::env::var(CHILD) { + let cancellation = listen().unwrap(); + // The synchronous phase has returned; no Tokio runtime exists yet. + Command::new("kill") + .args([&signal, &std::process::id().to_string()]) + .status() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !cancellation.is_cancelled() { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let next_phase = async { panic!("cancelled command admitted the next phase") }; + assert!(read::<()>(&cancellation, next_phase).await.is_err()); + // Cancellation is sticky across subscribers and cleanup can still finish. + assert!(listen().unwrap().is_cancelled()); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + }); + return; + } + for signal in ["-INT", "-TERM"] { + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "cancellation::tests::interrupts_survive_the_handoff_to_an_async_phase", + ]) + .env(CHILD, signal) + .status() + .unwrap(); + assert!(status.success(), "{signal}: {status}"); + } + } +} diff --git a/crates/ployz/src/compose/build.rs b/crates/ployz/src/compose/build.rs index fe983f518e..a3b2ebc0e0 100644 --- a/crates/ployz/src/compose/build.rs +++ b/crates/ployz/src/compose/build.rs @@ -155,23 +155,49 @@ pub fn capture_build( .get("context") .and_then(Value::as_str) .is_some_and(is_ssh_context); - if let Some(recipe) = capture_recipe(&name, build, options, project, &mut inputs)? { + let recipe = capture_recipe(&name, build, options, project, &mut inputs)?; + let railpack = recipe.is_some(); + if let Some(recipe) = recipe { railpack_recipes.push(recipe); } - let platform = requested_platform(&name, build)?.or_else(|| { - project + let mut platforms = requested_platforms(&name, build, railpack)?; + if railpack && platforms.len() > 1 { + for field in ["provenance", "sbom"] { + if build.get(field).is_some_and(|value| { + !matches!(value, Value::Null | Value::Bool(false)) + && value.as_str() != Some("false") + }) { + return Err(invalid_build(&format!( + "multi-platform Railpack does not support build.{field}; use false or a single platform", + ))); + } + build.remove(field); + } + } + if platforms.is_empty() + && let Some(platform) = project .environment .get("DOCKER_DEFAULT_PLATFORM") - .filter(|platform| !platform.is_empty()) - .cloned() - }); - if let Some(platform) = &platform { + .filter(|p| !p.is_empty()) + { + platforms.push(platform.clone()); + } + if railpack + && platforms + .iter() + .any(|p| !matches!(p.as_str(), "linux/amd64" | "linux/arm64")) + { + return Err(invalid_build( + "Railpack supports only linux/amd64 and linux/arm64", + )); + } + if !platforms.is_empty() { build.insert( Value::String("platforms".into()), - Value::Sequence(vec![Value::String(platform.clone())]), + Value::Sequence(platforms.iter().cloned().map(Value::String).collect()), ); } - targets.push(ployz_build::Target { name, platform }); + targets.push(ployz_build::Target { name, platforms }); retain_service_image_tag(&service.name, image, build)?; if let Some(ssh) = build .get_mut(Value::String("ssh".into())) @@ -550,7 +576,11 @@ impl CapturedBuild { /// # Errors /// Fails if the runner cannot execute the build or the result cannot be /// bound to the content it produced. - pub fn execute(&self, docker: Option<&Path>) -> Result, ComposeError> { + pub fn execute( + &self, + docker: Option<&Path>, + cancellation: &tokio_util::sync::CancellationToken, + ) -> Result, ComposeError> { let mut environment = self.environment.clone(); environment.insert( "HOME".into(), @@ -568,18 +598,21 @@ impl CapturedBuild { .to_string_lossy() .into_owned(), ); - let images = ployz_build::execute(&ployz_build::Request { - compose_file: Path::new("compose.yaml"), - working_dir: self.inputs.root(), - environment: &environment, - docker, - targets: &self.targets, - railpack: &self.railpack, - build_args: &self.options.build_args, - output: self.options.output, - no_cache: self.options.no_cache, - pull: self.options.pull, - }) + let images = ployz_build::execute( + &ployz_build::Request { + compose_file: Path::new("compose.yaml"), + working_dir: self.inputs.root(), + environment: &environment, + docker, + targets: &self.targets, + railpack: &self.railpack, + build_args: &self.options.build_args, + output: self.options.output, + no_cache: self.options.no_cache, + pull: self.options.pull, + }, + cancellation, + ) // BuildKit diagnoses its own failure; name the Builds it was running. .map_err(|source| ComposeError::Build { services: self @@ -613,8 +646,9 @@ pub fn execute_build( options: &BuildOptions, load: &LoadOptions, project: &mut ComposeProject, + cancellation: &tokio_util::sync::CancellationToken, ) -> Result, ComposeError> { - capture_build(plan, options, project)?.execute(load.docker.as_deref()) + capture_build(plan, options, project)?.execute(load.docker.as_deref(), cancellation) } /// Refuse a setting Ployz cannot pass on, naming it rather than dropping it. @@ -685,36 +719,38 @@ fn refuse_unpassable_settings( Ok(()) } -/// The single platform this Service asks for, if it asks for one. -/// -/// A Dockerfile Build produces one image for one platform, so several -/// requested platforms are refused here, where the Service is named. -fn requested_platform( +/// Dockerfiles remain single-platform; Railpack assembles explicit variants. +fn requested_platforms( service: &str, build: &serde_norway::Mapping, -) -> Result, ComposeError> { - let Some(platforms) = build - .get(Value::String("platforms".into())) - .and_then(Value::as_sequence) - else { - return Ok(None); + railpack: bool, +) -> Result, ComposeError> { + let Some(value) = build.get("platforms") else { + return Ok(Vec::new()); }; - match platforms.as_slice() { - [] => Ok(None), - [platform] => platform - .as_str() - .map(ToOwned::to_owned) - .map(Some) - .ok_or_else(|| { - invalid_build(&format!( - "service '{service}' has an invalid build platform" - )) - }), - several => Err(invalid_build(&format!( + let platforms = value + .as_sequence() + .ok_or_else(|| invalid_build("build.platforms must be a list"))?; + if !railpack && platforms.len() > 1 { + return Err(invalid_build(&format!( "service '{service}' requests {} build platforms; a Dockerfile Build produces one platform", - several.len() - ))), + platforms.len() + ))); } + platforms + .iter() + .map(|platform| { + platform + .as_str() + .filter(|p| !p.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + invalid_build(&format!( + "service '{service}' has an invalid build platform" + )) + }) + }) + .collect() } /// Upstream translation uses explicit build tags alone; keep the Service image @@ -893,6 +929,39 @@ impl BuildSpec { mod tests { use super::*; + #[test] + fn disabled_multi_platform_attestations_pass_remote_validation() { + let root = std::env::temp_dir().join(format!( + "ployz-disabled-attestations-{}", + std::process::id() + )); + std::fs::create_dir_all(&root).unwrap(); + for settings in [ + "provenance: false", + "sbom: false", + "provenance: false, sbom: false", + ] { + let mut project = crate::compose::parse_normalized( + &format!("services:\n api:\n build: {{context: ., x-recipe: railpack, platforms: [linux/amd64, linux/arm64], {settings}}}\n"), + &root, + ).unwrap(); + let options = BuildOptions::default(); + let plan = plan_build(&project, &options).unwrap(); + let captured = capture_build(&plan, &options, &mut project).unwrap(); + let definition = ployz_build::remote::Definition { + targets: captured.targets, + output: options.output, + no_cache: options.no_cache, + pull: options.pull, + }; + let recipes = + ployz_build::remote::validate_capture(captured.inputs.root(), &definition) + .unwrap_or_else(|error| panic!("{settings}: {error}")); + assert_eq!(recipes.len(), 1); + } + std::fs::remove_dir_all(root).unwrap(); + } + #[test] fn git_context_suffix_rules_follow_buildkit_transports() { let commit = "0123456789abcdef0123456789abcdef01234567"; diff --git a/crates/ployz/src/compose/remote_build.rs b/crates/ployz/src/compose/remote_build.rs index 29251ae250..9d8a8e3071 100644 --- a/crates/ployz/src/compose/remote_build.rs +++ b/crates/ployz/src/compose/remote_build.rs @@ -216,14 +216,18 @@ fn validate_outcome( && output == ployz_build::Output::Load => { if images.iter().any(|image| { - image + !image .reference - .parse::() - .ok() - .and_then(|reference| reference.digest().map(str::to_owned)) - .is_none() + .strip_prefix("sha256:") + .is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) || image.tags.is_empty() - || !linux_platform(&image.platform) + || image.platforms.is_empty() + || image + .platforms + .iter() + .any(|platform| !linux_platform(platform)) }) { return unknown( Stage::Output, @@ -290,9 +294,10 @@ mod tests { use ployz_build::{BuiltImage, Output, TargetEvidence, WorkEvidence}; let machine_id = MachineId::random(); let image = BuiltImage { - reference: format!("example.test/api@sha256:{}", "1".repeat(64)), + reference: format!("sha256:{}", "1".repeat(64)), tags: vec!["example.test/api:built".into()], - platform: "linux/arm/v7".into(), + platforms: vec!["linux/arm/v7".into()], + location: "unix:///var/run/docker.sock".into(), }; let work = WorkEvidence(std::collections::BTreeMap::from([( "api".into(), @@ -311,7 +316,7 @@ mod tests { Outcome::Images { machine_id, images: vec![BuiltImage { - platform: "linux//v7".into(), + platforms: vec!["linux//v7".into()], ..image.clone() }], }, diff --git a/crates/ployz/src/deploy/apply.rs b/crates/ployz/src/deploy/apply.rs index 8ddce24740..483e17f6c0 100644 --- a/crates/ployz/src/deploy/apply.rs +++ b/crates/ployz/src/deploy/apply.rs @@ -82,12 +82,15 @@ async fn apply_spec( ); return Ok(()); } + let cancellation = crate::cancellation::on_ctrl_c(); + let _stop_listener = cancellation.clone().drop_guard(); finish( stream_confirm( client, &preview, format!("Running service {}", requested.name), Ink::detect(io::stdout()), + &cancellation, ) .await, &format!("Deployed to {context}"), @@ -155,10 +158,12 @@ pub(crate) async fn deploy_project( client: &mut Client, candidate: &CapturedCompose, builds: &[BuiltService], + cancellation: &CancellationToken, gate: ConfirmGate<'_>, ) -> Result<(), Failure> { - let machines = client.machines().await?; - let outcome = push_project_images(client, builds, &machines).await?; + let machines = + crate::cancellation::read(cancellation, async { Ok(client.machines().await?) }).await?; + let outcome = push_project_images(client, builds, &machines, cancellation).await?; print_pushed_images(&outcome); if !outcome.failures.is_empty() { return Err(Failure::usage(format!( @@ -166,10 +171,11 @@ pub(crate) async fn deploy_project( outcome.failures.join("; ") ))); } - let preview = plan_project(client, candidate, machines).await?; + let preview = + crate::cancellation::read(cancellation, plan_project(client, candidate, machines)).await?; println!("Captured candidate {}", candidate.id()); print_warnings(&preview); - confirm_and_execute(client, &preview, gate).await + confirm_and_execute(client, &preview, gate, cancellation).await } pub(crate) async fn deploy_scale( @@ -191,6 +197,8 @@ pub(crate) async fn deploy_scale( name: project_name, source: gate.project.source, }; + let cancellation = crate::cancellation::on_ctrl_c(); + let _stop_listener = cancellation.clone().drop_guard(); confirm_and_execute( client, &preview, @@ -199,6 +207,7 @@ pub(crate) async fn deploy_scale( context: gate.context, project: &project, }, + &cancellation, ) .await } @@ -207,6 +216,7 @@ async fn confirm_and_execute( client: &Client, preview: &DeployPlan, gate: ConfirmGate<'_>, + cancellation: &CancellationToken, ) -> Result<(), Failure> { let source = Some(gate.project.source.to_string()); print!( @@ -216,7 +226,13 @@ async fn confirm_and_execute( if preview.noop() { return Ok(()); } - if !gate.auto_confirm && !confirm(&render::confirm_prompt(gate.context))? { + if !gate.auto_confirm + && !crate::cancellation::read( + cancellation, + confirm(&render::confirm_prompt(gate.context), cancellation), + ) + .await? + { println!("No changes were made."); return Ok(()); } @@ -226,6 +242,7 @@ async fn confirm_and_execute( preview, format!("Deploying to {}", gate.context), Ink::detect(io::stdout()), + cancellation, ) .await, &format!("Deployed to {}", gate.context), @@ -250,12 +267,15 @@ pub(crate) async fn remove_project( if preview.noop() { return Ok(()); } + let cancellation = crate::cancellation::on_ctrl_c(); + let _stop_listener = cancellation.clone().drop_guard(); finish( stream_confirm( client, &preview, format!("Removing Project {name} from {context}"), Ink::detect(io::stdout()), + &cancellation, ) .await, &format!("Removed Project {name} from {context}"), @@ -269,15 +289,10 @@ async fn stream_confirm( preview: &DeployPlan, title: String, ink: Ink, + cancel: &CancellationToken, ) -> (DeployOutcome, ProgressPrinter) { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - let cancel = CancellationToken::new(); - let abort = cancel.clone(); - let ctrl_c = tokio::spawn(async move { - let _ = tokio::signal::ctrl_c().await; - abort.cancel(); - }); - let execute = client.confirm(preview, &cancel, Some(tx)); + let execute = client.confirm(preview, cancel, Some(tx)); tokio::pin!(execute); let mut printer = ProgressPrinter::new(title, ink); let outcome = loop { @@ -295,7 +310,6 @@ async fn stream_confirm( } } }; - ctrl_c.abort(); (outcome, printer) } @@ -398,7 +412,7 @@ fn print_warnings(preview: &DeployPreview) { } } -fn confirm(prompt: &str) -> Result { +async fn confirm(prompt: &str, cancellation: &CancellationToken) -> Result { if !io::stdin().is_terminal() || !io::stdout().is_terminal() { return Err(Failure::usage( "confirmation requires a terminal; pass --yes to continue", @@ -406,8 +420,8 @@ fn confirm(prompt: &str) -> Result { } print!("{prompt}"); io::stdout().flush()?; - let mut input = String::new(); - io::stdin().read_line(&mut input)?; + let input = + crate::cancellation::read_line(cancellation, io::BufReader::new(io::stdin())).await?; Ok(matches!(input.trim(), "y" | "Y" | "yes" | "YES")) } diff --git a/crates/ployz/src/deploy/pipeline.rs b/crates/ployz/src/deploy/pipeline.rs index 2ad3613c5b..fbc6a86629 100644 --- a/crates/ployz/src/deploy/pipeline.rs +++ b/crates/ployz/src/deploy/pipeline.rs @@ -281,11 +281,12 @@ pub(super) async fn push_project_images( client: &mut Client, builds: &[BuiltService], machines: &[MachineObservation], + cancellation: &CancellationToken, ) -> Result { let mut pushed = Vec::new(); let mut failures = Vec::new(); for service in builds { - match push_image(client, service, machines).await { + match push_image(client, service, machines, cancellation).await { Ok((images, service_failures)) => { pushed.extend(images); failures.extend(service_failures); @@ -528,6 +529,7 @@ async fn push_image( client: &mut Client, service: &BuiltService, machines: &[MachineObservation], + cancellation: &CancellationToken, ) -> Result<(Vec, Vec), PushError> { let targets = service .machines @@ -535,9 +537,15 @@ async fn push_image( .map(ToString::to_string) .collect::>(); // Deliver the content this command built, not whatever the tag now holds. - let result = - crate::image::push_using_machines(client, service.content(), None, &targets, machines) - .await?; + let result = crate::image::push_using_machines( + client, + service.content(), + None, + &targets, + machines, + cancellation, + ) + .await?; let pushed = result .successes .iter() diff --git a/crates/ployz/src/handlers/build.rs b/crates/ployz/src/handlers/build.rs index c20641afeb..11cf6085df 100644 --- a/crates/ployz/src/handlers/build.rs +++ b/crates/ployz/src/handlers/build.rs @@ -113,7 +113,8 @@ pub(super) fn run(matches: &ArgMatches) -> Result<(), Error> { report_remote(outcome) }); } - let built = execute_build(&plan, &options, &load, &mut project)?; + let cancellation = crate::cancellation::listen()?; + let built = execute_build(&plan, &options, &load, &mut project, &cancellation)?; match options.output { Output::Validate => { println!("Validated the selected builds. No image was produced."); @@ -128,7 +129,7 @@ pub(super) fn run(matches: &ArgMatches) -> Result<(), Error> { println!( "Built {} ({}) as {} in local Docker", service.built.tags.join(", "), - service.built.platform, + service.built.platforms.join(", "), service.built.reference, ); } @@ -145,11 +146,20 @@ pub(super) fn run(matches: &ArgMatches) -> Result<(), Error> { ); let runtime = runtime()?; let failures = runtime.block_on(async { - let mut client = connect_client(matches, context).await?; + let mut client = + crate::cancellation::read(&cancellation, connect_client(matches, context)).await?; let mut failures = Vec::new(); for service in &built { let targets = push_targets(&explicit, &service.machines); - match crate::image::push(&mut client, service.content(), None, &targets).await { + match crate::image::push( + &mut client, + service.content(), + None, + &targets, + &cancellation, + ) + .await + { Ok(result) => failures.extend(report_push(&service.image, result)), Err(error) => failures.push(push_failure(&service.image, error)?), } @@ -222,7 +232,7 @@ fn report_remote(outcome: ployz_build::remote::Outcome) -> Result<(), Error> { println!( "Built {} ({}) as {} on Machine {machine_id}", image.tags.join(", "), - image.platform, + image.platforms.join(", "), image.reference ); } diff --git a/crates/ployz/src/handlers/deploy.rs b/crates/ployz/src/handlers/deploy.rs index 2e3f3644da..5aeec25530 100644 --- a/crates/ployz/src/handlers/deploy.rs +++ b/crates/ployz/src/handlers/deploy.rs @@ -56,13 +56,18 @@ pub(super) fn deploy(root: &ArgMatches) -> Result<(), Error> { let skip_health_monitor = matches.get_flag("skip-health"); let mut options = plan_options(force_recreate, skip_health_monitor); options.selected = selected_attempts(&project, &string_values(matches, "service"))?; - let (candidate, builds) = prepare_deploy(matches, &load, project, &resolved, options)?; + let cancellation = crate::cancellation::listen()?; + let (candidate, builds) = + prepare_deploy(matches, &load, project, &resolved, options, &cancellation)?; runtime()?.block_on(async { - let mut client = connect_client(root, context.as_deref()).await?; + let mut client = + crate::cancellation::read(&cancellation, connect_client(root, context.as_deref())) + .await?; deploy_project( &mut client, &candidate, &builds, + &cancellation, crate::deploy::ConfirmGate { auto_confirm: yes, context: context.as_deref().unwrap_or("default"), @@ -257,6 +262,7 @@ fn prepare_deploy( mut project: ComposeProject, resolved: &ResolvedProject, options: ployz_core::PlanOptions, + cancellation: &tokio_util::sync::CancellationToken, ) -> Result<(CapturedCompose, Vec), Error> { let selected = string_values(matches, "service"); for warning in &project.warnings { @@ -294,7 +300,7 @@ fn prepare_deploy( // Every required Build finishes before any application change begins. let built = match captured_build { Some(build) => build - .execute(load.docker.as_deref()) + .execute(load.docker.as_deref(), cancellation) .map_err(crate::deploy::DeployError::from)?, None => Vec::new(), }; diff --git a/crates/ployz/src/handlers/deploy_tests.rs b/crates/ployz/src/handlers/deploy_tests.rs index fb37d759e8..e8e5c4c649 100644 --- a/crates/ployz/src/handlers/deploy_tests.rs +++ b/crates/ployz/src/handlers/deploy_tests.rs @@ -342,7 +342,15 @@ fn deploy_preparation_captures_resolved_input_and_selection_before_planning() { ..Default::default() }; std::fs::write(&source, yaml.replace("captured", "later-edit")).unwrap(); - let (candidate, builds) = prepare_deploy(matches, &load, project, &resolved, options).unwrap(); + let (candidate, builds) = prepare_deploy( + matches, + &load, + project, + &resolved, + options, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(); assert!(builds.is_empty()); assert_eq!( std::fs::read_to_string(directory.join("calls")).unwrap(), @@ -415,6 +423,7 @@ fn a_failed_build_leaves_the_deployment_unattempted() { project, &resolved, ployz_core::PlanOptions::default(), + &tokio_util::sync::CancellationToken::new(), ) { Ok(_) => panic!("a failed build was admitted for deployment"), Err(error) => error.to_string(), diff --git a/crates/ployz/src/handlers/image.rs b/crates/ployz/src/handlers/image.rs index 996b348e81..787d3c7e7c 100644 --- a/crates/ployz/src/handlers/image.rs +++ b/crates/ployz/src/handlers/image.rs @@ -121,10 +121,14 @@ pub(super) fn push(matches: &ArgMatches) -> Result<(), Error> { let image = leaf .get_one::("image") .ok_or_else(|| Error::usage("image is required"))?; + let cancellation = crate::cancellation::listen()?; let result = runtime()?.block_on(async { - let mut client = connect_client( - matches, - leaf.get_one::("context").map(String::as_str), + let mut client = crate::cancellation::read( + &cancellation, + connect_client( + matches, + leaf.get_one::("context").map(String::as_str), + ), ) .await?; Ok::<_, Error>( @@ -133,6 +137,7 @@ pub(super) fn push(matches: &ArgMatches) -> Result<(), Error> { crate::image::ImageContent::tagged(image), leaf.get_one::("platform").map(String::as_str), &string_values(leaf, "machine"), + &cancellation, ) .await?, ) diff --git a/crates/ployz/src/handlers/mod.rs b/crates/ployz/src/handlers/mod.rs index cd2e5acb71..1528fda778 100644 --- a/crates/ployz/src/handlers/mod.rs +++ b/crates/ployz/src/handlers/mod.rs @@ -1,8 +1,8 @@ use std::{future::Future, path::Path, pin::Pin}; +use crate::cancellation::on_ctrl_c as cancellation_on_ctrl_c; use clap::{ArgMatches, Command}; use clap_complete::{Shell, generate}; -use tokio_util::sync::CancellationToken; use crate::failure::Failure; @@ -106,20 +106,6 @@ fn required(matches: &ArgMatches, name: &str) -> Result { .ok_or_else(|| Error::usage(format!("{name} is required"))) } -fn cancellation_on_ctrl_c() -> CancellationToken { - let cancellation = CancellationToken::new(); - let signal = cancellation.clone(); - tokio::spawn(async move { - tokio::select! { - () = signal.cancelled() => {} - result = tokio::signal::ctrl_c() => if result.is_ok() { - signal.cancel(); - } - } - }); - cancellation -} - fn runtime() -> Result { Ok(tokio::runtime::Builder::new_current_thread() .enable_all() diff --git a/crates/ployz/src/image.rs b/crates/ployz/src/image.rs index c234f9ff4d..db0bee22b7 100644 --- a/crates/ployz/src/image.rs +++ b/crates/ployz/src/image.rs @@ -1,7 +1,6 @@ use std::{ ffi::OsStr, future::Future, - pin::Pin, process::{Output, Stdio}, }; @@ -53,8 +52,6 @@ pub enum PushError { UnsupportedPlatform(String), #[error("image push cancelled")] Cancelled, - #[error("listen for image-push cancellation: {0}")] - Cancellation(#[source] std::io::Error), #[error("image '{0}' not found locally")] ImageNotFound(String), #[error("Machine target selection failed: {0}")] @@ -100,26 +97,25 @@ pub enum PushError { impl PushError { pub(crate) fn is_cancellation(&self) -> bool { - matches!(self, Self::Cancelled | Self::Cancellation(_)) + matches!(self, Self::Cancelled) || matches!(self, Self::CleanupAfter { primary, .. } if primary.is_cancellation()) } } -struct Cancellation { - signal: Pin>>>, +struct Cancellation<'token> { + token: &'token tokio_util::sync::CancellationToken, } -impl Cancellation { - fn new() -> Self { - Self { - signal: Box::pin(tokio::signal::ctrl_c()), - } +impl<'token> Cancellation<'token> { + fn new(token: &'token tokio_util::sync::CancellationToken) -> Self { + Self { token } } async fn race(&mut self, future: impl Future) -> Result { tokio::select! { + biased; + () = self.token.cancelled() => Err(PushError::Cancelled), output = future => Ok(output), - result = self.signal.as_mut() => Err(cancellation_error(result)), } } } @@ -156,11 +152,20 @@ pub async fn push( content: ImageContent<'_>, platform: Option<&str>, selectors: &[String], + cancellation: &tokio_util::sync::CancellationToken, ) -> Result, PushError> { - let machines = client - .call::(ListMachinesRequest {}, None) - .await?; - push_using_machines(client, content, platform, selectors, &machines.machines).await + let machines = Cancellation::new(cancellation) + .race(client.call::(ListMachinesRequest {}, None)) + .await??; + push_using_machines( + client, + content, + platform, + selectors, + &machines.machines, + cancellation, + ) + .await } pub(crate) async fn push_using_machines( @@ -169,8 +174,9 @@ pub(crate) async fn push_using_machines( platform: Option<&str>, selectors: &[String], machines: &[ployz_core::MachineObservation], + cancellation: &tokio_util::sync::CancellationToken, ) -> Result, PushError> { - let mut cancellation = Cancellation::new(); + let mut cancellation = Cancellation::new(cancellation); // TODO: without an explicit platform, Docker chooses what to push; target platforms are not inferred. let platform = platform.map(validated_platform).transpose()?; let image = content.published; @@ -295,7 +301,7 @@ async fn push_to_machine( platform: Option<&str>, machine: &Machine, mode: ProxyMode, - cancellation: &mut Cancellation, + cancellation: &mut Cancellation<'_>, ) -> Result { // EnsureImageIngest is idempotent; a dropped RPC must not fail the Machine. let opened = cancellation @@ -322,7 +328,7 @@ async fn pull_on_machine( image: &str, machine: &Machine, source: ImageIngestDestination, - cancellation: &mut Cancellation, + cancellation: &mut Cancellation<'_>, ) -> Result<(), PushError> { cancellation .race(client.call::( @@ -451,7 +457,7 @@ impl PushSession { mode: ProxyMode, content: ImageContent<'_>, platform: Option<&str>, - cancellation: &mut Cancellation, + cancellation: &mut Cancellation<'_>, ) -> Result<(), PushError> { let mut session = Self { proxy: ImageProxy::open(mode, cancellation).await?, @@ -575,13 +581,6 @@ impl PushSession { } } -fn cancellation_error(result: std::io::Result<()>) -> PushError { - match result { - Ok(()) => PushError::Cancelled, - Err(error) => PushError::Cancellation(error), - } -} - fn validate_push_reference(image: &str) -> Result<(), PushError> { let reference = image .parse::() diff --git a/crates/ployz/src/image/proxy.rs b/crates/ployz/src/image/proxy.rs index d703ece64f..30df6b6586 100644 --- a/crates/ployz/src/image/proxy.rs +++ b/crates/ployz/src/image/proxy.rs @@ -74,7 +74,7 @@ pub(super) struct ImageProxy { impl ImageProxy { pub(super) async fn open( mode: ProxyMode, - cancellation: &mut Cancellation, + cancellation: &mut Cancellation<'_>, ) -> Result { let mut proxy = Self { listener: Listener::bind(mode).await?, @@ -482,6 +482,25 @@ mod tests { } } + #[tokio::test] + async fn an_interrupt_from_the_build_prevents_image_push_work() { + let mut client = client(FailingDial { + attempts: Arc::new(AtomicUsize::new(0)), + error: || ConnectError::from(std::io::Error::other("must not dial")), + }); + let cancellation = tokio_util::sync::CancellationToken::new(); + cancellation.cancel(); + let result = super::super::push( + &mut client, + super::super::ImageContent::tagged("api:v1"), + None, + &[], + &cancellation, + ) + .await; + assert!(matches!(result, Err(PushError::Cancelled)), "{result:?}"); + } + #[tokio::test] async fn image_proxy_holds_docker_connection_across_a_dropped_machine_dial() { let echo = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -493,7 +512,8 @@ mod tests { stream.write_all(&buffer).await.unwrap(); }); - let mut cancellation = super::super::Cancellation::new(); + let token = tokio_util::sync::CancellationToken::new(); + let mut cancellation = super::super::Cancellation::new(&token); let mut proxy = ImageProxy::open(ProxyMode::Native, &mut cancellation) .await .unwrap(); diff --git a/crates/ployz/src/lib.rs b/crates/ployz/src/lib.rs index 38a9166450..6189d30977 100644 --- a/crates/ployz/src/lib.rs +++ b/crates/ployz/src/lib.rs @@ -1,3 +1,4 @@ +mod cancellation; pub mod changes; pub mod cli; mod cloud_enroll; diff --git a/crates/ployz/tests/build.rs b/crates/ployz/tests/build.rs index bf3f091df3..7607f7932a 100644 --- a/crates/ployz/tests/build.rs +++ b/crates/ployz/tests/build.rs @@ -196,7 +196,9 @@ fn captured_build_preserves_sources_configuration_and_builder_flags() { fs::remove_file(root.join("Dockerfile")).unwrap(); fs::remove_file(root.join("Dockerfile.dockerignore")).unwrap(); project.builds.clear(); - let outcome = build.execute(Some(&docker)).unwrap(); + let outcome = build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let calls = fs::read_to_string(calls).unwrap(); let bake = calls .lines() @@ -230,12 +232,9 @@ fn captured_build_preserves_sources_configuration_and_builder_flags() { ); let service = one_built(outcome); assert_eq!(service.image, "example.test/api:version2"); - assert_eq!( - service.built.reference, - format!("example.test/api@{FIRST_CONTENT}") - ); + assert_eq!(service.built.reference, FIRST_CONTENT); assert_eq!(service.built.tags, ["example.test/api:version2"]); - assert_eq!(service.built.platform, "linux/amd64"); + assert_eq!(service.built.platforms, ["linux/amd64"]); let override_yaml = fs::read_to_string(captured).unwrap(); assert!(override_yaml.contains("api")); assert!(override_yaml.contains("example.test/api:version2")); @@ -360,31 +359,26 @@ fn built_images_bind_to_exact_content_after_tag_reuse() { let first = capture_build(&plan, &options, &mut project) .unwrap() - .execute(Some(&docker)) + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) .unwrap(); // A later Build moves the same requested tag onto different content. fs::write(root.join("digest"), SECOND_CONTENT).unwrap(); let second = capture_build(&plan, &options, &mut project) .unwrap() - .execute(Some(&docker)) + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) .unwrap(); let (first, second) = (one_built(first), one_built(second)); assert_eq!(first.built.tags, second.built.tags); - assert_eq!( - first.built.reference, - format!("example.test/api@{FIRST_CONTENT}") - ); - assert_eq!( - second.built.reference, - format!("example.test/api@{SECOND_CONTENT}") - ); + assert_eq!(first.built.reference, FIRST_CONTENT); + assert_eq!(second.built.reference, SECOND_CONTENT); // Each attempt verified its own content instead of the shared tag. let calls = fs::read_to_string(root.join("calls")).unwrap(); for content in [FIRST_CONTENT, SECOND_CONTENT] { assert!( - calls.lines().any(|call| call - == format!("image inspect example.test/api@{content} --format {{{{json .}}}}")), + calls + .lines() + .any(|call| call == format!("image inspect {content} --format {{{{json .}}}}")), "{calls}" ); } @@ -412,7 +406,7 @@ fn an_image_the_store_does_not_hold_is_refused_as_a_result() { let plan = plan_build(&project, &options).unwrap(); let build = capture_build(&plan, &options, &mut project).unwrap(); - let error = match build.execute(Some(&docker)) { + let error = match build.execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) { Ok(outcome) => panic!("substituted content was reported as built: {outcome:?}"), Err(error) => error.to_string(), }; @@ -427,7 +421,7 @@ fn an_image_the_store_does_not_hold_is_refused_as_a_result() { #[test] fn several_requested_build_platforms_are_refused_with_the_service_named() { let mut project = parse_normalized( - "name: demo\nservices: {api: {build: {context: ., platforms: [linux/amd64, linux/arm64]}}}\n", + "name: demo\nservices: {api: {build: {context: ., dockerfile_inline: 'FROM scratch', platforms: [linux/amd64, linux/arm64]}}}\n", ".", ) .unwrap(); @@ -466,7 +460,7 @@ fn content_holding_several_platforms_is_refused_however_it_was_requested() { let plan = plan_build(&project, &options).unwrap(); let build = capture_build(&plan, &options, &mut project).unwrap(); - let error = match build.execute(Some(&docker)) { + let error = match build.execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) { Ok(built) => panic!("a multi-platform image was reported as built: {built:?}"), Err(error) => error.to_string(), }; @@ -638,7 +632,9 @@ secrets: }; let plan = plan_build(&project, &options).unwrap(); let build = capture_build(&plan, &options, &mut project).unwrap(); - build.execute(Some(&docker)).unwrap(); + build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let config: serde_norway::Value = serde_norway::from_str(&fs::read_to_string(root.join("override.yaml")).unwrap()).unwrap(); assert_eq!( diff --git a/crates/ployz/tests/build/capture.rs b/crates/ployz/tests/build/capture.rs index 326b2b14bb..ecf56c19cf 100644 --- a/crates/ployz/tests/build/capture.rs +++ b/crates/ployz/tests/build/capture.rs @@ -44,7 +44,9 @@ fn capture_rejects_links_outside_the_captured_source() { let plan = plan_build(&project, &options).unwrap(); let build = capture_build(&plan, &options, &mut project).unwrap(); fs::remove_dir_all(root.join("src")).unwrap(); - build.execute(Some(&docker)).unwrap(); + build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let config: serde_norway::Value = serde_norway::from_str(&fs::read_to_string(root.join("override.yaml")).unwrap()).unwrap(); let context = config @@ -126,7 +128,9 @@ fn resolved_file_credentials_stay_private_across_captures() { let docker = root.join("docker"); write_docker(&docker, &root); let build = capture_build(&plan, &options, &mut project).unwrap(); - build.execute(Some(&docker)).unwrap(); + build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let config: serde_norway::Value = serde_norway::from_str(&fs::read_to_string(root.join("override.yaml")).unwrap()).unwrap(); let staged = root.join("relocated"); @@ -147,7 +151,9 @@ fn resolved_file_credentials_stay_private_across_captures() { ); fs::write(root.join("src/token"), "later-provider-value").unwrap(); let second = capture_build(&plan, &options, &mut project).unwrap(); - second.execute(Some(&docker)).unwrap(); + second + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); assert_eq!( project.services["api"].container.environment["TOKEN"], "private-file-token" @@ -188,7 +194,9 @@ fn builds_use_captured_explicit_registry_credentials_and_proxies() { fs::write(root.join("auth/config.json"), "changed after capture").unwrap(); let docker = root.join("docker"); write_docker(&docker, &root); - build.execute(Some(&docker)).unwrap(); + build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let captured: serde_json::Value = serde_json::from_str(&fs::read_to_string(root.join("docker-config.json")).unwrap()) .unwrap(); @@ -251,7 +259,9 @@ secrets: fs::remove_dir_all(root.join("src")).unwrap(); let docker = root.join("docker"); write_docker(&docker, &root); - build.execute(Some(&docker)).unwrap(); + build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let config: serde_norway::Value = serde_norway::from_str(&fs::read_to_string(root.join("override.yaml")).unwrap()).unwrap(); let context = |service: &str| { @@ -345,7 +355,9 @@ fn authored_configuration_follows_dockerignore() { let build = capture_build(&plan, &options, &mut project).unwrap(); let docker = root.join("docker"); write_docker(&docker, &root); - build.execute(Some(&docker)).unwrap(); + build + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(); let config: serde_norway::Value = serde_norway::from_str(&fs::read_to_string(root.join("override.yaml")).unwrap()).unwrap(); let context = root.join("relocated").join( @@ -444,7 +456,7 @@ fn default_platform_is_captured_and_verified_unless_compose_overrides_it() { let build = capture_build(&plan, &options, &mut project).unwrap(); fs::write(root.join(".env"), "DOCKER_DEFAULT_PLATFORM=linux/amd64\n").unwrap(); // The recording executor reports AMD64: an ARM64 request must reject it. - let result = build.execute(Some(&docker)); + let result = build.execute(Some(&docker), &tokio_util::sync::CancellationToken::new()); let expected = if declared.is_empty() { assert!( result @@ -454,7 +466,7 @@ fn default_platform_is_captured_and_verified_unless_compose_overrides_it() { ); "linux/arm64" } else { - assert_eq!(one_built(result.unwrap()).built.platform, "linux/amd64"); + assert_eq!(one_built(result.unwrap()).built.platforms, ["linux/amd64"]); "linux/amd64" }; let config: serde_norway::Value = @@ -531,7 +543,7 @@ fn ssh_docker_hosts_are_refused_but_git_contexts_keep_the_captured_agent_socket( let plan = plan_build(&project, &options).unwrap(); let build = capture_build(&plan, &options, &mut project).unwrap(); fs::write(root.join(".env"), "SSH_AUTH_SOCK=/tmp/changed-agent.sock\n").unwrap(); - let result = build.execute(Some(&docker)); + let result = build.execute(Some(&docker), &tokio_util::sync::CancellationToken::new()); if host.starts_with("ssh://") { assert!(result.unwrap_err().to_string().contains("local Docker")); assert!(!root.join("docker-environment").exists()); diff --git a/crates/ployz/tests/build/railpack.rs b/crates/ployz/tests/build/railpack.rs index 79b632359b..08c2f65528 100644 --- a/crates/ployz/tests/build/railpack.rs +++ b/crates/ployz/tests/build/railpack.rs @@ -112,11 +112,25 @@ secrets: let mut hashes = Vec::new(); for (value, capture) in captures { fs::write(root.join("preparation-fails"), "").unwrap(); - let failed = capture.execute(Some(&docker)).unwrap_err().to_string(); + let failed = capture + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap_err() + .to_string(); assert!(failed.contains("Railpack preparation"), "{failed}"); fs::remove_file(root.join("preparation-fails")).unwrap(); - let result = one_built(capture.execute(Some(&docker)).unwrap()); - assert_eq!(one_built(capture.execute(Some(&docker)).unwrap()), result); + let result = one_built( + capture + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(), + ); + assert_eq!( + one_built( + capture + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap() + ), + result + ); assert!(!format!("{result:?}").contains("private-token")); let received = root.join("received"); assert_eq!( @@ -205,3 +219,384 @@ fn railpack_refuses_frontend_options_it_cannot_honor() { } fs::remove_dir_all(root).unwrap(); } + +#[test] +fn railpack_accepts_explicit_linux_architectures_and_rejects_other_platforms() { + let root = + std::env::temp_dir().join(format!("ployz-railpack-platforms-{}", std::process::id())); + fs::create_dir_all(&root).unwrap(); + for (platforms, accepted) in [ + ("linux/amd64, linux/arm64", true), + ("linux/arm64", true), + ("linux/amd64, windows/amd64", false), + ("linux/arm/v7", false), + ] { + let mut project = parse_normalized(&format!("services:\n api:\n build: {{context: ., x-recipe: railpack, platforms: [{platforms}]}}\n"), &root).unwrap(); + let options = BuildOptions::default(); + let plan = plan_build(&project, &options).unwrap(); + let result = capture_build(&plan, &options, &mut project); + assert_eq!(result.is_ok(), accepted, "{platforms}: {:?}", result.err()); + } + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn multi_platform_railpack_refuses_requested_attestations() { + let root = std::env::temp_dir().join(format!( + "ployz-railpack-attestations-{}", + std::process::id() + )); + fs::create_dir_all(&root).unwrap(); + for (field, platforms, value, accepted) in [ + ("provenance", "linux/amd64, linux/arm64", "true", false), + ("provenance", "linux/amd64, linux/arm64", "mode=max", false), + ("provenance", "linux/amd64, linux/arm64", "false", true), + ("provenance", "linux/amd64", "true", true), + ("provenance", "linux/arm64", "mode=max", true), + ("sbom", "linux/amd64, linux/arm64", "true", false), + ( + "sbom", + "linux/amd64, linux/arm64", + "generator=docker/scout-sbom-indexer", + false, + ), + ("sbom", "linux/amd64, linux/arm64", "false", true), + ("sbom", "linux/amd64", "true", true), + ] { + let mut project = parse_normalized(&format!("services:\n api:\n build: {{context: ., x-recipe: railpack, platforms: [{platforms}], {field}: {value}}}\n"), &root).unwrap(); + let options = BuildOptions::default(); + let plan = plan_build(&project, &options).unwrap(); + let result = capture_build(&plan, &options, &mut project); + if accepted { + assert!( + result.is_ok(), + "{field}, {platforms}, {value}: {:?}", + result.err() + ); + } else { + let error = result + .err() + .expect("requested attestation must not be discarded") + .to_string(); + assert!( + error.contains("multi-platform Railpack") + && error.contains(&format!("build.{field}")), + "{error}" + ); + } + } + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn multi_platform_build_requires_complete_content_and_cleans_up_failed_attempts() { + let root = std::env::temp_dir().join(format!("ployz-railpack-outcomes-{}", std::process::id())); + fs::create_dir_all(&root).unwrap(); + let docker = write_multi_docker(&root); + let mut project = parse_normalized("services:\n api:\n image: example.test/api:check\n build: {context: ., x-recipe: railpack, platforms: [linux/amd64, linux/arm64]}\n", &root).unwrap(); + let options = BuildOptions::default(); + let plan = plan_build(&project, &options).unwrap(); + let captured = capture_build(&plan, &options, &mut project).unwrap(); + for (failure, diagnostic) in [ + ("create-fails", "assembly image unavailable"), + ( + "create-fails-after-creation", + "assembly create response lost", + ), + ("assembly-fails", "assemble Railpack"), + ("import-fails", "load assembled"), + ("missing-content", "verify Railpack platform content"), + ("solve-fails", "the build"), + ("worker-missing", "reported no capability"), + ] { + fs::write(root.join(failure), "").unwrap(); + let error = captured + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap_err() + .to_string(); + assert!(error.contains(diagnostic), "{error}"); + assert!(!error.contains("cleanup could not be confirmed"), "{error}"); + assert!(!root.join("assembly").exists()); + assert!(!root.join("builder").exists()); + let capture_root = fs::read_to_string(root.join("capture-root")).unwrap(); + assert!( + !Path::new(capture_root.trim()) + .join("private/railpack") + .exists() + ); + fs::remove_file(root.join(failure)).unwrap(); + } + let first = one_built( + captured + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(), + ); + assert_eq!(first.built.platforms, ["linux/amd64", "linux/arm64"]); + assert_eq!(first.built.reference, FIRST_CONTENT); + assert_eq!(first.built.tags, ["example.test/api:check"]); + fs::write(root.join("digest"), SECOND_CONTENT).unwrap(); + let second = one_built( + captured + .execute(Some(&docker), &tokio_util::sync::CancellationToken::new()) + .unwrap(), + ); + assert_ne!(first.built.reference, second.built.reference); + assert_eq!(first.built.tags, second.built.tags); + fs::remove_dir_all(root).unwrap(); +} + +fn write_multi_docker(root: &Path) -> std::path::PathBuf { + let base = root.join("base-docker"); + write_docker(&base, root); + fs::write(root.join("image"), "example.test/api:check").unwrap(); + fs::write(root.join("digest"), FIRST_CONTENT).unwrap(); + fs::write( + root.join("media"), + "application/vnd.oci.image.index.v1+json", + ) + .unwrap(); + let docker = root.join("docker"); + fs::write( + &docker, + format!( + r#"#!/bin/sh +root='{}' +case "$1" in --ready) exit 0 ;; esac +case "$*" in + 'buildx bake '*) + if [ -f "$root/cancel-solve" ]; then pwd > "$root/capture-root"; touch "$root/active"; exec sleep 60; fi + if [ -f "$root/solve-fails" ]; then pwd > "$root/capture-root"; exit 1; fi ;; + 'buildx ls '*) + if [ -f "$root/worker-missing" ]; then printf '{{}}'; exit 0; fi ;; + 'create --name '*-assemble*) + pwd > "$root/capture-root" + if [ -f "$root/cancel-create" ]; then touch "$root/active"; exec sleep 60; fi + if [ -f "$root/create-fails" ]; then printf 'assembly image unavailable' >&2; exit 1; fi + touch "$root/assembly" + if [ -f "$root/cancel-create-after-creation" ]; then touch "$root/active"; exec sleep 60; fi + if [ -f "$root/create-fails-after-creation" ]; then printf 'assembly create response lost' >&2; exit 1; fi ;; + 'rm --force '*-assemble) + if [ -f "$root/cleanup-fails" ] && [ -f "$root/active" ]; then printf 'assembly still running' >&2; exit 1; fi + if [ ! -f "$root/assembly" ]; then printf "Error response from daemon: No such container: %s" "$3" >&2; exit 1; fi + rm -f "$root/assembly" ;; + 'start '*-assemble) exit 0 ;; + 'exec '*-assemble' regctl index create '*) + if [ -f "$root/cancel-assembly" ]; then touch "$root/active"; exec sleep 60; fi + test ! -f "$root/assembly-fails"; exit $? ;; + 'exec '*-assemble' regctl image digest '*) cat "$root/digest"; exit 0 ;; + 'exec '*-assemble' regctl '*) exit 0 ;; + 'cp '*:/tmp/variant.tar) exit 0 ;; + 'image load '*) test ! -f "$root/import-fails"; exit $? ;; + 'image save '*) test ! -f "$root/missing-content"; exit $? ;; + 'image tag '*) if [ -f "$root/cleanup-fails" ]; then touch "$root/active"; fi; exit 0 ;; +esac +exec "$root/base-docker" "$@" +"#, + root.display() + ), + ) + .unwrap(); + fs::set_permissions(&docker, fs::Permissions::from_mode(0o700)).unwrap(); + docker +} + +#[test] +fn railpack_cancellation_stops_work_and_releases_private_inputs_and_admission() { + const CHILD_ROOT: &str = "PLOYZ_TEST_CANCEL_BUILD_ROOT"; + if let Some(root) = std::env::var_os(CHILD_ROOT) { + let root = std::path::PathBuf::from(root); + use ployz_build::{Admission, HostPolicy, Railpack, Request, Target}; + use std::collections::BTreeMap; + use std::os::unix::fs::DirBuilderExt as _; + fs::create_dir_all(root.join("private")).unwrap(); + fs::create_dir_all(root.join("source")).unwrap(); + let state = root.join(format!("state-{}", std::process::id())); + fs::DirBuilder::new().mode(0o700).create(&state).unwrap(); + let policy = HostPolicy { + state_directory: state, + configuration_file: root.join("build.yaml"), + docker: root.join("docker"), + ..HostPolicy::default() + }; + let targets = [Target { + name: "api".into(), + platforms: vec!["linux/amd64".into(), "linux/arm64".into()], + }]; + let recipes = [Railpack { + name: "api".into(), + context: "source".into(), + variables: BTreeMap::new(), + refresh_cache: false, + }]; + let request = Request { + compose_file: &root.join("compose.yaml"), + working_dir: &root, + environment: &BTreeMap::from([("PATH".into(), std::env::var("PATH").unwrap())]), + docker: Some(&policy.docker), + targets: &targets, + railpack: &recipes, + build_args: &[], + output: ployz_build::Output::Load, + no_cache: false, + pull: false, + }; + let admission = Admission::try_acquire_with(&policy).unwrap(); + let cancelled = admission.cancellation(); + let marker = root.join("interrupt"); + std::thread::spawn(move || { + while !marker.exists() { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + cancelled.cancel(); + }); + let error = ployz_build::execute_admitted(&request, admission, &|_| {}) + .unwrap_err() + .to_string(); + if root.join("cleanup-fails").exists() { + assert!( + error.contains("cleanup could not be confirmed") + && error.contains("assembly still running"), + "{error}" + ); + assert!(root.join("assembly").exists()); + } else { + assert!(error.contains("cancelled"), "{error}"); + assert!(!root.join("assembly").exists()); + } + assert!(!root.join("builder").exists()); + let capture_root = fs::read_to_string(root.join("capture-root")).unwrap(); + assert!( + !Path::new(capture_root.trim()) + .join("private/railpack") + .exists() + ); + return; + } + let root = std::env::temp_dir().join(format!("ployz-railpack-cancel-{}", std::process::id())); + fs::create_dir_all(&root).unwrap(); + write_multi_docker(&root); + for (phase, cleanup_fails) in [ + ("create", false), + ("create-after-creation", false), + ("solve", false), + ("assembly", false), + ("assembly", true), + ] { + if cleanup_fails { + fs::write(root.join("cleanup-fails"), "").unwrap(); + } + let marker = root.join(format!("cancel-{phase}")); + fs::write(&marker, "").unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "railpack::railpack_cancellation_stops_work_and_releases_private_inputs_and_admission", "--nocapture"]) + .env(CHILD_ROOT, &root) + .env("HOME", &root).spawn().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + while !root.join("active").exists() { + if child.try_wait().unwrap().is_some() || std::time::Instant::now() > deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("build did not reach its {phase} workload"); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + fs::write(root.join("interrupt"), "").unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success()); + break; + } + if std::time::Instant::now() > deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("cancelled build did not finish cleanup"); + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + fs::remove_file(root.join("interrupt")).unwrap(); + fs::remove_file(marker).unwrap(); + fs::remove_file(root.join("active")).unwrap(); + } + fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn multi_platform_failures_preserve_output_stage_and_completed_image_evidence() { + use ployz_build::{ + Admission, HostPolicy, Progress, Request, Stage, Target, TargetEvidence, WorkEvidence, + }; + use std::collections::BTreeMap; + let root = std::env::temp_dir().join(format!("ployz-railpack-evidence-{}", std::process::id())); + let capture = root.join("capture"); + fs::create_dir_all(capture.join("private")).unwrap(); + fs::create_dir(capture.join("source")).unwrap(); + use std::os::unix::fs::DirBuilderExt as _; + fs::DirBuilder::new() + .mode(0o700) + .create(root.join("state")) + .unwrap(); + fs::write(capture.join("compose.yaml"), "services: {}\n").unwrap(); + let docker = write_multi_docker(&root); + let targets = [Target { + name: "api".into(), + platforms: vec!["linux/amd64".into(), "linux/arm64".into()], + }]; + let recipes = [ployz_build::Railpack { + name: "api".into(), + context: "source".into(), + variables: BTreeMap::new(), + refresh_cache: false, + }]; + let environment = BTreeMap::from([("PATH".into(), std::env::var("PATH").unwrap())]); + let policy = HostPolicy { + configuration_file: root.join("build.yaml"), + state_directory: root.join("state"), + docker, + active_timeout: std::time::Duration::from_secs(15), + ..HostPolicy::default() + }; + let request = Request { + compose_file: &capture.join("compose.yaml"), + working_dir: &capture, + environment: &environment, + docker: Some(&policy.docker), + targets: &targets, + railpack: &recipes, + build_args: &[], + output: ployz_build::Output::Load, + no_cache: false, + pull: false, + }; + for (failure, stage, completed) in [ + ("assembly-fails", Stage::Output, false), + ("import-fails", Stage::Output, false), + ("cleanup-fails", Stage::Cleanup, true), + ] { + fs::write(root.join(failure), "").unwrap(); + let evidence = std::sync::Mutex::new(WorkEvidence::new(&targets)); + let error = ployz_build::execute_admitted( + &request, + Admission::try_acquire_with(&policy).unwrap(), + &|event: Progress| { + evidence.lock().unwrap().observe(&event); + }, + ) + .unwrap_err(); + assert_eq!(error.stage(), stage, "{error}"); + assert_eq!(error.is_unknown(), completed, "{error}"); + let evidence = evidence.into_inner().unwrap(); + let evidence = evidence.0.get("api").unwrap(); + if completed { + let TargetEvidence::Image(image) = evidence else { + panic!("completed image evidence was lost: {evidence:?}"); + }; + assert_eq!(image.reference, FIRST_CONTENT); + assert_eq!(image.platforms, ["linux/amd64", "linux/arm64"]); + } else { + assert_eq!(evidence, &TargetEvidence::Unknown); + } + fs::remove_file(root.join(failure)).unwrap(); + } + fs::remove_dir_all(root).unwrap(); +} diff --git a/crates/ployz/tests/build_layer3.rs b/crates/ployz/tests/build_layer3.rs index ce2dce0bc3..049a9eb912 100644 --- a/crates/ployz/tests/build_layer3.rs +++ b/crates/ployz/tests/build_layer3.rs @@ -91,7 +91,14 @@ fn compose_build_basic_pushes_only_buildable_resolved_images() { ); cleanup.images = plan.iter().map(|service| service.image.clone()).collect(); - execute_build(&plan, &options, &load, &mut project).unwrap(); + execute_build( + &plan, + &options, + &load, + &mut project, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(); for image in &cleanup.images { // Publication retains no local copy, so the registry is the only source. let _ = Command::new("docker").args(["image", "rm", image]).status(); @@ -163,11 +170,20 @@ fn local_dockerfile_build_loads_a_runnable_image_and_reuses_its_retained_cache() let options = BuildOptions::default(); let plan = plan_build(&project, &options).unwrap(); - let first = one_built(execute_build(&plan, &options, &load, &mut project).unwrap()); + let first = one_built( + execute_build( + &plan, + &options, + &load, + &mut project, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(), + ); assert_eq!(first.image, image); assert!(first.built.tags.iter().any(|tag| tag.ends_with(&image))); - assert_eq!(first.built.platform, host_platform()); - assert!(first.built.reference.contains("@sha256:")); + assert_eq!(first.built.platforms, [host_platform()]); + assert!(first.built.reference.starts_with("sha256:")); // The image is in the local store under exactly the content just built. command(["image", "inspect", &first.built.reference]); let stamp = output(["run", "--rm", &first.built.reference, "cat", "/built-at"]); @@ -184,7 +200,16 @@ fn local_dockerfile_build_loads_a_runnable_image_and_reuses_its_retained_cache() command(["volume", "inspect", &cache_volume()]); // A second Build recreates the builder and reuses that cache. - let rebuilt = one_built(execute_build(&plan, &options, &load, &mut project).unwrap()); + let rebuilt = one_built( + execute_build( + &plan, + &options, + &load, + &mut project, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(), + ); assert_eq!( output(["run", "--rm", &rebuilt.built.reference, "cat", "/built-at"]), stamp, @@ -313,7 +338,11 @@ RUN --mount=type=secret,id=token test ! -e /source/token && test ! -e /source/ig "runtime" ); fs::remove_dir_all(&root).unwrap(); - let built = one_built(captured.execute(None).unwrap()); + let built = one_built( + captured + .execute(None, &tokio_util::sync::CancellationToken::new()) + .unwrap(), + ); assert_eq!( output(["run", "--rm", &built.built.reference, "cat", "/values"]), "cli\nservice\nprivate-$VALUE\n$LATER\nprivate-$VALUE\n" @@ -423,7 +452,16 @@ async fn railpack_build_and_deploy_preserve_variables_cache_and_failure_boundari ..Default::default() }; let plan = plan_build(&project, &options).unwrap(); - let built = one_built(execute_build(&plan, &options, &load, &mut project).unwrap()); + let built = one_built( + execute_build( + &plan, + &options, + &load, + &mut project, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(), + ); let result = output([ "run", "--rm", @@ -458,7 +496,13 @@ async fn railpack_build_and_deploy_preserve_variables_cache_and_failure_boundari ); } previous = Some(content); - assert!(built.built.platform.starts_with("linux/")); + assert!( + built + .built + .platforms + .iter() + .all(|p| p.starts_with("linux/")) + ); assert!(built.built.tags.contains(&image)); assert!( output([ @@ -582,5 +626,123 @@ async fn railpack_build_and_deploy_preserve_variables_cache_and_failure_boundari fs::remove_dir_all(root).unwrap(); } +/// Informing evidence only: on AMD64 hosts ARM64 execution uses binfmt/QEMU. +/// Native ARM64 Machine and packaged release proof remain qualification work. +#[test] +#[ignore = "informing: requires Docker containerd storage and AMD64/ARM64 worker support"] +fn railpack_builds_complete_amd64_arm64_content_and_runs_without_registry_fallback() { + let root = std::env::temp_dir().join(format!("ployz-l3-805-{}", std::process::id())); + fs::create_dir_all(&root).unwrap(); + let image = format!("railpack.invalid/ployz-{}:multi", std::process::id()); + let extra = format!("railpack.invalid/ployz-{}:extra", std::process::id()); + let mut cleanup = LocalBuild { + images: vec![image.clone(), extra.clone()], + }; + fs::write(root.join("compose.yaml"), format!("services:\n app:\n image: {image}\n build:\n context: .\n platforms: [linux/amd64, linux/arm64]\n tags: [{extra}]\n")).unwrap(); + fs::write(root.join("package.json"), r#"{"name":"multi-check","version":"1.0.0","engines":{"node":"22.16.0"},"scripts":{"start":"node index.js"}}"#).unwrap(); + fs::write(root.join("index.js"), "console.log(process.arch);").unwrap(); + let load = LoadOptions { + command: "build".into(), + working_dir: Some(root.clone()), + ..Default::default() + }; + let mut project = load_project(&load).unwrap(); + let options = BuildOptions::default(); + let plan = plan_build(&project, &options).unwrap(); + let built = one_built( + execute_build( + &plan, + &options, + &load, + &mut project, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(), + ); + cleanup.images.push(built.built.reference.clone()); + assert_eq!(built.built.platforms, ["linux/amd64", "linux/arm64"]); + assert!(built.built.tags.contains(&image)); + assert!(built.built.tags.contains(&extra)); + // Moving every requested tag must not change or lose the Build's identity. + for tag in &built.built.tags { + command(["image", "tag", ployz_build::BUILDKIT_IMAGE, tag]); + } + eprintln!( + "Execution host: {}; non-native variants below are EMULATED", + host_platform() + ); + for (platform, expected) in [("linux/amd64", "x64"), ("linux/arm64", "arm64")] { + assert_eq!( + output([ + "run", + "--rm", + "--pull", + "never", + "--platform", + platform, + &built.built.reference + ]) + .lines() + .last(), + Some(expected) + ); + } + // Saving from the execution store reads every blob; check the resulting + // OCI inventory and hashes independently of the executor and regctl. + let archive = root.join("observed.tar"); + command([ + "image", + "save", + "--output", + archive.to_str().unwrap(), + &built.built.reference, + ]); + let inventory = Command::new("python3") + .args([ + "-c", + r#" +import hashlib, json, sys, tarfile +with tarfile.open(sys.argv[1]) as archive: + def read(digest): + data = archive.extractfile('blobs/' + digest.replace(':', '/')).read() + assert 'sha256:' + hashlib.sha256(data).hexdigest() == digest + return data + index = json.loads(read(sys.argv[2])) + platforms = set() + for descriptor in index['manifests']: + manifest = json.loads(read(descriptor['digest'])) + config = json.loads(read(manifest['config']['digest'])) + platforms.add(config['os'] + '/' + config['architecture']) + for layer in manifest['layers']: + read(layer['digest']) + assert platforms == {'linux/amd64', 'linux/arm64'}, platforms + print('Verified index, manifests, configurations and all layer hashes:', sorted(platforms)) +"#, + archive.to_str().unwrap(), + &built.built.reference, + ]) + .status() + .unwrap(); + assert!(inventory.success()); + assert!( + output([ + "ps", + "-aq", + "--filter", + &format!("name={}-assemble", ployz_build::builder_name()) + ]) + .trim() + .is_empty() + ); + assert!( + !Command::new("docker") + .args(["buildx", "inspect", &ployz_build::builder_name()]) + .status() + .unwrap() + .success() + ); + command(["volume", "inspect", &cache_volume()]); + fs::remove_dir_all(root).unwrap(); +} #[path = "build_layer3/policy.rs"] mod policy; diff --git a/crates/ployz/tests/build_layer3/remote.rs b/crates/ployz/tests/build_layer3/remote.rs index 67e2d0c3df..88270173e6 100644 --- a/crates/ployz/tests/build_layer3/remote.rs +++ b/crates/ployz/tests/build_layer3/remote.rs @@ -226,7 +226,7 @@ async fn request( remote::encode(&Input::Start(Definition { targets: vec![ployz_build::Target { name: "app".into(), - platform: None, + platforms: Vec::new(), }], output: Output::Load, no_cache: false, diff --git a/crates/ployz/tests/compose_cli.rs b/crates/ployz/tests/compose_cli.rs index 4f4fa5597d..4761d040d2 100644 --- a/crates/ployz/tests/compose_cli.rs +++ b/crates/ployz/tests/compose_cli.rs @@ -284,7 +284,14 @@ esac ..Default::default() }; let plan = plan_build(&project, &options).unwrap(); - execute_build(&plan, &options, &load, &mut project).unwrap(); + execute_build( + &plan, + &options, + &load, + &mut project, + &tokio_util::sync::CancellationToken::new(), + ) + .unwrap(); let calls = fs::read_to_string(calls).unwrap(); assert!(calls.lines().any(|argument| argument == "--check")); assert!(calls.lines().any(|argument| argument == "api")); diff --git a/crates/ployz/tests/connect/support.rs b/crates/ployz/tests/connect/support.rs index 71b280ade7..804db582e6 100644 --- a/crates/ployz/tests/connect/support.rs +++ b/crates/ployz/tests/connect/support.rs @@ -372,9 +372,10 @@ impl MachineRpc for DiscoveryService { .targets .iter() .map(|_| ployz_build::BuiltImage { - reference: format!("example.test/api@sha256:{}", "1".repeat(64)), + reference: format!("sha256:{}", "1".repeat(64)), tags: vec!["example.test/api:built".into()], - platform: "linux/amd64".into(), + platforms: vec!["linux/amd64".into()], + location: "unix:///var/run/docker.sock".into(), }) .collect(), }, diff --git a/crates/ployzd/src/build/tests.rs b/crates/ployzd/src/build/tests.rs index 821abfdaa7..91aa4c72df 100644 --- a/crates/ployzd/src/build/tests.rs +++ b/crates/ployzd/src/build/tests.rs @@ -101,7 +101,7 @@ impl Fixture { remote::encode(&Input::Start(Definition { targets: vec![ployz_build::Target { name: "api".into(), - platform: None, + platforms: Vec::new(), }], output, no_cache: false, @@ -222,11 +222,8 @@ async fn captured_build_crosses_owned_rpc_and_returns_only_remote_image_evidence let [image] = images.as_slice() else { panic!("expected one image: {images:?}") }; - assert_eq!( - image.reference, - format!("example.test/api@sha256:{}", "1".repeat(64)) - ); - assert_eq!(image.platform, "linux/amd64"); + assert_eq!(image.reference, format!("sha256:{}", "1".repeat(64))); + assert_eq!(image.platforms, ["linux/amd64"]); assert_eq!( fs::read_to_string(fixture.root.join("received-payload")).unwrap(), "captured-before-edit" @@ -383,7 +380,7 @@ async fn upload_timeout_stops_before_execution_and_releases_admission() { .send(Ok(remote::encode(&Input::Start(Definition { targets: vec![ployz_build::Target { name: "api".into(), - platform: None, + platforms: Vec::new(), }], output: Output::Load, no_cache: false,