From 2a8570c9a10a8e6d7b8d6f178945f8934c659145 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 8 Sep 2026 23:26:20 +0000 Subject: [PATCH 01/30] Start shared Machine installer and upgrade implementation (#822) From c13bcd88d98d1a3a1d85bfcfc22f9166c5b0ca7f Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 00:06:14 +0000 Subject: [PATCH 02/30] feat(ployzd): install Machines through daemon --- .goreleaser.yaml | 2 +- crates/ployzd/src/installer/host.rs | 338 ++++++++++++++++++ crates/ployzd/src/installer/mod.rs | 386 +++++++++++++++++++++ crates/ployzd/src/installer/release.rs | 457 +++++++++++++++++++++++++ crates/ployzd/src/installer/storage.rs | 443 ++++++++++++++++++++++++ crates/ployzd/src/lib.rs | 1 + crates/ployzd/src/main.rs | 56 ++- docs/RELEASE.md | 2 +- evidence/product-paths.tsv | 2 + scripts/pack-release.sh | 8 +- scripts/verify-release.sh | 4 +- 11 files changed, 1691 insertions(+), 8 deletions(-) create mode 100644 crates/ployzd/src/installer/host.rs create mode 100644 crates/ployzd/src/installer/mod.rs create mode 100644 crates/ployzd/src/installer/release.rs create mode 100644 crates/ployzd/src/installer/storage.rs create mode 100644 evidence/product-paths.tsv diff --git a/.goreleaser.yaml b/.goreleaser.yaml index cddc6c8cb8..ca7b1e8506 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -117,7 +117,7 @@ archives: checksum: name_template: checksums.txt - ids: [ployz] + ids: [ployz, ployzd] changelog: disable: true diff --git a/crates/ployzd/src/installer/host.rs b/crates/ployzd/src/installer/host.rs new file mode 100644 index 0000000000..06924bf65b --- /dev/null +++ b/crates/ployzd/src/installer/host.rs @@ -0,0 +1,338 @@ +//! Explicit Machine host preparation and systemd lifecycle management. + +use std::{ + ffi::OsStr, + fs::{self, File, OpenOptions}, + io::Write, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + path::Path, + process::{Command, Stdio}, +}; + +use super::release::{Release, fetch, installed_release}; +use super::{ + Error, InstallPaths, PLOYZ_USER, command_exists, run_apt, run_apt_with_env, run_host, systemctl, +}; + +const DOCKER_DAEMON_CONFIG: &str = r#"{ + "features": { "containerd-snapshotter": true }, + "live-restore": true, + "log-driver": "json-file", + "log-opts": { "max-size": "10m", "max-file": "3" } +}"#; + +pub(super) fn install_prerequisites() -> Result<(), Error> { + if command_exists("curl") { + return Ok(()); + } + if command_exists("apt-get") { + run_apt("refresh packages for prerequisites", ["update", "-qq"])?; + run_apt_with_env( + "install prerequisites", + ["install", "-y", "-qq", "curl", "ca-certificates"], + )?; + } else if command_exists("dnf") { + run_host( + "install prerequisites", + "dnf", + ["install", "-y", "curl", "ca-certificates"], + )?; + } else if command_exists("yum") { + run_host( + "install prerequisites", + "yum", + ["install", "-y", "curl", "ca-certificates"], + )?; + } else if command_exists("pacman") { + run_host( + "install prerequisites", + "pacman", + ["-Sy", "--noconfirm", "curl", "ca-certificates"], + )?; + } else if command_exists("zypper") { + run_host( + "install prerequisites", + "zypper", + ["--non-interactive", "install", "curl", "ca-certificates"], + )?; + } else { + return Err(Error::Command { + stage: "install prerequisites".into(), + message: "curl is required and no supported package manager was found".into(), + }); + } + Ok(()) +} + +pub(super) fn create_user_and_directories( + group_user: Option<&str>, + paths: &InstallPaths, +) -> Result<(), Error> { + let mut exists = Command::new("id"); + exists + .arg(PLOYZ_USER) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + if !exists + .status() + .map_err(|source| Error::Io { + stage: "inspect Ployz service account", + source, + })? + .success() + { + run_host( + "create Ployz service account", + "useradd", + [ + "--system", + "--home-dir", + "/nonexistent", + "--shell", + "/usr/sbin/nologin", + "--user-group", + PLOYZ_USER, + ], + )?; + } + if let Some(user) = group_user { + run_host( + "add operator to Ployz group", + "gpasswd", + ["--add", user, PLOYZ_USER], + )?; + } + let data = paths.data_dir.to_string_lossy(); + let run = paths.run_dir.to_string_lossy(); + run_host( + "create Ployz directories", + "install", + [ + "-d", "-m", "0750", "-o", PLOYZ_USER, "-g", PLOYZ_USER, &data, &run, + ], + )?; + Ok(()) +} + +pub(super) fn verify_software_prerequisites(paths: &InstallPaths) -> Result<(), Error> { + if !command_exists("dockerd") { + return Err(Error::Command { + stage: "software-only preflight".into(), + message: "Docker is not installed; run ployzd install without --software-only first" + .into(), + }); + } + let service_account = Command::new("id") + .arg(PLOYZ_USER) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|source| Error::Io { + stage: "software-only preflight", + source, + })?; + if !service_account.success() { + return Err(Error::Command { + stage: "software-only preflight".into(), + message: "the Ployz service account is missing; run ployzd install without --software-only first".into(), + }); + } + for path in [&paths.data_dir, &paths.run_dir] { + if !path.is_dir() { + return Err(Error::Command { + stage: "software-only preflight".into(), + message: format!( + "{} is missing; run ployzd install without --software-only first", + path.display() + ), + }); + } + } + Ok(()) +} + +pub(super) fn install_systemd(paths: &InstallPaths, install_only: bool) -> Result<(), Error> { + fs::create_dir_all(&paths.systemd_dir).map_err(|source| Error::Io { + stage: "create systemd unit directory", + source, + })?; + let bin = paths.bin_dir.display(); + write_file_atomically( + &paths.systemd_dir.join("ployz.service"), + &format!( + "[Unit]\nDescription=Ployz Machine daemon\nAfter=network-online.target docker.service\nWants=network-online.target\n\n[Service]\nType=notify\nExecStart={bin}/ployzd\n# Set PLOYZ_LOG=debug in /etc/default/ployz to raise verbosity.\nEnvironmentFile=-/etc/default/ployz\nTimeoutStartSec=20\nTimeoutStopSec=15\nRestart=always\nRestartPreventExitStatus=78\nRestartSec=2\nNoNewPrivileges=true\nProtectSystem=full\nProtectControlGroups=true\nProtectHome=read-only\nProtectKernelTunables=true\nPrivateTmp=true\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nRestrictNamespaces=true\n\n[Install]\nWantedBy=multi-user.target\n" + ), + "write systemd unit", + )?; + write_file_atomically( + &paths.systemd_dir.join("ployz-volume-plugin.socket"), + "[Unit]\nDescription=Ployz Docker Volume plugin socket\nBefore=docker.service\n\n[Socket]\nListenStream=/run/docker/plugins/ployz.sock\nSocketMode=0660\nDirectoryMode=0755\nAccept=no\nService=ployz-volume-plugin.service\n\n[Install]\nWantedBy=sockets.target\n", + "write systemd unit", + )?; + write_file_atomically( + &paths.systemd_dir.join("ployz-volume-plugin.service"), + &format!( + "[Unit]\nDescription=Ployz Docker Volume plugin\nBefore=docker.service\nAfter=zfs-import.target zfs-mount.service ployz-volume-plugin.socket\nRequires=ployz-volume-plugin.socket docker.service\n\n[Service]\nType=simple\nExecStart={bin}/ployzd volume-plugin\nSockets=ployz-volume-plugin.socket\nEnvironmentFile=-/etc/default/ployz\nRestart=on-failure\nRestartSec=2\nNoNewPrivileges=true\nRestrictAddressFamilies=AF_UNIX\nRestrictNamespaces=true\n" + ), + "write systemd unit", + )?; + if !install_only { + systemctl("reload systemd units", ["daemon-reload"])?; + systemctl("enable daemon", ["enable", "ployz.service"])?; + systemctl( + "enable volume plugin socket", + ["enable", "--now", "ployz-volume-plugin.socket"], + )?; + } + Ok(()) +} + +pub(super) fn write_file_atomically( + path: &Path, + content: &str, + stage: &'static str, +) -> Result<(), Error> { + let parent = path.parent().ok_or_else(|| { + Error::Verification(format!( + "systemd unit path {} has no parent", + path.display() + )) + })?; + let temporary = parent.join(format!( + ".{}-{}", + path.file_name() + .and_then(OsStr::to_str) + .unwrap_or("ployz-unit"), + std::process::id() + )); + let result = (|| { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o644) + .open(&temporary)?; + file.write_all(content.as_bytes())?; + file.sync_all()?; + fs::rename(&temporary, path)?; + File::open(parent)?.sync_all() + })(); + if result.is_err() { + let _ = fs::remove_file(&temporary); + } + result.map_err(|source| Error::Io { stage, source }) +} + +pub(super) async fn install_docker(paths: &InstallPaths, install_only: bool) -> Result<(), Error> { + if command_exists("dockerd") { + if !install_only { + let mut command = Command::new("docker"); + command.args(["info", "-f", "{{ .DriverStatus }}"]); + let snapshotter = command.output().ok().is_some_and(|output| { + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("io.containerd.snapshotter") + }); + if !snapshotter { + eprintln!( + "WARNING: Docker is retained unchanged; enable its containerd image store for best results" + ); + } + } + return Ok(()); + } + if !command_exists("apt-get") { + return Err(Error::Command { + stage: "install Docker".into(), + message: "Docker is absent and apt-get is unavailable; install Docker, then retry" + .into(), + }); + } + let script = fetch("https://get.docker.com", "download Docker installer").await?; + let mut command = Command::new("bash"); + command.args(["-o", "pipefail"]); + command.stdin(Stdio::piped()); + let mut child = command.spawn().map_err(|source| Error::Io { + stage: "start Docker installer", + source, + })?; + child + .stdin + .take() + .ok_or_else(|| { + Error::Verification("Docker installer standard input was unavailable".into()) + })? + .write_all(&script) + .map_err(|source| Error::Io { + stage: "send Docker installer", + source, + })?; + let status = child.wait().map_err(|source| Error::Io { + stage: "wait for Docker installer", + source, + })?; + if !status.success() { + return Err(Error::Command { + stage: "install Docker".into(), + message: format!("exited with {status}"), + }); + } + let parent = paths.docker_config.parent().ok_or_else(|| { + Error::Verification(format!( + "Docker config path {} has no parent", + paths.docker_config.display() + )) + })?; + fs::create_dir_all(parent).map_err(|source| Error::Io { + stage: "create Docker configuration directory", + source, + })?; + write_file_atomically( + &paths.docker_config, + DOCKER_DAEMON_CONFIG, + "write Docker configuration", + )?; + if !install_only { + systemctl("restart Docker", ["restart", "docker"])?; + } + Ok(()) +} + +pub(super) fn verify_running_daemon(paths: &InstallPaths, target: &Release) -> Result<(), Error> { + systemctl( + "check daemon readiness", + ["is-active", "--quiet", "ployz.service"], + )?; + let output = systemctl( + "inspect running daemon", + ["show", "--property=MainPID", "--value", "ployz.service"], + )?; + let pid = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if pid.is_empty() || pid == "0" || !pid.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(Error::Verification(format!( + "ployz.service is active but did not report a daemon process ID ({pid:?})" + ))); + } + let running = fs::metadata(format!("/proc/{pid}/exe")).map_err(|source| Error::Io { + stage: "inspect running daemon executable", + source, + })?; + let installed = fs::metadata(paths.bin_dir.join("ployzd")).map_err(|source| Error::Io { + stage: "inspect installed daemon executable", + source, + })?; + if (running.dev(), running.ino()) != (installed.dev(), installed.ino()) { + return Err(Error::Verification( + "ployz.service is active but does not run the activated daemon executable".into(), + )); + } + match installed_release(&paths.bin_dir.join("ployzd")) { + Some(observed) if &observed == target => Ok(()), + Some(observed) => Err(Error::Verification(format!( + "activated daemon reported {}, expected {}", + observed.as_string(), + target.as_string() + ))), + None => Err(Error::Verification( + "activated daemon no longer reports a valid version".into(), + )), + } +} diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs new file mode 100644 index 0000000000..47f9659509 --- /dev/null +++ b/crates/ployzd/src/installer/mod.rs @@ -0,0 +1,386 @@ +//! Bounded local installation of a Ployz Machine release. + +mod host; +mod release; +mod storage; + +use std::{ + fs::{self, File, OpenOptions}, + io, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +use fs2::FileExt; +use ployz_core::StorageChoice; +use thiserror::Error; + +use self::{ + host::{ + create_user_and_directories, install_docker, install_prerequisites, install_systemd, + verify_running_daemon, verify_software_prerequisites, + }, + release::{install_binaries, resolve_release}, + storage::prepare_storage, +}; + +const PLOYZ_USER: &str = "ployz"; + +#[derive(Clone, Debug)] +pub struct InstallRequest { + /// A fixed version or the stable/beta channel to resolve once. + pub version: String, + /// Explicit storage preparation for a fresh Machine. + pub storage: StorageChoice, + /// Whether host prerequisites may be prepared. `false` is software-only replacement. + pub prepare_host: bool, + /// Write the release and units without starting systemd. + pub install_only: bool, + /// Existing operator to add to the Ployz service group while preparing the host. + pub group_user: Option, + /// Local release directory used by offline qualification. + pub release_dir: Option, +} + +/// What installation directly observed after activation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Readiness { + /// Files and unit definitions were installed, but systemd was deliberately not contacted. + InstallationOnly, + /// systemd reported the daemon active and its process is the verified target executable. + Running, +} + +/// The bounded outcome of one local installation attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InstallOutcome { + /// Exact release selected before host mutation. + pub target: String, + /// Whether the daemon executable changed. + pub replaced: bool, + /// Readiness observed before the command returned. + pub readiness: Readiness, +} + +/// Installation failures name the first stage that did not complete. +#[derive(Debug, Error)] +pub enum Error { + #[error("run this command with sudo")] + NotRoot, + #[error("another Ployz installation is active")] + Busy, + #[error( + "software-only replacement cannot prepare {storage} storage; run without --software-only" + )] + StorageRequiresHostPreparation { storage: StorageChoice }, + #[error("nightly is not a supported release channel")] + Nightly, + #[error("invalid Ployz release version '{value}'; expected X.Y.Z or X.Y.Z-beta.N")] + InvalidVersion { value: String }, + #[error("Ployz Machine must be Linux")] + UnsupportedOs, + #[error("unsupported architecture: {0}")] + UnsupportedArchitecture(String), + #[error("Ployz requires systemd")] + SystemdRequired, + #[error("release selection: {0}")] + ReleaseSelection(String), + #[error("artifact verification: {0}")] + Verification(String), + #[error("{stage}: {message}")] + Command { stage: String, message: String }, + #[error("{stage}: {source}")] + Io { + stage: &'static str, + #[source] + source: io::Error, + }, +} + +#[derive(Clone, Debug)] +pub(super) struct InstallPaths { + pub(super) bin_dir: PathBuf, + pub(super) systemd_dir: PathBuf, + pub(super) data_dir: PathBuf, + pub(super) run_dir: PathBuf, + pub(super) docker_config: PathBuf, + pub(super) modprobe_dir: PathBuf, +} + +impl InstallPaths { + fn system() -> Self { + Self { + bin_dir: PathBuf::from("/usr/local/bin"), + systemd_dir: PathBuf::from("/etc/systemd/system"), + data_dir: PathBuf::from("/var/lib/ployz"), + run_dir: PathBuf::from("/run/ployz"), + docker_config: PathBuf::from("/etc/docker/daemon.json"), + modprobe_dir: PathBuf::from("/etc/modprobe.d"), + } + } +} + +struct InstallLock(File); + +impl Drop for InstallLock { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +/// Install the selected release through the one Machine-local installation seam. +/// +/// # Errors +/// +/// Returns the observed first failed installation stage. Acquisition and preflight failures +/// happen before activation, so the installed daemon remains untouched. +pub async fn install(request: InstallRequest) -> Result { + install_at(request, InstallPaths::system()).await +} + +async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result { + if !request.prepare_host && request.storage != StorageChoice::None { + return Err(Error::StorageRequiresHostPreparation { + storage: request.storage, + }); + } + require_root()?; + let _lock = claim_lock(&paths.run_dir)?; + verify_system(request.install_only)?; + let target = resolve_release(&request).await?; + + if request.prepare_host { + prepare_storage(request.storage, &paths)?; + install_prerequisites()?; + let inherited_group = sudo_user(); + create_user_and_directories( + request.group_user.as_deref().or(inherited_group.as_deref()), + &paths, + )?; + } else { + verify_software_prerequisites(&paths)?; + } + + let mut replaced = !paths.systemd_dir.join("ployz.service").is_file(); + replaced |= install_binaries(&request, &paths, &target).await?; + install_systemd(&paths, request.install_only)?; + if request.prepare_host { + install_docker(&paths, request.install_only).await?; + } + + let readiness = if request.install_only { + Readiness::InstallationOnly + } else { + if replaced { + systemctl("restart daemon", ["restart", "ployz.service"])?; + systemctl( + "restart volume plugin", + ["try-restart", "ployz-volume-plugin.service"], + )?; + } + verify_running_daemon(&paths, &target)?; + Readiness::Running + }; + Ok(InstallOutcome { + target: target.as_string(), + replaced, + readiness, + }) +} + +fn require_root() -> Result<(), Error> { + let mut command = Command::new("id"); + command.arg("-u"); + let output = run_command("check installation privilege", &mut command)?; + if String::from_utf8_lossy(&output.stdout).trim() == "0" { + Ok(()) + } else { + Err(Error::NotRoot) + } +} + +fn claim_lock(run_dir: &Path) -> Result { + fs::create_dir_all(run_dir).map_err(|source| Error::Io { + stage: "create installation lock directory", + source, + })?; + fs::set_permissions(run_dir, fs::Permissions::from_mode(0o750)).map_err(|source| { + Error::Io { + stage: "set installation lock directory permissions", + source, + } + })?; + let file = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .mode(0o600) + .open(run_dir.join(".install.lock")) + .map_err(|source| Error::Io { + stage: "open installation lock", + source, + })?; + file.try_lock_exclusive().map_err(|source| { + if source.kind() == io::ErrorKind::WouldBlock { + Error::Busy + } else { + Error::Io { + stage: "lock installation", + source, + } + } + })?; + Ok(InstallLock(file)) +} + +fn verify_system(install_only: bool) -> Result<(), Error> { + if std::env::consts::OS != "linux" { + return Err(Error::UnsupportedOs); + } + daemon_archive()?; + if !install_only && !Path::new("/run/systemd/system").is_dir() { + return Err(Error::SystemdRequired); + } + Ok(()) +} + +pub(super) fn daemon_archive() -> Result<&'static str, Error> { + match std::env::consts::ARCH { + "x86_64" => Ok("ployzd_linux_amd64.tar.gz"), + "aarch64" => Ok("ployzd_linux_arm64.tar.gz"), + architecture => Err(Error::UnsupportedArchitecture(architecture.into())), + } +} + +fn sudo_user() -> Option { + std::env::var("SUDO_USER") + .ok() + .filter(|user| !user.is_empty()) +} + +pub(super) fn command_exists(name: &str) -> bool { + std::env::var_os("PATH").is_some_and(|path| { + std::env::split_paths(&path).any(|directory| { + let candidate = directory.join(name); + candidate.is_file() + && candidate + .metadata() + .is_ok_and(|metadata| metadata.permissions().mode() & 0o111 != 0) + }) + }) +} + +pub(super) fn run_command( + stage: impl Into, + command: &mut Command, +) -> Result { + let stage = stage.into(); + let output = command.output().map_err(|source| Error::Io { + stage: "start host command", + source, + })?; + if output.status.success() { + Ok(output) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + Err(Error::Command { + stage, + message: if stderr.is_empty() { + format!("exited with {}", output.status) + } else { + stderr + }, + }) + } +} + +pub(super) fn systemctl(stage: &str, args: [&str; N]) -> Result { + let mut command = Command::new("systemctl"); + command.args(args); + run_command(stage, &mut command) +} + +pub(super) fn run_host( + stage: &str, + program: &str, + args: [&str; N], +) -> Result { + let mut command = Command::new(program); + command.args(args); + run_command(stage, &mut command) +} + +pub(super) fn run_apt(stage: &str, args: [&str; N]) -> Result { + let mut command = Command::new("apt-get"); + command.args(["-o", "DPkg::Lock::Timeout=300"]); + command.args(args); + run_command(stage, &mut command) +} + +pub(super) fn run_apt_with_env( + stage: &str, + args: [&str; N], +) -> Result { + let mut command = Command::new("apt-get"); + command + .args(["-o", "DPkg::Lock::Timeout=300"]) + .args(args) + .env("DEBIAN_FRONTEND", "noninteractive"); + run_command(stage, &mut command) +} + +#[cfg(test)] +mod tests { + use super::release::{Release, checksum_for}; + use super::*; + + #[test] + fn release_versions_accept_published_shapes_and_order_beta_before_stable() { + assert_eq!(Release::parse("v1.2.3").unwrap().as_string(), "1.2.3"); + assert_eq!( + Release::parse("1.2.3-beta.4").unwrap().as_string(), + "1.2.3-beta.4" + ); + assert!(Release::parse("1.2.3-beta.4").unwrap() < Release::parse("1.2.3").unwrap()); + } + + #[test] + fn release_versions_reject_non_published_shapes() { + for invalid in ["", "v", "1.2", "1.2.3-rc.1", "1.2.3-beta.x", "1.2.3.4"] { + assert!(Release::parse(invalid).is_err(), "{invalid}"); + } + } + + #[test] + fn checksum_parser_requires_the_requested_daemon_archive() { + let archive = "ployzd_linux_amd64.tar.gz"; + let checksum = "2d20468cbeb9745b56fdf16897f963773baeecf45c4ea2e00a46e8d82da6ee9f"; + assert_eq!( + checksum_for(format!("{checksum} {archive}\n").as_bytes(), archive), + Some(checksum.into()) + ); + assert_eq!(checksum_for(b"bad other.tar.gz\n", archive), None); + } + + #[tokio::test] + async fn software_only_rejects_storage_preparation_before_host_mutation() { + let error = install_at( + InstallRequest { + version: "1.2.3".into(), + storage: StorageChoice::Zfs, + prepare_host: false, + install_only: true, + group_user: None, + release_dir: None, + }, + InstallPaths::system(), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + Error::StorageRequiresHostPreparation { .. } + )); + } +} diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs new file mode 100644 index 0000000000..f918162212 --- /dev/null +++ b/crates/ployzd/src/installer/release.rs @@ -0,0 +1,457 @@ +//! Release selection, acquisition, verification, and activation. + +use std::{ + cmp::Ordering as Compare, + ffi::OsStr, + fs::{self, File, OpenOptions}, + io::{self, Write}, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; + +use sha2::{Digest, Sha256}; + +use super::{Error, InstallPaths, InstallRequest, daemon_archive, run_command}; + +const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; +const CHANNEL_URL: &str = "https://ployz.sh"; +const RELEASE_API: &str = "https://api.github.com/repos/getployz/ployz2"; +static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct Release { + major: u64, + minor: u64, + patch: u64, + /// `None` is a stable release, which sorts after its beta releases. + beta: Option, +} + +impl Release { + pub(super) fn parse(value: &str) -> Result { + let value = value.strip_prefix('v').unwrap_or(value); + let (core, beta) = match value.split_once("-beta.") { + Some((core, beta)) => (core, Some(beta)), + None => (value, None), + }; + if core.contains('-') { + return Err(Error::InvalidVersion { + value: value.into(), + }); + } + let mut pieces = core.split('.'); + let parse_part = |part: Option<&str>| { + part.filter(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit())) + .and_then(|part| part.parse().ok()) + }; + let (Some(major), Some(minor), Some(patch), None) = ( + parse_part(pieces.next()), + parse_part(pieces.next()), + parse_part(pieces.next()), + pieces.next(), + ) else { + return Err(Error::InvalidVersion { + value: value.into(), + }); + }; + let beta = match beta { + Some(beta) if beta.bytes().all(|byte| byte.is_ascii_digit()) => { + beta.parse().map(Some).map_err(|_| Error::InvalidVersion { + value: value.into(), + })? + } + Some(_) => { + return Err(Error::InvalidVersion { + value: value.into(), + }); + } + None => None, + }; + Ok(Self { + major, + minor, + patch, + beta, + }) + } + + pub(super) fn as_string(&self) -> String { + let Self { + major, + minor, + patch, + beta, + } = self; + match beta { + Some(beta) => format!("{major}.{minor}.{patch}-beta.{beta}"), + None => format!("{major}.{minor}.{patch}"), + } + } +} + +impl Ord for Release { + fn cmp(&self, other: &Self) -> Compare { + (self.major, self.minor, self.patch) + .cmp(&(other.major, other.minor, other.patch)) + .then_with(|| match (self.beta, other.beta) { + (Some(left), Some(right)) => left.cmp(&right), + (Some(_), None) => Compare::Less, + (None, Some(_)) => Compare::Greater, + (None, None) => Compare::Equal, + }) + } +} + +impl PartialOrd for Release { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +pub(super) async fn resolve_release(request: &InstallRequest) -> Result { + let requested = request + .version + .strip_prefix('v') + .unwrap_or(&request.version); + if requested == "nightly" { + return Err(Error::Nightly); + } + match requested { + "" | "latest" | "stable" => { + let value = release_file(request, "stable").await?; + Release::parse(value.trim()) + } + "beta" => { + let value = release_file(request, "beta").await?; + Release::parse(value.trim()) + } + version => Release::parse(version), + } +} + +async fn release_file(request: &InstallRequest, name: &str) -> Result { + let bytes = if let Some(directory) = &request.release_dir { + fs::read(directory.join(name)).map_err(|source| Error::Io { + stage: "read local release channel", + source, + })? + } else { + fetch(&format!("{CHANNEL_URL}/{name}"), "resolve release channel").await? + }; + String::from_utf8(bytes) + .map_err(|error| Error::ReleaseSelection(format!("{name} channel is not UTF-8: {error}"))) +} + +pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { + let client = reqwest::Client::builder() + .https_only(true) + .user_agent("ployzd-installer") + .build() + .map_err(|error| Error::ReleaseSelection(format!("build download client: {error}")))?; + let response = client + .get(url) + .send() + .await + .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}")))? + .error_for_status() + .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}")))?; + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}"))) +} + +fn release_url(target: &Release, file: &str) -> String { + format!( + "{RELEASE_REPOSITORY}/releases/download/v{}/{file}", + target.as_string() + ) +} + +pub(super) async fn install_binaries( + request: &InstallRequest, + paths: &InstallPaths, + target: &Release, +) -> Result { + let installed = installed_release(&paths.bin_dir.join("ployzd")); + let pinned = !matches!( + request + .version + .strip_prefix('v') + .unwrap_or(&request.version), + "" | "latest" | "stable" | "beta" + ); + let replace = match installed { + None => true, + Some(ref installed) if request.release_dir.is_some() => true, + Some(ref installed) if pinned => installed != target, + Some(ref installed) => installed < target, + }; + if !replace { + println!( + "ployzd {} retained", + installed + .expect("replace covers missing release") + .as_string() + ); + return Ok(false); + } + + let archive = daemon_archive()?; + let stage = Staging::new(&paths.bin_dir)?; + let archive_path = stage.path.join(archive); + let checksum = published_checksum(request, target, archive).await?; + let archive_bytes = if let Some(directory) = &request.release_dir { + fs::read(directory.join(archive)).map_err(|source| Error::Io { + stage: "read local daemon archive", + source, + })? + } else { + fetch(&release_url(target, archive), "download daemon archive").await? + }; + write_private(&archive_path, &archive_bytes, "stage daemon archive")?; + verify_checksum(&archive_path, &checksum)?; + extract_archive(&archive_path, &stage.path)?; + let daemon = stage.path.join("ployzd"); + let uninstall = stage.path.join("ployz-uninstall"); + verify_executable(&daemon, target)?; + verify_uninstall(&uninstall)?; + activate(&daemon, &uninstall, paths)?; + Ok(true) +} + +pub(super) fn installed_release(path: &Path) -> Option { + if !path.is_file() { + return None; + } + let output = Command::new(path).arg("version").output().ok()?; + output.status.success().then_some(())?; + Release::parse(String::from_utf8_lossy(&output.stdout).trim()).ok() +} + +async fn published_checksum( + request: &InstallRequest, + target: &Release, + archive: &str, +) -> Result { + let checksums = if let Some(directory) = &request.release_dir { + fs::read(directory.join("checksums.txt")).map_err(|source| Error::Io { + stage: "read local release checksums", + source, + })? + } else { + fetch( + &release_url(target, "checksums.txt"), + "download release checksums", + ) + .await? + }; + if let Some(checksum) = checksum_for(&checksums, archive) { + return Ok(checksum); + } + if request.release_dir.is_some() { + return Err(Error::Verification(format!( + "checksums.txt has no hash for {archive}" + ))); + } + github_asset_digest(target, archive).await +} + +pub(super) fn checksum_for(checksums: &[u8], archive: &str) -> Option { + let checksums = std::str::from_utf8(checksums).ok()?; + checksums.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + let hash = fields.next()?; + let name = fields.next()?.trim_start_matches('*'); + (name == archive && valid_checksum(hash)).then(|| hash.to_ascii_lowercase()) + }) +} + +fn valid_checksum(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +async fn github_asset_digest(target: &Release, archive: &str) -> Result { + let endpoint = format!("{RELEASE_API}/releases/tags/v{}", target.as_string()); + let response = fetch(&endpoint, "read published daemon checksum").await?; + let release: serde_json::Value = serde_json::from_slice(&response).map_err(|error| { + Error::Verification(format!("published release metadata is invalid: {error}")) + })?; + let digest = release + .get("assets") + .and_then(serde_json::Value::as_array) + .and_then(|assets| { + assets.iter().find_map(|asset| { + (asset.get("name").and_then(serde_json::Value::as_str) == Some(archive)) + .then(|| asset.get("digest").and_then(serde_json::Value::as_str)) + .flatten() + }) + }) + .and_then(|digest| digest.strip_prefix("sha256:")) + .filter(|digest| valid_checksum(digest)) + .ok_or_else(|| { + Error::Verification(format!( + "published release metadata has no SHA-256 digest for {archive}" + )) + })?; + Ok(digest.to_ascii_lowercase()) +} + +fn verify_checksum(path: &Path, expected: &str) -> Result<(), Error> { + let bytes = fs::read(path).map_err(|source| Error::Io { + stage: "read staged daemon archive", + source, + })?; + let actual = hex::encode(Sha256::digest(bytes)); + if actual == expected { + Ok(()) + } else { + Err(Error::Verification(format!( + "{} checksum was {actual}, expected {expected}", + path.file_name() + .and_then(OsStr::to_str) + .unwrap_or("daemon archive") + ))) + } +} + +fn extract_archive(archive: &Path, destination: &Path) -> Result<(), Error> { + let mut command = Command::new("tar"); + command + .args(["-xzf"]) + .arg(archive) + .arg("-C") + .arg(destination); + run_command("extract staged daemon archive", &mut command)?; + Ok(()) +} + +fn verify_executable(path: &Path, target: &Release) -> Result<(), Error> { + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).map_err(|source| Error::Io { + stage: "mark staged daemon executable", + source, + })?; + let mut command = Command::new(path); + command.arg("version"); + let output = run_command("preflight staged daemon", &mut command)?; + let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if observed == target.as_string() { + Ok(()) + } else { + Err(Error::Verification(format!( + "staged daemon reported version {observed:?}, expected {}", + target.as_string() + ))) + } +} + +fn verify_uninstall(path: &Path) -> Result<(), Error> { + let metadata = path.metadata().map_err(|source| Error::Io { + stage: "preflight staged uninstall command", + source, + })?; + if metadata.is_file() && metadata.len() != 0 { + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).map_err(|source| Error::Io { + stage: "mark staged uninstall command executable", + source, + }) + } else { + Err(Error::Verification( + "daemon archive does not contain a usable ployz-uninstall command".into(), + )) + } +} + +pub(super) struct Staging { + pub(super) path: PathBuf, +} + +impl Staging { + pub(super) fn new(parent: &Path) -> Result { + fs::create_dir_all(parent).map_err(|source| Error::Io { + stage: "create daemon installation directory", + source, + })?; + for _ in 0..100 { + let sequence = STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed); + let path = parent.join(format!(".ployz-stage-{}-{sequence}", std::process::id())); + match fs::create_dir(&path) { + Ok(()) => { + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).map_err( + |source| Error::Io { + stage: "protect daemon staging directory", + source, + }, + )?; + return Ok(Self { path }); + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, + Err(source) => { + return Err(Error::Io { + stage: "create daemon staging directory", + source, + }); + } + } + } + Err(Error::Verification( + "could not create a private daemon staging directory".into(), + )) + } +} + +impl Drop for Staging { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +pub(super) fn write_private(path: &Path, bytes: &[u8], stage: &'static str) -> Result<(), Error> { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(path) + .map_err(|source| Error::Io { stage, source })?; + file.write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|source| Error::Io { stage, source }) +} + +fn activate(daemon: &Path, uninstall: &Path, paths: &InstallPaths) -> Result<(), Error> { + let installed = paths.bin_dir.join("ployzd"); + let previous = paths.bin_dir.join("ployzd.previous"); + match fs::remove_file(&previous) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(source) => { + return Err(Error::Io { + stage: "remove previous daemon release", + source, + }); + } + } + if installed.exists() { + fs::hard_link(&installed, &previous).map_err(|source| Error::Io { + stage: "retain previous daemon release", + source, + })?; + } + fs::rename(daemon, &installed).map_err(|source| Error::Io { + stage: "activate verified daemon", + source, + })?; + fs::rename(uninstall, paths.bin_dir.join("ployz-uninstall")).map_err(|source| Error::Io { + stage: "activate uninstall command", + source, + })?; + File::open(&paths.bin_dir) + .and_then(|directory| directory.sync_all()) + .map_err(|source| Error::Io { + stage: "persist daemon activation", + source, + }) +} diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs new file mode 100644 index 0000000000..5842ce64bd --- /dev/null +++ b/crates/ployzd/src/installer/storage.rs @@ -0,0 +1,443 @@ +//! Optional ZFS preparation for a fresh Machine. + +use std::{ + ffi::OsStr, + fs::{self, OpenOptions}, + io::Write, + path::Path, + process::Command, + sync::atomic::{AtomicU64, Ordering}, +}; + +use ployz_core::StorageChoice; + +use super::{ + Error, InstallPaths, command_exists, run_apt, run_apt_with_env, run_command, run_host, +}; +use super::{ + host::write_file_atomically, + release::{Staging, write_private}, +}; + +const ZFS_SMOKE_BYTES: u64 = 128 * 1024 * 1024; +static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +pub(super) fn prepare_storage(storage: StorageChoice, paths: &InstallPaths) -> Result<(), Error> { + match storage { + StorageChoice::None => Ok(()), + StorageChoice::Zfs => prepare_zfs(paths), + } +} + +fn prepare_zfs(paths: &InstallPaths) -> Result<(), Error> { + let os = operating_system_id()?; + if os != "ubuntu" { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "ZFS storage preparation is not supported on {os} yet; use a supported Ubuntu release" + ), + }); + } + let container = container_virtualization(); + if container == "openvz" || (Path::new("/proc/vz").is_dir() && !Path::new("/proc/bc").is_dir()) + { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: "OpenVZ does not allow this Machine to load the host ZFS kernel module".into(), + }); + } + if container == "lxc" && lxc_is_unprivileged()? { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: + "Unprivileged LXC does not allow this Machine to load the host ZFS kernel module" + .into(), + }); + } + if !command_exists("apt-get") { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: "Ubuntu apt-get is required for ZFS storage preparation".into(), + }); + } + let kernel = uname("-r", "read running kernel")?; + require_host_root_reserve(ZFS_SMOKE_BYTES)?; + let cap = zfs_arc_max()?; + persist_zfs_arc_max(paths, cap)?; + install_zfs_packages(&kernel)?; + run_host("load ZFS kernel module", "modprobe", ["zfs"])?; + set_and_verify_zfs_arc_max(cap)?; + validate_zfs()?; + println!("ZFS storage preparation validated; no Machine Pool was created"); + Ok(()) +} + +fn operating_system_id() -> Result { + let value = fs::read_to_string("/etc/os-release").map_err(|source| Error::Io { + stage: "identify Linux distribution for ZFS storage preparation", + source, + })?; + value + .lines() + .find_map(|line| line.strip_prefix("ID=")) + .map(|id| id.trim_matches('"').to_owned()) + .filter(|id| !id.is_empty()) + .ok_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: "Could not identify the Linux distribution for ZFS storage preparation".into(), + }) +} + +fn container_virtualization() -> String { + Command::new("systemd-detect-virt") + .arg("--container") + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned()) + .unwrap_or_default() +} + +fn lxc_is_unprivileged() -> Result { + let map = fs::read_to_string("/proc/self/uid_map").map_err(|source| Error::Io { + stage: "inspect LXC user namespace", + source, + })?; + let mut fields = map.split_whitespace(); + Ok(matches!( + (fields.next(), fields.next()), + (Some("0"), Some(outside)) if outside != "0" + )) +} + +fn uname(arg: &str, stage: &str) -> Result { + let output = run_host(stage, "uname", [arg])?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if value.is_empty() { + Err(Error::Command { + stage: stage.into(), + message: "returned no value".into(), + }) + } else { + Ok(value) + } +} + +fn require_host_root_reserve(allocation: u64) -> Result<(), Error> { + let output = run_host( + "inspect host-root capacity for ZFS storage preparation", + "df", + ["-B1", "--output=size,avail", "/"], + )?; + let line = String::from_utf8_lossy(&output.stdout) + .lines() + .last() + .unwrap_or_default() + .to_owned(); + let mut fields = line.split_whitespace(); + let parse = |value: Option<&str>| value.and_then(|value| value.parse::().ok()); + let (Some(size), Some(available), None) = + (parse(fields.next()), parse(fields.next()), fields.next()) + else { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: "Could not read host-root capacity for ZFS storage preparation".into(), + }); + }; + let reserve = (size / 4).max(10 * 1024 * 1024 * 1024); + if available < reserve.saturating_add(allocation) { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Host root has {available} bytes available; ZFS validation needs {allocation} bytes while preserving the {reserve}-byte host-root reserve" + ), + }); + } + Ok(()) +} + +fn zfs_arc_max() -> Result { + let memory = fs::read_to_string("/proc/meminfo").map_err(|source| Error::Io { + stage: "read total RAM for ZFS ARC limit", + source, + })?; + let kib = memory + .lines() + .find_map(|line| { + line.strip_prefix("MemTotal:") + .and_then(|value| value.split_whitespace().next()) + }) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: "Could not read total RAM for the ZFS ARC limit".into(), + })?; + Ok((kib.saturating_mul(1024) / 4).clamp(256 * 1024 * 1024, 1024 * 1024 * 1024)) +} + +fn persist_zfs_arc_max(paths: &InstallPaths, cap: u64) -> Result<(), Error> { + fs::create_dir_all(&paths.modprobe_dir).map_err(|source| Error::Io { + stage: "create ZFS module configuration directory", + source, + })?; + write_file_atomically( + &paths.modprobe_dir.join("ployz-zfs.conf"), + &format!("options zfs zfs_arc_max={cap}\n"), + "persist ZFS ARC limit", + ) +} + +fn package_has_zfs_module(files: &str, kernel: &str) -> bool { + let marker = format!("/lib/modules/{kernel}/"); + files.lines().any(|line| { + line.contains(&marker) && (line.ends_with("/zfs.ko") || line.contains("/zfs.ko.")) + }) +} + +fn install_zfs_packages(kernel: &str) -> Result<(), Error> { + run_apt("refresh Ubuntu packages for ZFS", ["update", "-qq"])?; + let mut package = None; + for candidate in [ + format!("linux-main-modules-zfs-{kernel}"), + format!("linux-modules-zfs-{kernel}"), + format!("linux-modules-{kernel}"), + format!("linux-modules-extra-{kernel}"), + ] { + let mut show = Command::new("apt-cache"); + show.args(["show", &candidate]); + if !show + .status() + .map_err(|source| Error::Io { + stage: "inspect Ubuntu ZFS module packages", + source, + })? + .success() + { + continue; + } + let files = command_stdout( + "inspect installed Ubuntu ZFS module package", + "dpkg-query", + ["-L", &candidate], + )?; + if package_has_zfs_module(&files, kernel) { + package = Some(candidate); + break; + } + let scratch = Staging::new(Path::new("/var/tmp"))?; + let mut download = Command::new("apt-get"); + download + .args(["-o", "DPkg::Lock::Timeout=300", "download", &candidate]) + .current_dir(&scratch.path); + if !run_command("download Ubuntu ZFS module package", &mut download).is_ok() { + continue; + } + let archive = fs::read_dir(&scratch.path) + .map_err(|source| Error::Io { + stage: "inspect downloaded Ubuntu ZFS module package", + source, + })? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension() == Some(OsStr::new("deb"))); + let Some(archive) = archive else { continue }; + let mut contents = Command::new("dpkg-deb"); + contents.args(["-c"]).arg(&archive); + let output = run_command( + "inspect downloaded Ubuntu ZFS module package", + &mut contents, + )?; + if package_has_zfs_module(&String::from_utf8_lossy(&output.stdout), kernel) { + package = Some(candidate); + break; + } + } + let package = package.ok_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Ubuntu has no packaged ZFS module for the running kernel {kernel}; install a supported Ubuntu kernel and retry" + ), + })?; + run_apt_with_env( + "install ZFS packages", + [ + "install", + "-y", + "-qq", + "--no-install-recommends", + "zfsutils-linux", + &package, + ], + )?; + let files = command_stdout( + "verify installed Ubuntu ZFS module package", + "dpkg-query", + ["-L", &package], + )?; + if package_has_zfs_module(&files, kernel) { + Ok(()) + } else { + Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Installed package {package} does not supply the ZFS module for running kernel {kernel}" + ), + }) + } +} + +fn command_stdout( + stage: &str, + program: &str, + args: [&str; N], +) -> Result { + let output = run_host(stage, program, args)?; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +fn set_and_verify_zfs_arc_max(cap: u64) -> Result<(), Error> { + let path = Path::new("/sys/module/zfs/parameters/zfs_arc_max"); + let mut file = OpenOptions::new() + .write(true) + .open(path) + .map_err(|source| Error::Io { + stage: "apply ZFS ARC limit", + source, + })?; + writeln!(file, "{cap}").map_err(|source| Error::Io { + stage: "apply ZFS ARC limit", + source, + })?; + let observed = fs::read_to_string(path).map_err(|source| Error::Io { + stage: "verify ZFS ARC limit", + source, + })?; + if observed.trim() == cap.to_string() { + Ok(()) + } else { + Err(Error::Verification(format!( + "loaded ZFS module reports zfs_arc_max={} instead of the required {cap}", + observed.trim() + ))) + } +} + +fn validate_zfs() -> Result<(), Error> { + require_host_root_reserve(ZFS_SMOKE_BYTES)?; + let stage = Staging::new(Path::new("/var/tmp"))?; + let backing = stage.path.join("backing"); + write_private(&backing, b"", "create ZFS smoke backing file")?; + let pool = format!( + "ployz-smoke-{}-{}", + std::process::id(), + STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed) + ); + let mut pool_created = false; + let result = (|| { + let mut allocate = Command::new("fallocate"); + allocate + .args(["-l", &ZFS_SMOKE_BYTES.to_string()]) + .arg(&backing); + run_command("preallocate ZFS smoke backing file", &mut allocate)?; + let mut stat = Command::new("stat"); + stat.args(["-c", "%b %B"]).arg(&backing); + let output = run_command("verify ZFS smoke backing allocation", &mut stat)?; + let stdout = String::from_utf8_lossy(&output.stdout); + let mut fields = stdout.split_whitespace(); + let blocks = fields.next().and_then(|value| value.parse::().ok()); + let block_size = fields.next().and_then(|value| value.parse::().ok()); + if blocks + .zip(block_size) + .is_none_or(|(blocks, size)| blocks.saturating_mul(size) < ZFS_SMOKE_BYTES) + { + return Err(Error::Verification(format!( + "ZFS smoke backing file {} is sparse", + backing.display() + ))); + } + let mut mount = Command::new("findmnt"); + mount.args(["-n", "-o", "TARGET", "-T"]).arg(&backing); + let output = run_command("verify ZFS smoke backing filesystem", &mut mount)?; + if String::from_utf8_lossy(&output.stdout).trim() != "/" { + return Err(Error::Verification(format!( + "ZFS smoke backing file {} is not on the host root filesystem", + backing.display() + ))); + } + let mut create = Command::new("zpool"); + create + .args(["create", "-f", "-m", "none", "-o", "cachefile=none", &pool]) + .arg(&backing); + run_command("create temporary ZFS smoke Pool", &mut create)?; + pool_created = true; + run_host( + "query temporary ZFS smoke Pool", + "zpool", + ["list", "-Hp", "-o", "name,size,alloc,free", &pool], + )?; + run_host( + "query temporary ZFS smoke dataset", + "zfs", + ["list", "-Hp", "-o", "name,mountpoint", &pool], + )?; + Ok(()) + })(); + let cleanup = cleanup_zfs_smoke(&pool, &backing, pool_created); + match (result, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) => Err(error), + (Ok(()), Err(error)) => Err(error), + (Err(error), Err(cleanup)) => { + Err(Error::Verification(format!("{cleanup} (after: {error})"))) + } + } +} + +fn cleanup_zfs_smoke(pool: &str, backing: &Path, pool_created: bool) -> Result<(), Error> { + let existing_pool = if pool_created { + true + } else { + command_stdout( + "inspect temporary ZFS smoke Pool after failed creation", + "zpool", + ["list", "-H", "-o", "name"], + )? + .lines() + .any(|name| name == pool) + }; + if existing_pool { + let mut destroy = Command::new("zpool"); + destroy.args(["destroy", "-f", pool]); + run_command("destroy temporary ZFS smoke Pool", &mut destroy)?; + } + let pools = command_stdout( + "verify temporary ZFS smoke Pool cleanup", + "zpool", + ["list", "-H", "-o", "name"], + )?; + if pools.lines().any(|name| name == pool) { + return Err(Error::Verification(format!( + "Temporary ZFS smoke Pool {pool} remains after destroy" + ))); + } + let datasets = command_stdout( + "verify temporary ZFS smoke dataset cleanup", + "zfs", + ["list", "-H", "-o", "name"], + )?; + if datasets.lines().any(|name| { + name == pool + || name + .strip_prefix(pool) + .is_some_and(|suffix| suffix.starts_with('/')) + }) { + return Err(Error::Verification(format!( + "Temporary ZFS smoke dataset {pool} remains after destroy" + ))); + } + fs::remove_file(backing).map_err(|source| Error::Io { + stage: "remove temporary ZFS smoke backing file", + source, + }) +} diff --git a/crates/ployzd/src/lib.rs b/crates/ployzd/src/lib.rs index e28a5ddab7..5b99dc4955 100644 --- a/crates/ployzd/src/lib.rs +++ b/crates/ployzd/src/lib.rs @@ -22,6 +22,7 @@ pub(crate) mod filesystem; mod host_capacity; mod hosted_dns; pub(crate) mod ingress; +pub mod installer; pub mod logs; pub mod machine; pub mod machine_api; diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index f470533146..2635c9a73e 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -12,10 +12,11 @@ use std::{ }; use clap::{Parser, Subcommand}; -use ployz_core::DOCKER_NETWORK_CONFLICT_EXIT_STATUS; +use ployz_core::{DOCKER_NETWORK_CONFLICT_EXIT_STATUS, StorageChoice}; use ployzd::{ daemon::{ContainerMode, Daemon, DaemonConfig, Error, wait_until_socket_accepts}, diag, + installer::{InstallRequest, Readiness}, machine::DEFAULT_DATA_DIR, network::NetworkError, }; @@ -53,6 +54,27 @@ enum Command { DialStdio, /// Serve the Docker Volume plugin on its systemd socket. VolumePlugin, + /// Install or replace this Machine's daemon release. + Install { + /// Release channel (stable or beta) or exact published version. + #[arg(long, default_value = "stable")] + version: String, + /// Prepare ZFS storage, or leave this Machine stateless. + #[arg(long, default_value = "none")] + storage: StorageChoice, + /// Replace daemon software only; do not install Docker, OS packages, or prepare storage. + #[arg(long)] + software_only: bool, + /// Write files and units but do not contact or start systemd. + #[arg(long)] + install_only: bool, + /// Add this existing operator to the Ployz service group during host preparation. + #[arg(long, value_name = "USER")] + group_user: Option, + /// Read a verified release from this local directory. Used by offline qualification. + #[arg(long, hide = true, value_name = "DIR")] + release_dir: Option, + }, } fn main() -> ExitCode { @@ -97,6 +119,38 @@ async fn run(args: Args) -> Result<(), Error> { if matches!(args.command, Some(Command::DialStdio)) { return dial_stdio(&args.socket).await.map_err(Error::from); } + if let Some(Command::Install { + version, + storage, + software_only, + install_only, + group_user, + release_dir, + }) = args.command + { + let outcome = ployzd::installer::install(InstallRequest { + version, + storage, + prepare_host: !software_only, + install_only, + group_user, + release_dir, + }) + .await + .map_err(io::Error::other)?; + match outcome.readiness { + Readiness::InstallationOnly => { + println!( + "Ployz {} installed; systemd was not started", + outcome.target + ); + } + Readiness::Running => { + println!("Ployz {} is running and ready", outcome.target); + } + } + return Ok(()); + } diag::init(args.log_level.as_deref()) .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; if matches!(args.command, Some(Command::VolumePlugin)) { diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 590c7ba2bd..bf561f0c85 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -81,7 +81,7 @@ Goreleaser does not touch the tap (`--skip=homebrew`); `scripts/promote-release. ## Machine daemon -`scripts/install.sh` on Linux. Same version tokens: `latest` / `stable` / `beta` / pin. Set `PLOYZ_VERSION`. +`ployzd install` on Linux installs or replaces a Machine daemon. It accepts `--version stable`, `--version beta`, or an exact version; use `--software-only` for ordinary replacement after the Machine has already been prepared. The bootstrap script remains available for the first executable during the migration. ## Cloud Relay diff --git a/evidence/product-paths.tsv b/evidence/product-paths.tsv new file mode 100644 index 0000000000..0215494ea0 --- /dev/null +++ b/evidence/product-paths.tsv @@ -0,0 +1,2 @@ +path rung1 rung2 rung3 rung4 rung5 +machine-installer crates/ployzd/src/installer.rs::tests - - - ployzd install with a verified musl release on the disposable systemd Machine diff --git a/scripts/pack-release.sh b/scripts/pack-release.sh index 2ebd3ae7e3..fc799ff203 100755 --- a/scripts/pack-release.sh +++ b/scripts/pack-release.sh @@ -15,10 +15,12 @@ cli_archives=$(printf '%s\n' \ ployz_linux_arm64.tar.gz \ ployz_macos_amd64.tar.gz \ ployz_macos_arm64.tar.gz) +daemon_archives=$(printf '%s\n' \ + ployzd_linux_amd64.tar.gz \ + ployzd_linux_arm64.tar.gz) all_archives=$(printf '%s\n' \ $cli_archives \ - ployzd_linux_amd64.tar.gz \ - ployzd_linux_arm64.tar.gz \ + $daemon_archives \ ployz-relay_linux_amd64.tar.gz \ ployz-relay_linux_arm64.tar.gz) @@ -34,7 +36,7 @@ repo=${GITHUB_REPOSITORY:-getployz/ployz2} ( cd "$DIST" # shellcheck disable=SC2086 - sha256sum $cli_archives | sort -k2 > checksums.txt + sha256sum $cli_archives $daemon_archives | sort -k2 > checksums.txt ) write_homebrew_formula_from_checksums "$DIST/checksums.txt" "$DIST/homebrew/ployz.rb" "$version" "$tag" "$repo" diff --git a/scripts/verify-release.sh b/scripts/verify-release.sh index baa14cfdbe..d1804c7e93 100755 --- a/scripts/verify-release.sh +++ b/scripts/verify-release.sh @@ -49,8 +49,8 @@ check_archive() { check_checksums_and_formula() { checksum_names=$(awk '{print $2}' "$DIST/checksums.txt" | sort) - expected_checksums=$(cli_archives | sort) - [ "$checksum_names" = "$expected_checksums" ] || fail "checksums.txt must cover only CLI archives" + expected_checksums=$(printf '%s\n' "$(cli_archives)" ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz | sort) + [ "$checksum_names" = "$expected_checksums" ] || fail "checksums.txt must cover CLI and daemon archives" formula=$(find "$DIST" -name 'ployz.rb' -print -quit) [ -n "$formula" ] || fail "Homebrew formula was not generated" From d18aed4bfa602878e3e3370a4f04ce6ad4ed3d92 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 00:49:53 +0000 Subject: [PATCH 03/30] fix(ployzd): harden Machine installer --- Cargo.lock | 1 + crates/ployzd/Cargo.toml | 1 + crates/ployzd/src/installer/host.rs | 71 ++-- crates/ployzd/src/installer/mod.rs | 388 ++++++++++++++++----- crates/ployzd/src/installer/release.rs | 460 +++++++++++++++---------- crates/ployzd/src/installer/storage.rs | 74 ++-- crates/ployzd/src/main.rs | 103 +++++- evidence/product-paths.tsv | 2 +- 8 files changed, 744 insertions(+), 356 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58424b291c..60e65149f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2270,6 +2270,7 @@ dependencies = [ "reqwest", "rusqlite", "sd-notify", + "semver", "serde", "serde_json", "sha2", diff --git a/crates/ployzd/Cargo.toml b/crates/ployzd/Cargo.toml index 91806cf742..2ac76ac3fa 100644 --- a/crates/ployzd/Cargo.toml +++ b/crates/ployzd/Cargo.toml @@ -34,6 +34,7 @@ rusqlite = { version = "0.40.2", default-features = false, features = ["bundled" sd-notify = "0.5.0" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" +semver = "1.0.28" sha2 = "0.11.0" shell-words = "1.1.1" thiserror = "2.0.20" diff --git a/crates/ployzd/src/installer/host.rs b/crates/ployzd/src/installer/host.rs index 06924bf65b..ed29fde22e 100644 --- a/crates/ployzd/src/installer/host.rs +++ b/crates/ployzd/src/installer/host.rs @@ -1,18 +1,19 @@ //! Explicit Machine host preparation and systemd lifecycle management. use std::{ - ffi::OsStr, - fs::{self, File, OpenOptions}, + fs, io::Write, - os::unix::fs::{MetadataExt, OpenOptionsExt}, + os::unix::fs::MetadataExt, path::Path, process::{Command, Stdio}, }; -use super::release::{Release, fetch, installed_release}; -use super::{ - Error, InstallPaths, PLOYZ_USER, command_exists, run_apt, run_apt_with_env, run_host, systemctl, -}; +use semver::Version; + +use crate::filesystem::atomic_write; + +use super::release::{fetch, installed_release}; +use super::{Error, InstallPaths, PLOYZ_USER, command_exists, run_apt, run_host, systemctl}; const DOCKER_DAEMON_CONFIG: &str = r#"{ "features": { "containerd-snapshotter": true }, @@ -26,10 +27,15 @@ pub(super) fn install_prerequisites() -> Result<(), Error> { return Ok(()); } if command_exists("apt-get") { - run_apt("refresh packages for prerequisites", ["update", "-qq"])?; - run_apt_with_env( + run_apt( + "refresh packages for prerequisites", + ["update", "-qq"], + None, + )?; + run_apt( "install prerequisites", ["install", "-y", "-qq", "curl", "ca-certificates"], + None, )?; } else if command_exists("dnf") { run_host( @@ -192,34 +198,7 @@ pub(super) fn write_file_atomically( content: &str, stage: &'static str, ) -> Result<(), Error> { - let parent = path.parent().ok_or_else(|| { - Error::Verification(format!( - "systemd unit path {} has no parent", - path.display() - )) - })?; - let temporary = parent.join(format!( - ".{}-{}", - path.file_name() - .and_then(OsStr::to_str) - .unwrap_or("ployz-unit"), - std::process::id() - )); - let result = (|| { - let mut file = OpenOptions::new() - .create_new(true) - .write(true) - .mode(0o644) - .open(&temporary)?; - file.write_all(content.as_bytes())?; - file.sync_all()?; - fs::rename(&temporary, path)?; - File::open(parent)?.sync_all() - })(); - if result.is_err() { - let _ = fs::remove_file(&temporary); - } - result.map_err(|source| Error::Io { stage, source }) + atomic_write(path, content.as_bytes(), 0o644).map_err(|source| Error::Io { stage, source }) } pub(super) async fn install_docker(paths: &InstallPaths, install_only: bool) -> Result<(), Error> { @@ -239,13 +218,6 @@ pub(super) async fn install_docker(paths: &InstallPaths, install_only: bool) -> } return Ok(()); } - if !command_exists("apt-get") { - return Err(Error::Command { - stage: "install Docker".into(), - message: "Docker is absent and apt-get is unavailable; install Docker, then retry" - .into(), - }); - } let script = fetch("https://get.docker.com", "download Docker installer").await?; let mut command = Command::new("bash"); command.args(["-o", "pipefail"]); @@ -296,7 +268,10 @@ pub(super) async fn install_docker(paths: &InstallPaths, install_only: bool) -> Ok(()) } -pub(super) fn verify_running_daemon(paths: &InstallPaths, target: &Release) -> Result<(), Error> { +pub(super) async fn verify_running_daemon( + paths: &InstallPaths, + target: &Version, +) -> Result<(), Error> { systemctl( "check daemon readiness", ["is-active", "--quiet", "ployz.service"], @@ -324,12 +299,10 @@ pub(super) fn verify_running_daemon(paths: &InstallPaths, target: &Release) -> R "ployz.service is active but does not run the activated daemon executable".into(), )); } - match installed_release(&paths.bin_dir.join("ployzd")) { + match installed_release(&paths.bin_dir.join("ployzd")).await? { Some(observed) if &observed == target => Ok(()), Some(observed) => Err(Error::Verification(format!( - "activated daemon reported {}, expected {}", - observed.as_string(), - target.as_string() + "activated daemon reported {observed}, expected {target}" ))), None => Err(Error::Verification( "activated daemon no longer reports a valid version".into(), diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index 47f9659509..f1ff3056f5 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -25,22 +25,35 @@ use self::{ storage::prepare_storage, }; +pub use self::release::{ReleaseRequest, ReleaseSource}; + const PLOYZ_USER: &str = "ployz"; +/// Explicit host work associated with one Machine installation attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum Preparation { + /// Replace the daemon only after confirming the Machine is already prepared. + SoftwareOnly, + /// Prepare the host and optionally configure its requested storage. + PrepareHost { + /// Explicit storage preparation for a fresh Machine. + storage: StorageChoice, + /// Existing operator to add to the Ployz service group during host preparation. + group_user: Option, + }, +} + +/// Choices for one local Machine installation attempt. #[derive(Clone, Debug)] pub struct InstallRequest { - /// A fixed version or the stable/beta channel to resolve once. - pub version: String, - /// Explicit storage preparation for a fresh Machine. - pub storage: StorageChoice, - /// Whether host prerequisites may be prepared. `false` is software-only replacement. - pub prepare_host: bool, + /// A fixed version or a channel resolved once before host mutation. + pub release: ReleaseRequest, + /// Trusted release source. Published releases never accept caller-provided URLs. + pub source: ReleaseSource, + /// Explicit host preparation or ordinary software-only replacement. + pub preparation: Preparation, /// Write the release and units without starting systemd. pub install_only: bool, - /// Existing operator to add to the Ployz service group while preparing the host. - pub group_user: Option, - /// Local release directory used by offline qualification. - pub release_dir: Option, } /// What installation directly observed after activation. @@ -57,8 +70,6 @@ pub enum Readiness { pub struct InstallOutcome { /// Exact release selected before host mutation. pub target: String, - /// Whether the daemon executable changed. - pub replaced: bool, /// Readiness observed before the command returned. pub readiness: Readiness, } @@ -70,10 +81,6 @@ pub enum Error { NotRoot, #[error("another Ployz installation is active")] Busy, - #[error( - "software-only replacement cannot prepare {storage} storage; run without --software-only" - )] - StorageRequiresHostPreparation { storage: StorageChoice }, #[error("nightly is not a supported release channel")] Nightly, #[error("invalid Ployz release version '{value}'; expected X.Y.Z or X.Y.Z-beta.N")] @@ -119,12 +126,26 @@ impl InstallPaths { modprobe_dir: PathBuf::from("/etc/modprobe.d"), } } + + #[cfg(test)] + fn at(root: &Path) -> Self { + Self { + bin_dir: root.join("bin"), + systemd_dir: root.join("systemd"), + data_dir: root.join("data"), + run_dir: root.join("run"), + docker_config: root.join("docker/daemon.json"), + modprobe_dir: root.join("modprobe"), + } + } } struct InstallLock(File); impl Drop for InstallLock { fn drop(&mut self) { + // Forked children can inherit this file descriptor before exec; unlock the shared lock + // explicitly instead of waiting for every inherited descriptor to close. let _ = self.0.unlock(); } } @@ -140,51 +161,49 @@ pub async fn install(request: InstallRequest) -> Result { } async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result { - if !request.prepare_host && request.storage != StorageChoice::None { - return Err(Error::StorageRequiresHostPreparation { - storage: request.storage, - }); - } require_root()?; let _lock = claim_lock(&paths.run_dir)?; verify_system(request.install_only)?; - let target = resolve_release(&request).await?; - - if request.prepare_host { - prepare_storage(request.storage, &paths)?; - install_prerequisites()?; - let inherited_group = sudo_user(); - create_user_and_directories( - request.group_user.as_deref().or(inherited_group.as_deref()), - &paths, - )?; - } else { - verify_software_prerequisites(&paths)?; + let target = resolve_release(&request.release, &request.source).await?; + + match &request.preparation { + Preparation::SoftwareOnly => verify_software_prerequisites(&paths)?, + Preparation::PrepareHost { + storage, + group_user, + } => { + prepare_storage(*storage, &paths)?; + install_prerequisites()?; + let inherited_group = sudo_user(); + create_user_and_directories( + group_user.as_deref().or(inherited_group.as_deref()), + &paths, + )?; + } } - let mut replaced = !paths.systemd_dir.join("ployz.service").is_file(); - replaced |= install_binaries(&request, &paths, &target).await?; + let mut restart_required = !paths.systemd_dir.join("ployz.service").is_file(); + restart_required |= install_binaries(&request.source, &paths, &target).await?; install_systemd(&paths, request.install_only)?; - if request.prepare_host { + if matches!(request.preparation, Preparation::PrepareHost { .. }) { install_docker(&paths, request.install_only).await?; } let readiness = if request.install_only { Readiness::InstallationOnly } else { - if replaced { + if restart_required { systemctl("restart daemon", ["restart", "ployz.service"])?; systemctl( "restart volume plugin", ["try-restart", "ployz-volume-plugin.service"], )?; } - verify_running_daemon(&paths, &target)?; + verify_running_daemon(&paths, &target).await?; Readiness::Running }; Ok(InstallOutcome { - target: target.as_string(), - replaced, + target: target.to_string(), readiness, }) } @@ -311,76 +330,275 @@ pub(super) fn run_host( run_command(stage, &mut command) } -pub(super) fn run_apt(stage: &str, args: [&str; N]) -> Result { - let mut command = Command::new("apt-get"); - command.args(["-o", "DPkg::Lock::Timeout=300"]); - command.args(args); - run_command(stage, &mut command) -} - -pub(super) fn run_apt_with_env( +pub(super) fn run_apt( stage: &str, args: [&str; N], + working_directory: Option<&Path>, ) -> Result { let mut command = Command::new("apt-get"); command .args(["-o", "DPkg::Lock::Timeout=300"]) .args(args) .env("DEBIAN_FRONTEND", "noninteractive"); + if let Some(working_directory) = working_directory { + command.current_dir(working_directory); + } run_command(stage, &mut command) } #[cfg(test)] mod tests { - use super::release::{Release, checksum_for}; + use std::{ + env, + ffi::OsString, + io::Write, + os::unix::fs::{PermissionsExt, symlink}, + process::Command, + time::{SystemTime, UNIX_EPOCH}, + }; + + use semver::Version; + use super::*; + #[tokio::test] + async fn installation_interface_contract() { + if let Ok(case) = env::var("PLOYZ_INSTALLER_CONTRACT_CASE") { + run_installation_case(&case).await; + return; + } + + for case in [ + "success", + "missing", + "bad-checksum", + "corrupt", + "rejected-executable", + "hung-executable", + "software-prerequisite", + "busy", + ] { + let fixture = fixture(case); + create_installation_fixture(&fixture, case); + run_contract_child("installation_interface_contract", &fixture, case); + fs::remove_dir_all(fixture).unwrap(); + } + } + #[test] - fn release_versions_accept_published_shapes_and_order_beta_before_stable() { - assert_eq!(Release::parse("v1.2.3").unwrap().as_string(), "1.2.3"); - assert_eq!( - Release::parse("1.2.3-beta.4").unwrap().as_string(), - "1.2.3-beta.4" + fn zfs_candidate_download_contract() { + if env::var_os("PLOYZ_ZFS_CANDIDATE_CONTRACT").is_some() { + super::storage::install_zfs_packages("test-kernel").unwrap(); + return; + } + + let fixture = fixture("zfs-candidate"); + let commands = fixture.join("commands"); + fs::create_dir_all(&commands).unwrap(); + let marker = fixture.join("installed"); + write_script( + &commands.join("apt-cache"), + "[ \"$2\" = linux-main-modules-zfs-test-kernel ]", + ); + write_script( + &commands.join("dpkg-query"), + "if [ -f \"$PLOYZ_ZFS_INSTALLED\" ]; then echo /lib/modules/test-kernel/kernel/zfs.ko; exit 0; fi\nexit 1", ); - assert!(Release::parse("1.2.3-beta.4").unwrap() < Release::parse("1.2.3").unwrap()); + write_script( + &commands.join("apt-get"), + "case \"$*\" in\n *download*) : > module.deb ;;\n *install*) : > \"$PLOYZ_ZFS_INSTALLED\" ;;\nesac", + ); + write_script( + &commands.join("dpkg-deb"), + "echo /lib/modules/test-kernel/kernel/zfs.ko", + ); + run_contract_child_with_environment( + "zfs_candidate_download_contract", + &fixture, + OsString::from("PLOYZ_ZFS_CANDIDATE_CONTRACT"), + OsString::from("1"), + [( + OsString::from("PLOYZ_ZFS_INSTALLED"), + marker.into_os_string(), + )], + ); + fs::remove_dir_all(fixture).unwrap(); } - #[test] - fn release_versions_reject_non_published_shapes() { - for invalid in ["", "v", "1.2", "1.2.3-rc.1", "1.2.3-beta.x", "1.2.3.4"] { - assert!(Release::parse(invalid).is_err(), "{invalid}"); + async fn run_installation_case(case: &str) { + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + let paths = InstallPaths::at(&root); + fs::create_dir_all(&paths.data_dir).unwrap(); + let existing = if case == "success" { + None + } else { + Some(write_existing_daemon(&paths)) + }; + if case == "software-prerequisite" { + fs::remove_file(root.join("commands/dockerd")).unwrap(); + } + + let request = InstallRequest { + release: ReleaseRequest::Exact(Version::parse("1.2.3").unwrap()), + source: ReleaseSource::Local(root.join("release")), + preparation: Preparation::SoftwareOnly, + install_only: true, + }; + let result = if case == "busy" { + let _held = claim_lock(&paths.run_dir).unwrap(); + install_at(request, paths.clone()).await + } else { + install_at(request, paths.clone()).await + }; + + match case { + "success" => { + let outcome = result.unwrap(); + assert_eq!(outcome.target, "1.2.3"); + assert_eq!(outcome.readiness, Readiness::InstallationOnly); + assert!(paths.bin_dir.join("ployzd").is_file()); + assert!(paths.systemd_dir.join("ployz.service").is_file()); + } + "busy" => assert!(matches!(result, Err(Error::Busy))), + "software-prerequisite" => assert!(matches!(result, Err(Error::Command { .. }))), + "missing" | "bad-checksum" | "corrupt" | "rejected-executable" | "hung-executable" => { + assert!(result.is_err(), "{case} artifact was accepted"); + } + other => panic!("unknown contract case {other}"), + } + if let Some(existing) = existing { + assert_eq!(fs::read(paths.bin_dir.join("ployzd")).unwrap(), existing); } } - #[test] - fn checksum_parser_requires_the_requested_daemon_archive() { - let archive = "ployzd_linux_amd64.tar.gz"; - let checksum = "2d20468cbeb9745b56fdf16897f963773baeecf45c4ea2e00a46e8d82da6ee9f"; - assert_eq!( - checksum_for(format!("{checksum} {archive}\n").as_bytes(), archive), - Some(checksum.into()) - ); - assert_eq!(checksum_for(b"bad other.tar.gz\n", archive), None); + fn create_installation_fixture(root: &Path, case: &str) { + let commands = root.join("commands"); + let release = root.join("release"); + let payload = root.join("payload"); + fs::create_dir_all(&commands).unwrap(); + fs::create_dir_all(&release).unwrap(); + fs::create_dir_all(&payload).unwrap(); + write_script(&commands.join("id"), "if [ \"$1\" = -u ]; then echo 0; fi"); + write_script(&commands.join("dockerd"), "exit 0"); + symlink("/usr/bin/tar", commands.join("tar")).unwrap(); + + let daemon = match case { + "rejected-executable" => "case \"$1\" in version) echo 9.9.9 ;; esac", + "hung-executable" => "case \"$1\" in version) sleep 1; echo 1.2.3 ;; esac", + _ => "case \"$1\" in version) echo 1.2.3 ;; esac", + }; + write_script(&payload.join("ployzd"), daemon); + write_script(&payload.join("ployz-uninstall"), "exit 0"); + if case != "missing" { + let archive = release.join(daemon_archive().unwrap()); + let status = Command::new("tar") + .args(["-czf"]) + .arg(&archive) + .args(["-C"]) + .arg(&payload) + .args(["ployzd", "ployz-uninstall"]) + .status() + .unwrap(); + assert!(status.success()); + let checksum = String::from_utf8( + Command::new("sha256sum") + .arg(&archive) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .to_owned(); + fs::write( + release.join("checksums.txt"), + format!("{checksum} {}\n", daemon_archive().unwrap()), + ) + .unwrap(); + if case == "corrupt" { + fs::OpenOptions::new() + .append(true) + .open(archive) + .unwrap() + .write_all(b"corrupt") + .unwrap(); + } + } else { + fs::write( + release.join("checksums.txt"), + format!("{} {}\n", "0".repeat(64), daemon_archive().unwrap()), + ) + .unwrap(); + } + if case == "bad-checksum" { + fs::write(release.join("checksums.txt"), "bad checksum\n").unwrap(); + } } - #[tokio::test] - async fn software_only_rejects_storage_preparation_before_host_mutation() { - let error = install_at( - InstallRequest { - version: "1.2.3".into(), - storage: StorageChoice::Zfs, - prepare_host: false, - install_only: true, - group_user: None, - release_dir: None, - }, - InstallPaths::system(), + fn write_existing_daemon(paths: &InstallPaths) -> Vec { + fs::create_dir_all(&paths.bin_dir).unwrap(); + let existing = b"#!/bin/sh\n[ \"$1\" = version ] && echo 1.2.2\n".to_vec(); + fs::write(paths.bin_dir.join("ployzd"), &existing).unwrap(); + fs::set_permissions( + paths.bin_dir.join("ployzd"), + fs::Permissions::from_mode(0o755), ) - .await - .unwrap_err(); - assert!(matches!( - error, - Error::StorageRequiresHostPreparation { .. } + .unwrap(); + existing + } + + fn run_contract_child(test: &str, root: &Path, case: &str) { + run_contract_child_with_environment( + test, + root, + OsString::from("PLOYZ_INSTALLER_CONTRACT_CASE"), + OsString::from(case), + [], + ); + } + + fn run_contract_child_with_environment( + test: &str, + root: &Path, + key: OsString, + value: OsString, + extra: [(OsString, OsString); N], + ) { + let mut command = Command::new(env::current_exe().unwrap()); + command + .args(["--exact", test, "--nocapture"]) + .env("PLOYZ_INSTALLER_CONTRACT_ROOT", root) + .env("PATH", root.join("commands")) + .env(key, value); + for (key, value) in extra { + command.env(key, value); + } + let output = command.output().unwrap(); + assert!( + output.status.success(), + "contract child {test} failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + + fn write_script(path: &Path, body: &str) { + fs::write(path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + + fn fixture(name: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = env::temp_dir().join(format!( + "ployzd-installer-{name}-{}-{nonce}", + std::process::id() )); + fs::create_dir(&root).unwrap(); + root } } diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs index f918162212..3cf9b92ce8 100644 --- a/crates/ployzd/src/installer/release.rs +++ b/crates/ployzd/src/installer/release.rs @@ -1,152 +1,150 @@ //! Release selection, acquisition, verification, and activation. use std::{ - cmp::Ordering as Compare, ffi::OsStr, fs::{self, File, OpenOptions}, io::{self, Write}, os::unix::fs::{OpenOptionsExt, PermissionsExt}, path::{Path, PathBuf}, - process::Command, + str::FromStr, sync::atomic::{AtomicU64, Ordering}, + time::Duration, }; +#[cfg(not(test))] +use std::os::unix::fs::chown; + +use semver::Version; +use serde::Deserialize; use sha2::{Digest, Sha256}; +use tokio::{process::Command, time::timeout}; -use super::{Error, InstallPaths, InstallRequest, daemon_archive, run_command}; +use super::{Error, InstallPaths, daemon_archive, run_command}; const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; const CHANNEL_URL: &str = "https://ployz.sh"; const RELEASE_API: &str = "https://api.github.com/repos/getployz/ployz2"; +const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(not(test))] +const VERSION_COMMAND_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(test)] +const VERSION_COMMAND_TIMEOUT: Duration = Duration::from_millis(100); static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); +/// A trusted release target selected at the CLI boundary. #[derive(Clone, Debug, Eq, PartialEq)] -pub(super) struct Release { - major: u64, - minor: u64, - patch: u64, - /// `None` is a stable release, which sorts after its beta releases. - beta: Option, +pub enum ReleaseRequest { + /// Resolve the stable channel once for this installation attempt. + Stable, + /// Resolve the beta channel once for this installation attempt. + Beta, + /// Install this exact published daemon version. + Exact(Version), } -impl Release { - pub(super) fn parse(value: &str) -> Result { +impl FromStr for ReleaseRequest { + type Err = Error; + + fn from_str(value: &str) -> Result { let value = value.strip_prefix('v').unwrap_or(value); - let (core, beta) = match value.split_once("-beta.") { - Some((core, beta)) => (core, Some(beta)), - None => (value, None), - }; - if core.contains('-') { - return Err(Error::InvalidVersion { - value: value.into(), - }); + match value { + "" | "latest" | "stable" => Ok(Self::Stable), + "beta" => Ok(Self::Beta), + "nightly" => Err(Error::Nightly), + value => parse_release(value).map(Self::Exact), } - let mut pieces = core.split('.'); - let parse_part = |part: Option<&str>| { - part.filter(|part| !part.is_empty() && part.bytes().all(|byte| byte.is_ascii_digit())) - .and_then(|part| part.parse().ok()) - }; - let (Some(major), Some(minor), Some(patch), None) = ( - parse_part(pieces.next()), - parse_part(pieces.next()), - parse_part(pieces.next()), - pieces.next(), - ) else { - return Err(Error::InvalidVersion { - value: value.into(), - }); - }; - let beta = match beta { - Some(beta) if beta.bytes().all(|byte| byte.is_ascii_digit()) => { - beta.parse().map(Some).map_err(|_| Error::InvalidVersion { - value: value.into(), - })? + } +} + +/// A release source that is fixed by the local installer invocation. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ReleaseSource { + /// Ployz's fixed, trusted published release and channel endpoints. + Published, + /// An operator-provided local release directory used only by offline qualification. + Local(PathBuf), +} + +impl ReleaseSource { + fn is_local(&self) -> bool { + matches!(self, Self::Local(_)) + } + + async fn channel(&self, name: &str) -> Result { + let bytes = match self { + Self::Published => { + fetch(&format!("{CHANNEL_URL}/{name}"), "resolve release channel").await? } - Some(_) => { - return Err(Error::InvalidVersion { - value: value.into(), - }); + Self::Local(directory) => { + fs::read(directory.join(name)).map_err(|source| Error::Io { + stage: "read local release channel", + source, + })? } - None => None, }; - Ok(Self { - major, - minor, - patch, - beta, + String::from_utf8(bytes).map_err(|error| { + Error::ReleaseSelection(format!("{name} channel is not UTF-8: {error}")) }) } - pub(super) fn as_string(&self) -> String { - let Self { - major, - minor, - patch, - beta, - } = self; - match beta { - Some(beta) => format!("{major}.{minor}.{patch}-beta.{beta}"), - None => format!("{major}.{minor}.{patch}"), + async fn release_file( + &self, + target: &Version, + file: &str, + stage: &'static str, + ) -> Result, Error> { + match self { + Self::Published => fetch(&release_url(target, file), stage).await, + Self::Local(directory) => fs::read(directory.join(file)).map_err(|source| Error::Io { + stage: "read local release artifact", + source, + }), } } } -impl Ord for Release { - fn cmp(&self, other: &Self) -> Compare { - (self.major, self.minor, self.patch) - .cmp(&(other.major, other.minor, other.patch)) - .then_with(|| match (self.beta, other.beta) { - (Some(left), Some(right)) => left.cmp(&right), - (Some(_), None) => Compare::Less, - (None, Some(_)) => Compare::Greater, - (None, None) => Compare::Equal, - }) - } +#[derive(Deserialize)] +struct GitHubRelease { + assets: Vec, } -impl PartialOrd for Release { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } +#[derive(Deserialize)] +struct GitHubAsset { + name: String, + digest: Option, } -pub(super) async fn resolve_release(request: &InstallRequest) -> Result { - let requested = request - .version - .strip_prefix('v') - .unwrap_or(&request.version); - if requested == "nightly" { - return Err(Error::Nightly); - } - match requested { - "" | "latest" | "stable" => { - let value = release_file(request, "stable").await?; - Release::parse(value.trim()) - } - "beta" => { - let value = release_file(request, "beta").await?; - Release::parse(value.trim()) - } - version => Release::parse(version), +pub(super) async fn resolve_release( + request: &ReleaseRequest, + source: &ReleaseSource, +) -> Result { + match request { + ReleaseRequest::Stable => parse_release(source.channel("stable").await?.trim()), + ReleaseRequest::Beta => parse_release(source.channel("beta").await?.trim()), + ReleaseRequest::Exact(version) => Ok(version.clone()), } } -async fn release_file(request: &InstallRequest, name: &str) -> Result { - let bytes = if let Some(directory) = &request.release_dir { - fs::read(directory.join(name)).map_err(|source| Error::Io { - stage: "read local release channel", - source, - })? - } else { - fetch(&format!("{CHANNEL_URL}/{name}"), "resolve release channel").await? - }; - String::from_utf8(bytes) - .map_err(|error| Error::ReleaseSelection(format!("{name} channel is not UTF-8: {error}"))) +fn parse_release(value: &str) -> Result { + let version = Version::parse(value).map_err(|_| Error::InvalidVersion { + value: value.into(), + })?; + let pre = version.pre.as_str(); + let beta_is_valid = pre.strip_prefix("beta.").is_some_and(|number| { + !number.is_empty() && number.bytes().all(|byte| byte.is_ascii_digit()) + }); + if !version.build.is_empty() || (!pre.is_empty() && !beta_is_valid) { + return Err(Error::InvalidVersion { + value: value.into(), + }); + } + Ok(version) } pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { let client = reqwest::Client::builder() .https_only(true) + .timeout(DOWNLOAD_TIMEOUT) .user_agent("ployzd-installer") .build() .map_err(|error| Error::ReleaseSelection(format!("build download client: {error}")))?; @@ -164,38 +162,21 @@ pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}"))) } -fn release_url(target: &Release, file: &str) -> String { - format!( - "{RELEASE_REPOSITORY}/releases/download/v{}/{file}", - target.as_string() - ) +fn release_url(target: &Version, file: &str) -> String { + format!("{RELEASE_REPOSITORY}/releases/download/v{target}/{file}") } pub(super) async fn install_binaries( - request: &InstallRequest, + source: &ReleaseSource, paths: &InstallPaths, - target: &Release, + target: &Version, ) -> Result { - let installed = installed_release(&paths.bin_dir.join("ployzd")); - let pinned = !matches!( - request - .version - .strip_prefix('v') - .unwrap_or(&request.version), - "" | "latest" | "stable" | "beta" - ); - let replace = match installed { - None => true, - Some(ref installed) if request.release_dir.is_some() => true, - Some(ref installed) if pinned => installed != target, - Some(ref installed) => installed < target, - }; + let installed = installed_release(&paths.bin_dir.join("ployzd")).await?; + let replace = replacement_required(source, installed.as_ref(), target); if !replace { println!( "ployzd {} retained", - installed - .expect("replace covers missing release") - .as_string() + installed.expect("a skipped replacement has an installed release") ); return Ok(false); } @@ -203,56 +184,58 @@ pub(super) async fn install_binaries( let archive = daemon_archive()?; let stage = Staging::new(&paths.bin_dir)?; let archive_path = stage.path.join(archive); - let checksum = published_checksum(request, target, archive).await?; - let archive_bytes = if let Some(directory) = &request.release_dir { - fs::read(directory.join(archive)).map_err(|source| Error::Io { - stage: "read local daemon archive", - source, - })? - } else { - fetch(&release_url(target, archive), "download daemon archive").await? - }; + let checksum = published_checksum(source, target, archive).await?; + let archive_bytes = source + .release_file(target, archive, "download daemon archive") + .await?; write_private(&archive_path, &archive_bytes, "stage daemon archive")?; verify_checksum(&archive_path, &checksum)?; extract_archive(&archive_path, &stage.path)?; let daemon = stage.path.join("ployzd"); let uninstall = stage.path.join("ployz-uninstall"); - verify_executable(&daemon, target)?; + verify_executable(&daemon, target).await?; verify_uninstall(&uninstall)?; + sync_staged_files(&daemon, &uninstall, &stage.path)?; activate(&daemon, &uninstall, paths)?; Ok(true) } -pub(super) fn installed_release(path: &Path) -> Option { +fn replacement_required( + source: &ReleaseSource, + installed: Option<&Version>, + target: &Version, +) -> bool { + source.is_local() || installed.is_none_or(|installed| installed != target) +} + +pub(super) async fn installed_release(path: &Path) -> Result, Error> { if !path.is_file() { - return None; + return Ok(None); } - let output = Command::new(path).arg("version").output().ok()?; - output.status.success().then_some(())?; - Release::parse(String::from_utf8_lossy(&output.stdout).trim()).ok() + let output = version_command(path, "inspect installed daemon").await?; + if !output.status.success() { + return Err(Error::Verification(format!( + "installed daemon version check exited with {}", + output.status + ))); + } + parse_release(String::from_utf8_lossy(&output.stdout).trim()) + .map(Some) + .map_err(|_| Error::Verification("installed daemon reported an invalid version".into())) } async fn published_checksum( - request: &InstallRequest, - target: &Release, + source: &ReleaseSource, + target: &Version, archive: &str, ) -> Result { - let checksums = if let Some(directory) = &request.release_dir { - fs::read(directory.join("checksums.txt")).map_err(|source| Error::Io { - stage: "read local release checksums", - source, - })? - } else { - fetch( - &release_url(target, "checksums.txt"), - "download release checksums", - ) - .await? - }; + let checksums = source + .release_file(target, "checksums.txt", "download release checksums") + .await?; if let Some(checksum) = checksum_for(&checksums, archive) { return Ok(checksum); } - if request.release_dir.is_some() { + if source.is_local() { return Err(Error::Verification(format!( "checksums.txt has no hash for {archive}" ))); @@ -274,22 +257,17 @@ fn valid_checksum(value: &str) -> bool { value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) } -async fn github_asset_digest(target: &Release, archive: &str) -> Result { - let endpoint = format!("{RELEASE_API}/releases/tags/v{}", target.as_string()); +async fn github_asset_digest(target: &Version, archive: &str) -> Result { + let endpoint = format!("{RELEASE_API}/releases/tags/v{target}"); let response = fetch(&endpoint, "read published daemon checksum").await?; - let release: serde_json::Value = serde_json::from_slice(&response).map_err(|error| { + let release: GitHubRelease = serde_json::from_slice(&response).map_err(|error| { Error::Verification(format!("published release metadata is invalid: {error}")) })?; let digest = release - .get("assets") - .and_then(serde_json::Value::as_array) - .and_then(|assets| { - assets.iter().find_map(|asset| { - (asset.get("name").and_then(serde_json::Value::as_str) == Some(archive)) - .then(|| asset.get("digest").and_then(serde_json::Value::as_str)) - .flatten() - }) - }) + .assets + .iter() + .find(|asset| asset.name == archive) + .and_then(|asset| asset.digest.as_deref()) .and_then(|digest| digest.strip_prefix("sha256:")) .filter(|digest| valid_checksum(digest)) .ok_or_else(|| { @@ -319,9 +297,11 @@ fn verify_checksum(path: &Path, expected: &str) -> Result<(), Error> { } fn extract_archive(archive: &Path, destination: &Path) -> Result<(), Error> { - let mut command = Command::new("tar"); + let mut command = std::process::Command::new("tar"); command - .args(["-xzf"]) + // Release archives are produced by a different build user. The daemon must not inherit + // that user's numeric ownership when root installs it on a Machine. + .args(["--no-same-owner", "-xzf"]) .arg(archive) .arg("-C") .arg(destination); @@ -329,25 +309,47 @@ fn extract_archive(archive: &Path, destination: &Path) -> Result<(), Error> { Ok(()) } -fn verify_executable(path: &Path, target: &Release) -> Result<(), Error> { +async fn verify_executable(path: &Path, target: &Version) -> Result<(), Error> { fs::set_permissions(path, fs::Permissions::from_mode(0o755)).map_err(|source| Error::Io { stage: "mark staged daemon executable", source, })?; - let mut command = Command::new(path); - command.arg("version"); - let output = run_command("preflight staged daemon", &mut command)?; + let output = version_command(path, "preflight staged daemon").await?; + if !output.status.success() { + return Err(Error::Verification(format!( + "staged daemon version check exited with {}", + output.status + ))); + } let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if observed == target.as_string() { + if observed == target.to_string() { Ok(()) } else { Err(Error::Verification(format!( - "staged daemon reported version {observed:?}, expected {}", - target.as_string() + "staged daemon reported version {observed:?}, expected {target}" ))) } } +async fn version_command(path: &Path, stage: &str) -> Result { + let mut command = Command::new(path); + command.arg("version").kill_on_drop(true); + match timeout(VERSION_COMMAND_TIMEOUT, command.output()).await { + Ok(Ok(output)) => Ok(output), + Ok(Err(error)) => Err(Error::Command { + stage: stage.into(), + message: error.to_string(), + }), + Err(_) => Err(Error::Command { + stage: stage.into(), + message: format!( + "timed out after {} seconds", + VERSION_COMMAND_TIMEOUT.as_secs() + ), + }), + } +} + fn verify_uninstall(path: &Path) -> Result<(), Error> { let metadata = path.metadata().map_err(|source| Error::Io { stage: "preflight staged uninstall command", @@ -365,6 +367,23 @@ fn verify_uninstall(path: &Path) -> Result<(), Error> { } } +fn sync_staged_files(daemon: &Path, uninstall: &Path, staging: &Path) -> Result<(), Error> { + for path in [daemon, uninstall] { + File::open(path) + .and_then(|file| file.sync_all()) + .map_err(|source| Error::Io { + stage: "persist verified staged release", + source, + })?; + } + File::open(staging) + .and_then(|directory| directory.sync_all()) + .map_err(|source| Error::Io { + stage: "persist verified staging directory", + source, + }) +} + pub(super) struct Staging { pub(super) path: PathBuf, } @@ -424,6 +443,10 @@ pub(super) fn write_private(path: &Path, bytes: &[u8], stage: &'static str) -> R fn activate(daemon: &Path, uninstall: &Path, paths: &InstallPaths) -> Result<(), Error> { let installed = paths.bin_dir.join("ployzd"); let previous = paths.bin_dir.join("ployzd.previous"); + let directory = File::open(&paths.bin_dir).map_err(|source| Error::Io { + stage: "open daemon installation directory", + source, + })?; match fs::remove_file(&previous) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::NotFound => {} @@ -439,19 +462,86 @@ fn activate(daemon: &Path, uninstall: &Path, paths: &InstallPaths) -> Result<(), stage: "retain previous daemon release", source, })?; + root_ownership(&previous, "secure retained previous daemon release")?; } + directory.sync_all().map_err(|source| Error::Io { + stage: "persist previous daemon release", + source, + })?; fs::rename(daemon, &installed).map_err(|source| Error::Io { stage: "activate verified daemon", source, })?; - fs::rename(uninstall, paths.bin_dir.join("ployz-uninstall")).map_err(|source| Error::Io { + root_ownership(&installed, "secure activated daemon")?; + directory.sync_all().map_err(|source| Error::Io { + stage: "persist daemon activation", + source, + })?; + let installed_uninstall = paths.bin_dir.join("ployz-uninstall"); + fs::rename(uninstall, &installed_uninstall).map_err(|source| Error::Io { stage: "activate uninstall command", source, })?; - File::open(&paths.bin_dir) - .and_then(|directory| directory.sync_all()) - .map_err(|source| Error::Io { - stage: "persist daemon activation", - source, - }) + root_ownership(&installed_uninstall, "secure activated uninstall command")?; + directory.sync_all().map_err(|source| Error::Io { + stage: "persist uninstall command activation", + source, + }) +} + +fn root_ownership(path: &Path, stage: &'static str) -> Result<(), Error> { + #[cfg(not(test))] + chown(path, Some(0), Some(0)).map_err(|source| Error::Io { stage, source })?; + #[cfg(test)] + let _ = (path, stage); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn published_release_requests_accept_only_trusted_shapes() { + assert!(matches!("stable".parse(), Ok(ReleaseRequest::Stable))); + assert!(matches!("beta".parse(), Ok(ReleaseRequest::Beta))); + assert_eq!( + "v1.2.3-beta.4".parse::().unwrap(), + ReleaseRequest::Exact(Version::parse("1.2.3-beta.4").unwrap()) + ); + for invalid in ["1.2", "1.2.3-rc.1", "1.2.3-beta.x", "1.2.3+build.1"] { + assert!(invalid.parse::().is_err(), "{invalid}"); + } + } + + #[test] + fn checksum_parser_requires_the_requested_daemon_archive() { + let archive = "ployzd_linux_amd64.tar.gz"; + let checksum = "2d20468cbeb9745b56fdf16897f963773baeecf45c4ea2e00a46e8d82da6ee9f"; + assert_eq!( + checksum_for(format!("{checksum} {archive}\n").as_bytes(), archive), + Some(checksum.into()) + ); + assert_eq!(checksum_for(b"bad other.tar.gz\n", archive), None); + } + + #[test] + fn resolved_channels_replace_the_exact_target_and_skip_only_the_same_target() { + let target = Version::parse("1.2.3").unwrap(); + assert!(!replacement_required( + &ReleaseSource::Published, + Some(&target), + &target + )); + assert!(replacement_required( + &ReleaseSource::Published, + Some(&Version::parse("1.2.4").unwrap()), + &target + )); + assert!(replacement_required( + &ReleaseSource::Local(PathBuf::from("qualification")), + Some(&target), + &target + )); + } } diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs index 5842ce64bd..3548405136 100644 --- a/crates/ployzd/src/installer/storage.rs +++ b/crates/ployzd/src/installer/storage.rs @@ -6,21 +6,17 @@ use std::{ io::Write, path::Path, process::Command, - sync::atomic::{AtomicU64, Ordering}, }; use ployz_core::StorageChoice; -use super::{ - Error, InstallPaths, command_exists, run_apt, run_apt_with_env, run_command, run_host, -}; +use super::{Error, InstallPaths, command_exists, run_apt, run_command, run_host}; use super::{ host::write_file_atomically, release::{Staging, write_private}, }; const ZFS_SMOKE_BYTES: u64 = 128 * 1024 * 1024; -static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); pub(super) fn prepare_storage(storage: StorageChoice, paths: &InstallPaths) -> Result<(), Error> { match storage { @@ -195,9 +191,24 @@ fn package_has_zfs_module(files: &str, kernel: &str) -> bool { }) } -fn install_zfs_packages(kernel: &str) -> Result<(), Error> { - run_apt("refresh Ubuntu packages for ZFS", ["update", "-qq"])?; +fn installed_package_files(candidate: &str) -> Result, Error> { + let output = Command::new("dpkg-query") + .args(["-L", candidate]) + .output() + .map_err(|source| Error::Io { + stage: "inspect installed Ubuntu ZFS module package", + source, + })?; + Ok(output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).into_owned())) +} + +pub(super) fn install_zfs_packages(kernel: &str) -> Result<(), Error> { + run_apt("refresh Ubuntu packages for ZFS", ["update", "-qq"], None)?; let mut package = None; + let mut download_error = None; for candidate in [ format!("linux-main-modules-zfs-{kernel}"), format!("linux-modules-zfs-{kernel}"), @@ -216,21 +227,19 @@ fn install_zfs_packages(kernel: &str) -> Result<(), Error> { { continue; } - let files = command_stdout( - "inspect installed Ubuntu ZFS module package", - "dpkg-query", - ["-L", &candidate], - )?; - if package_has_zfs_module(&files, kernel) { + if installed_package_files(&candidate)? + .is_some_and(|files| package_has_zfs_module(&files, kernel)) + { package = Some(candidate); break; } let scratch = Staging::new(Path::new("/var/tmp"))?; - let mut download = Command::new("apt-get"); - download - .args(["-o", "DPkg::Lock::Timeout=300", "download", &candidate]) - .current_dir(&scratch.path); - if !run_command("download Ubuntu ZFS module package", &mut download).is_ok() { + if let Err(error) = run_apt( + "download Ubuntu ZFS module package", + ["download", &candidate], + Some(&scratch.path), + ) { + download_error = Some(error); continue; } let archive = fs::read_dir(&scratch.path) @@ -241,7 +250,11 @@ fn install_zfs_packages(kernel: &str) -> Result<(), Error> { .filter_map(Result::ok) .map(|entry| entry.path()) .find(|path| path.extension() == Some(OsStr::new("deb"))); - let Some(archive) = archive else { continue }; + let Some(archive) = archive else { + return Err(Error::Verification(format!( + "downloaded Ubuntu ZFS module package {candidate} contained no Debian archive" + ))); + }; let mut contents = Command::new("dpkg-deb"); contents.args(["-c"]).arg(&archive); let output = run_command( @@ -253,13 +266,15 @@ fn install_zfs_packages(kernel: &str) -> Result<(), Error> { break; } } - let package = package.ok_or_else(|| Error::Command { - stage: "prepare ZFS storage".into(), - message: format!( - "Ubuntu has no packaged ZFS module for the running kernel {kernel}; install a supported Ubuntu kernel and retry" - ), + let package = package.ok_or_else(|| { + download_error.unwrap_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Ubuntu has no packaged ZFS module for the running kernel {kernel}; install a supported Ubuntu kernel and retry" + ), + }) })?; - run_apt_with_env( + run_apt( "install ZFS packages", [ "install", @@ -269,6 +284,7 @@ fn install_zfs_packages(kernel: &str) -> Result<(), Error> { "zfsutils-linux", &package, ], + None, )?; let files = command_stdout( "verify installed Ubuntu ZFS module package", @@ -328,11 +344,9 @@ fn validate_zfs() -> Result<(), Error> { let stage = Staging::new(Path::new("/var/tmp"))?; let backing = stage.path.join("backing"); write_private(&backing, b"", "create ZFS smoke backing file")?; - let pool = format!( - "ployz-smoke-{}-{}", - std::process::id(), - STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed) - ); + // The installer lock serializes smoke Pools, and the process ID is the shell installer's + // existing collision boundary. + let pool = format!("ployz-smoke-{}", std::process::id()); let mut pool_created = false; let result = (|| { let mut allocate = Command::new("fallocate"); diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index 2635c9a73e..3336c550ad 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -16,7 +16,7 @@ use ployz_core::{DOCKER_NETWORK_CONFLICT_EXIT_STATUS, StorageChoice}; use ployzd::{ daemon::{ContainerMode, Daemon, DaemonConfig, Error, wait_until_socket_accepts}, diag, - installer::{InstallRequest, Readiness}, + installer::{InstallRequest, Preparation, Readiness, ReleaseRequest, ReleaseSource}, machine::DEFAULT_DATA_DIR, network::NetworkError, }; @@ -128,16 +128,18 @@ async fn run(args: Args) -> Result<(), Error> { release_dir, }) = args.command { - let outcome = ployzd::installer::install(InstallRequest { + let request = install_request( version, storage, - prepare_host: !software_only, + software_only, install_only, group_user, release_dir, - }) - .await - .map_err(io::Error::other)?; + ) + .map_err(Error::from)?; + let outcome = ployzd::installer::install(request) + .await + .map_err(io::Error::other)?; match outcome.readiness { Readiness::InstallationOnly => { println!( @@ -169,6 +171,45 @@ async fn run(args: Args) -> Result<(), Error> { daemon.wait().await } +fn install_request( + version: String, + storage: StorageChoice, + software_only: bool, + install_only: bool, + group_user: Option, + release_dir: Option, +) -> io::Result { + let release = version + .parse::() + .map_err(io::Error::other)?; + let preparation = if software_only { + if storage != StorageChoice::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--software-only cannot be combined with --storage", + )); + } + if group_user.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--software-only cannot be combined with --group-user", + )); + } + Preparation::SoftwareOnly + } else { + Preparation::PrepareHost { + storage, + group_user, + } + }; + Ok(InstallRequest { + release, + source: release_dir.map_or(ReleaseSource::Published, ReleaseSource::Local), + preparation, + install_only, + }) +} + async fn dial_stdio(path: &Path) -> io::Result<()> { let stream = wait_until_socket_accepts(path, DIAL_STDIO_SOCKET_TIMEOUT).await?; let (mut socket_read, mut socket_write) = stream.into_split(); @@ -205,4 +246,54 @@ mod tests { ExitCode::FAILURE ); } + + #[test] + fn install_cli_rejects_host_options_for_software_only_replacement() { + let storage = install_request("stable".into(), StorageChoice::Zfs, true, false, None, None) + .unwrap_err(); + assert_eq!(storage.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + storage.to_string(), + "--software-only cannot be combined with --storage" + ); + + let group = install_request( + "stable".into(), + StorageChoice::None, + true, + false, + Some("operator".into()), + None, + ) + .unwrap_err(); + assert_eq!(group.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + group.to_string(), + "--software-only cannot be combined with --group-user" + ); + } + + #[test] + fn install_cli_builds_one_explicit_preparation_mode() { + let replacement = + install_request("1.2.3".into(), StorageChoice::None, true, true, None, None).unwrap(); + assert!(matches!(replacement.preparation, Preparation::SoftwareOnly)); + + let host = install_request( + "1.2.3".into(), + StorageChoice::Zfs, + false, + true, + Some("operator".into()), + None, + ) + .unwrap(); + assert!(matches!( + host.preparation, + Preparation::PrepareHost { + storage: StorageChoice::Zfs, + group_user: Some(_) + } + )); + } } diff --git a/evidence/product-paths.tsv b/evidence/product-paths.tsv index 0215494ea0..9906d23fc1 100644 --- a/evidence/product-paths.tsv +++ b/evidence/product-paths.tsv @@ -1,2 +1,2 @@ path rung1 rung2 rung3 rung4 rung5 -machine-installer crates/ployzd/src/installer.rs::tests - - - ployzd install with a verified musl release on the disposable systemd Machine +machine-installer crates/ployzd/src/installer/mod.rs::tests::{installation_interface_contract,zfs_candidate_download_contract}; crates/ployzd/src/installer/release.rs::tests - - - ployzd install --software-only with a foreign-owned freshly built musl release on the retained Ubuntu systemd Machine From 91b11a8efdaa8e932aecb4b6c72bea1b58d4ebab Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 00:58:59 +0000 Subject: [PATCH 04/30] test(ployzd): execute installer contracts --- crates/ployzd/src/installer/mod.rs | 52 +++++++++++++++++++++----- crates/ployzd/src/installer/release.rs | 5 +-- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index f1ff3056f5..f5cd250156 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -365,11 +365,14 @@ mod tests { async fn installation_interface_contract() { if let Ok(case) = env::var("PLOYZ_INSTALLER_CONTRACT_CASE") { run_installation_case(&case).await; + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + fs::write(root.join("child-completed"), format!("installation:{case}")).unwrap(); return; } for case in [ "success", + "same-target", "missing", "bad-checksum", "corrupt", @@ -389,6 +392,8 @@ mod tests { fn zfs_candidate_download_contract() { if env::var_os("PLOYZ_ZFS_CANDIDATE_CONTRACT").is_some() { super::storage::install_zfs_packages("test-kernel").unwrap(); + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + fs::write(root.join("child-completed"), "zfs-candidate").unwrap(); return; } @@ -417,11 +422,13 @@ mod tests { &fixture, OsString::from("PLOYZ_ZFS_CANDIDATE_CONTRACT"), OsString::from("1"), + "zfs-candidate", [( OsString::from("PLOYZ_ZFS_INSTALLED"), marker.into_os_string(), )], ); + assert!(fixture.join("installed").is_file()); fs::remove_dir_all(fixture).unwrap(); } @@ -429,10 +436,10 @@ mod tests { let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); let paths = InstallPaths::at(&root); fs::create_dir_all(&paths.data_dir).unwrap(); - let existing = if case == "success" { - None - } else { - Some(write_existing_daemon(&paths)) + let existing = match case { + "success" => None, + "same-target" => Some(write_existing_daemon(&paths, "1.2.3")), + _ => Some(write_existing_daemon(&paths, "1.2.2")), }; if case == "software-prerequisite" { fs::remove_file(root.join("commands/dockerd")).unwrap(); @@ -459,14 +466,31 @@ mod tests { assert!(paths.bin_dir.join("ployzd").is_file()); assert!(paths.systemd_dir.join("ployz.service").is_file()); } + "same-target" => { + let outcome = result.unwrap(); + assert_eq!(outcome.target, "1.2.3"); + let existing = existing.as_ref().unwrap(); + assert_ne!(fs::read(paths.bin_dir.join("ployzd")).unwrap(), *existing); + assert_eq!( + fs::read(paths.bin_dir.join("ployzd.previous")).unwrap(), + *existing + ); + } "busy" => assert!(matches!(result, Err(Error::Busy))), "software-prerequisite" => assert!(matches!(result, Err(Error::Command { .. }))), - "missing" | "bad-checksum" | "corrupt" | "rejected-executable" | "hung-executable" => { + "hung-executable" => { + assert!(matches!( + result, + Err(Error::Command { stage, message }) + if stage == "preflight staged daemon" && message == "timed out after 100ms" + )); + } + "missing" | "bad-checksum" | "corrupt" | "rejected-executable" => { assert!(result.is_err(), "{case} artifact was accepted"); } other => panic!("unknown contract case {other}"), } - if let Some(existing) = existing { + if let Some(existing) = existing.filter(|_| case != "same-target") { assert_eq!(fs::read(paths.bin_dir.join("ployzd")).unwrap(), existing); } } @@ -481,6 +505,8 @@ mod tests { write_script(&commands.join("id"), "if [ \"$1\" = -u ]; then echo 0; fi"); write_script(&commands.join("dockerd"), "exit 0"); symlink("/usr/bin/tar", commands.join("tar")).unwrap(); + symlink("/usr/bin/gzip", commands.join("gzip")).unwrap(); + symlink("/usr/bin/sleep", commands.join("sleep")).unwrap(); let daemon = match case { "rejected-executable" => "case \"$1\" in version) echo 9.9.9 ;; esac", @@ -537,9 +563,9 @@ mod tests { } } - fn write_existing_daemon(paths: &InstallPaths) -> Vec { + fn write_existing_daemon(paths: &InstallPaths, version: &str) -> Vec { fs::create_dir_all(&paths.bin_dir).unwrap(); - let existing = b"#!/bin/sh\n[ \"$1\" = version ] && echo 1.2.2\n".to_vec(); + let existing = format!("#!/bin/sh\n[ \"$1\" = version ] && echo {version}\n").into_bytes(); fs::write(paths.bin_dir.join("ployzd"), &existing).unwrap(); fs::set_permissions( paths.bin_dir.join("ployzd"), @@ -555,6 +581,7 @@ mod tests { root, OsString::from("PLOYZ_INSTALLER_CONTRACT_CASE"), OsString::from(case), + &format!("installation:{case}"), [], ); } @@ -564,11 +591,13 @@ mod tests { root: &Path, key: OsString, value: OsString, + completion: &str, extra: [(OsString, OsString); N], ) { + let test = format!("installer::tests::{test}"); let mut command = Command::new(env::current_exe().unwrap()); command - .args(["--exact", test, "--nocapture"]) + .args(["--exact", &test, "--nocapture"]) .env("PLOYZ_INSTALLER_CONTRACT_ROOT", root) .env("PATH", root.join("commands")) .env(key, value); @@ -582,6 +611,11 @@ mod tests { String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); + assert_eq!( + fs::read_to_string(root.join("child-completed")).unwrap(), + completion, + "contract child {test} did not complete its fixture", + ); } fn write_script(path: &Path, body: &str) { diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs index 3cf9b92ce8..71c614ac19 100644 --- a/crates/ployzd/src/installer/release.rs +++ b/crates/ployzd/src/installer/release.rs @@ -342,10 +342,7 @@ async fn version_command(path: &Path, stage: &str) -> Result Err(Error::Command { stage: stage.into(), - message: format!( - "timed out after {} seconds", - VERSION_COMMAND_TIMEOUT.as_secs() - ), + message: format!("timed out after {}ms", VERSION_COMMAND_TIMEOUT.as_millis()), }), } } From aea92ce754f2b26d31c969f225de6a33fe32374c Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 01:42:26 +0000 Subject: [PATCH 05/30] feat(setup): use shared Machine installer --- .github/workflows/release-contracts.yml | 1 - crates/ployz/src/handlers/cloud.rs | 9 +- crates/ployz/src/handlers/machine/init.rs | 5 +- crates/ployz/src/provisioning.rs | 593 ++++++++++++++++++++-- crates/ployz/tests/setup_cli.rs | 268 ++++++++++ docs/RELEASE.md | 2 +- evidence/product-paths.tsv | 1 + scripts/install.sh | 587 --------------------- scripts/qualify-release.sh | 44 +- scripts/test-daemon-lifecycle.sh | 141 +---- scripts/test-qualify-release.sh | 24 +- 11 files changed, 877 insertions(+), 798 deletions(-) create mode 100644 crates/ployz/tests/setup_cli.rs delete mode 100755 scripts/install.sh diff --git a/.github/workflows/release-contracts.yml b/.github/workflows/release-contracts.yml index cf812a8857..70ea93e209 100644 --- a/.github/workflows/release-contracts.yml +++ b/.github/workflows/release-contracts.yml @@ -21,7 +21,6 @@ on: - .github/workflows/release-contracts.yml - .goreleaser.yaml - install.sh - - scripts/install.sh - scripts/uninstall.sh - scripts/verify-release.sh - scripts/pack-release.sh diff --git a/crates/ployz/src/handlers/cloud.rs b/crates/ployz/src/handlers/cloud.rs index af490186bf..15d69d8bac 100644 --- a/crates/ployz/src/handlers/cloud.rs +++ b/crates/ployz/src/handlers/cloud.rs @@ -17,7 +17,7 @@ use crate::context::{ContextError, Transport}; pub(super) fn enroll(root: &ArgMatches) -> Result<(), Error> { enroll_with_installer(root, &|| { - crate::provisioning::provision_local(StorageChoice::None).map_err(Into::into) + crate::provisioning::synchronize_local_daemon().map_err(Into::into) }) } @@ -288,7 +288,7 @@ fn provision_storage(client: &Client, storage: StorageChoice) -> Result<(), Erro client.connection() ))); } - crate::provisioning::provision_local(storage)?; + crate::provisioning::provision_local(env!("CARGO_PKG_VERSION"), storage)?; Ok(()) } @@ -337,7 +337,10 @@ async fn connect_machine(matches: &ArgMatches) -> Result { { Ok(client) => Ok(client), Err(ConnectError::Context(ContextError::NoConfig)) => { - crate::provisioning::provision_local(ployz_core::StorageChoice::None)?; + crate::provisioning::provision_local( + env!("CARGO_PKG_VERSION"), + ployz_core::StorageChoice::None, + )?; wait_client(matches).await } Err(error) => Err(error.into()), diff --git a/crates/ployz/src/handlers/machine/init.rs b/crates/ployz/src/handlers/machine/init.rs index 2cede88c59..344d24ae79 100644 --- a/crates/ployz/src/handlers/machine/init.rs +++ b/crates/ployz/src/handlers/machine/init.rs @@ -51,8 +51,11 @@ pub(in crate::handlers) fn init(root: &ArgMatches) -> Result<(), Error> { let no_install = matches.get_flag("no-install"); let storage = crate::provisioning::resolve_storage(matches)?; if !no_install { + let version = matches + .get_one::("version") + .expect("version has a default"); if local { - crate::provisioning::provision_local(storage)?; + crate::provisioning::provision_local(version, storage)?; } else { crate::provisioning::provision(matches, storage)?; } diff --git a/crates/ployz/src/provisioning.rs b/crates/ployz/src/provisioning.rs index 7f38ff37fd..d93a561172 100644 --- a/crates/ployz/src/provisioning.rs +++ b/crates/ployz/src/provisioning.rs @@ -1,13 +1,23 @@ use std::{ + env, + ffi::OsString, + fs, io::{self, IsTerminal, Write}, - path::PathBuf, - process::{Command, Stdio}, + os::unix::fs::{DirBuilderExt, PermissionsExt}, + path::{Path, PathBuf}, + process::{Command, Output, Stdio}, + thread, + time::{Duration, Instant}, }; -use base64::{Engine, engine::general_purpose::STANDARD}; use clap::ArgMatches; use ployz_core::StorageChoice; +use sha2::{Digest, Sha256}; use thiserror::Error; +use uuid::Uuid; + +const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; +const BOOTSTRAP_VERSION_TIMEOUT: Duration = Duration::from_secs(10); #[derive(Debug, Error)] pub enum ProvisionError { @@ -31,6 +41,35 @@ pub enum ProvisionError { Sudo(#[source] io::Error), #[error("remote user {user} could not authenticate or obtain sudo privileges to install Ployz")] SudoRequired { user: String }, + #[error("inspect remote Machine platform: {0}")] + Platform(#[source] io::Error), + #[error("remote Machine platform inspection failed: {0}")] + PlatformFailed(String), + #[error("remote Machine platform inspection returned non-UTF-8 output")] + PlatformUtf8, + #[error("Ployz Machine must be Linux")] + UnsupportedOs, + #[error("unsupported Machine architecture: {0}")] + UnsupportedArchitecture(String), + #[error("{stage}: {source}")] + BootstrapIo { + stage: &'static str, + #[source] + source: io::Error, + }, + #[error("{stage} exited with {status}")] + BootstrapCommand { + stage: &'static str, + status: std::process::ExitStatus, + }, + #[error("bootstrap verification: {0}")] + BootstrapVerification(String), + #[error("bootstrap acquisition: {0}")] + BootstrapDownload(String), + #[error("transfer bootstrap daemon: {0}")] + Transfer(#[source] io::Error), + #[error("bootstrap daemon transfer exited with {status}")] + TransferFailed { status: std::process::ExitStatus }, #[error("run Ployz installer: {0}")] Install(#[source] io::Error), #[error("Ployz installer exited with {status}")] @@ -89,31 +128,287 @@ fn shell_quote(value: &str) -> String { format!("'{}'", value.replace('\'', "'\"'\"'")) } -fn encoded_installer() -> String { - STANDARD.encode(include_bytes!("../../../scripts/install.sh")) +fn daemon_archive(architecture: &str) -> Result<&'static str, ProvisionError> { + match architecture { + "x86_64" => Ok("ployzd_linux_amd64.tar.gz"), + "aarch64" => Ok("ployzd_linux_arm64.tar.gz"), + architecture => Err(ProvisionError::UnsupportedArchitecture( + architecture.to_owned(), + )), + } } -fn pipefail(pipeline: &str) -> String { - format!("set -o pipefail; {pipeline}") +struct Bootstrap { + directory: PathBuf, + daemon: PathBuf, +} + +impl Bootstrap { + fn new() -> Result { + let directory = env::temp_dir().join(format!("ployz-bootstrap-{}", Uuid::new_v4())); + fs::DirBuilder::new() + .mode(0o700) + .create(&directory) + .map_err(|source| ProvisionError::BootstrapIo { + stage: "create bootstrap directory", + source, + })?; + let daemon = directory.join("ployzd"); + Ok(Self { directory, daemon }) + } } -fn install_command( - script: &str, +impl Drop for Bootstrap { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +fn bootstrap_source() -> Option { + env::var_os("PLOYZ_RELEASE_DIR").map(PathBuf::from) +} + +fn acquire_bootstrap( + architecture: &str, + source: Option<&Path>, +) -> Result { + let archive = daemon_archive(architecture)?; + let bootstrap = Bootstrap::new()?; + let archive_path = bootstrap.directory.join(archive); + let checksums_path = bootstrap.directory.join("checksums.txt"); + if let Some(source) = source { + copy_bootstrap_file( + &source.join(archive), + &archive_path, + "copy bootstrap daemon archive", + )?; + copy_bootstrap_file( + &source.join("checksums.txt"), + &checksums_path, + "copy bootstrap release checksums", + )?; + } else { + let repository = env::var("PLOYZ_GITHUB_URL").unwrap_or_else(|_| RELEASE_REPOSITORY.into()); + let base = format!( + "{repository}/releases/download/v{}", + env!("CARGO_PKG_VERSION") + ); + download_bootstrap( + &format!("{base}/{archive}"), + &archive_path, + "download bootstrap daemon archive", + )?; + download_bootstrap( + &format!("{base}/checksums.txt"), + &checksums_path, + "download bootstrap release checksums", + )?; + } + verify_bootstrap_checksum(&archive_path, &checksums_path, archive)?; + let mut extract = Command::new("tar"); + extract + .arg("-xzf") + .arg(&archive_path) + .arg("-C") + .arg(&bootstrap.directory) + .arg("ployzd"); + bootstrap_status("extract bootstrap daemon", extract.status())?; + fs::set_permissions(&bootstrap.daemon, fs::Permissions::from_mode(0o700)).map_err( + |source| ProvisionError::BootstrapIo { + stage: "mark bootstrap daemon executable", + source, + }, + )?; + Ok(bootstrap) +} + +fn copy_bootstrap_file( + source: &Path, + destination: &Path, + stage: &'static str, +) -> Result<(), ProvisionError> { + fs::copy(source, destination) + .map(|_| ()) + .map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + +fn download_bootstrap( + url: &str, + destination: &Path, + stage: &'static str, +) -> Result<(), ProvisionError> { + let url = url.to_owned(); + let bytes = thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("{stage}: build download runtime: {error}"))?; + runtime.block_on(async { + let client = reqwest::Client::builder() + .https_only(true) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(120)) + .user_agent("ployz-bootstrap") + .build() + .map_err(|error| format!("{stage}: build download client: {error}"))?; + client + .get(url) + .send() + .await + .map_err(|error| format!("{stage}: {error}"))? + .error_for_status() + .map_err(|error| format!("{stage}: {error}"))? + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| format!("{stage}: {error}")) + }) + }) + .join() + .map_err(|_| ProvisionError::BootstrapDownload(format!("{stage}: worker panicked")))? + .map_err(ProvisionError::BootstrapDownload)?; + fs::write(destination, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + +fn bootstrap_status( + stage: &'static str, + result: io::Result, +) -> Result<(), ProvisionError> { + let status = result.map_err(|source| ProvisionError::BootstrapIo { stage, source })?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::BootstrapCommand { stage, status }) + } +} + +fn verify_bootstrap_checksum( + archive: &Path, + checksums: &Path, + archive_name: &str, +) -> Result<(), ProvisionError> { + let checksums = + fs::read_to_string(checksums).map_err(|source| ProvisionError::BootstrapIo { + stage: "read bootstrap release checksums", + source, + })?; + let expected = checksums + .lines() + .find_map(|line| { + let mut fields = line.split_whitespace(); + let hash = fields.next()?; + let name = fields.next()?.trim_start_matches('*'); + (name == archive_name + && hash.len() == 64 + && hash.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then(|| hash.to_ascii_lowercase()) + }) + .ok_or_else(|| { + ProvisionError::BootstrapVerification(format!( + "checksums.txt has no SHA-256 hash for {archive_name}" + )) + })?; + let bytes = fs::read(archive).map_err(|source| ProvisionError::BootstrapIo { + stage: "read bootstrap daemon archive", + source, + })?; + let actual = hex::encode(Sha256::digest(bytes)); + if actual == expected { + Ok(()) + } else { + Err(ProvisionError::BootstrapVerification(format!( + "{archive_name} checksum was {actual}, expected {expected}" + ))) + } +} + +fn command_output_with_timeout(command: &mut Command, timeout: Duration) -> io::Result { + let mut child = command + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let deadline = Instant::now() + timeout; + while Instant::now() < deadline { + if child.try_wait()?.is_some() { + return child.wait_with_output(); + } + thread::sleep(Duration::from_millis(10)); + } + child.kill()?; + let _ = child.wait(); + Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("timed out after {} seconds", timeout.as_secs()), + )) +} + +fn verify_bootstrap_version(command: &mut Command) -> Result<(), ProvisionError> { + let output = + command_output_with_timeout(command, BOOTSTRAP_VERSION_TIMEOUT).map_err(|source| { + ProvisionError::BootstrapIo { + stage: "preflight bootstrap daemon", + source, + } + })?; + if !output.status.success() { + return Err(ProvisionError::BootstrapCommand { + stage: "preflight bootstrap daemon", + status: output.status, + }); + } + let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if observed == env!("CARGO_PKG_VERSION") { + Ok(()) + } else { + Err(ProvisionError::BootstrapVerification(format!( + "bootstrap daemon reported version {observed:?}, expected {}", + env!("CARGO_PKG_VERSION") + ))) + } +} + +enum Preparation<'user> { + Host { + storage: StorageChoice, + group_user: Option<&'user str>, + }, + SoftwareOnly, +} + +fn normalized_version(version: &str) -> &str { + let version = version.strip_prefix('v').unwrap_or(version); + match version { + "" | "latest" => "stable", + version => version, + } +} + +fn install_arguments( version: &str, - group_user: Option<&str>, - via_sudo: bool, - storage: StorageChoice, -) -> String { - let script = shell_quote(script); - let version = shell_quote(version); - let group = group_user - .map(|user| format!("PLOYZ_GROUP_ADD_USER={} ", shell_quote(user))) - .unwrap_or_default(); - let sudo = if via_sudo { "sudo " } else { "" }; - format!( - "printf '%s' {script} | base64 -d | {sudo}{group}PLOYZ_STORAGE={} PLOYZ_VERSION={version} bash", - shell_quote(storage.as_str()) - ) + preparation: Preparation<'_>, + release_dir: Option<&Path>, +) -> Vec { + let mut arguments = vec![ + "install".into(), + "--version".into(), + normalized_version(version).into(), + ]; + match preparation { + Preparation::Host { + storage, + group_user, + } => { + arguments.extend(["--storage".into(), storage.as_str().into()]); + if let Some(user) = group_user { + arguments.extend(["--group-user".into(), user.into()]); + } + } + Preparation::SoftwareOnly => arguments.push("--software-only".into()), + } + if let Some(directory) = release_dir { + arguments.extend(["--release-dir".into(), directory.as_os_str().to_owned()]); + } + arguments } fn installer_status(result: io::Result) -> Result<(), ProvisionError> { @@ -183,6 +478,86 @@ fn ssh_command(matches: &ArgMatches) -> Result<(Command, String), ProvisionError Ok((command, destination)) } +fn scp_command(matches: &ArgMatches) -> Result<(Command, String), ProvisionError> { + let destination = matches + .get_one::("destination") + .ok_or(ProvisionError::MissingDestination)?; + let (destination, port) = ssh_parts(destination)?; + let mut command = Command::new("scp"); + command.arg("-o").arg(format!( + "ConnectTimeout={}", + crate::cli::ssh_timeout(matches).as_secs() + )); + command.args(["-o", "BatchMode=no"]); + command.args(crate::connect::ssh_control_args( + crate::connect::control_path().as_deref(), + )); + command.arg("-i").arg(ssh_key(matches)); + if let Some(port) = port { + command.arg("-P").arg(port); + } + Ok((command, destination)) +} + +fn scp_destination(destination: &str, path: &str) -> String { + let (user, host) = destination + .split_once('@') + .expect("validated SSH destinations include a user"); + if host.contains(':') && !host.starts_with('[') { + format!("{user}@[{host}]:{path}") + } else { + format!("{destination}:{path}") + } +} + +fn remote_platform(matches: &ArgMatches) -> Result { + let (mut platform, destination) = ssh_command(matches)?; + let output = platform + .arg(destination) + .arg("uname -s; uname -m") + .output() + .map_err(ProvisionError::Platform)?; + if !output.status.success() { + return Err(ProvisionError::PlatformFailed( + String::from_utf8_lossy(&output.stderr).trim().to_owned(), + )); + } + let output = String::from_utf8(output.stdout).map_err(|_| ProvisionError::PlatformUtf8)?; + let mut lines = output + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()); + if lines.next() != Some("Linux") { + return Err(ProvisionError::UnsupportedOs); + } + lines + .next() + .map(str::to_owned) + .ok_or_else(|| ProvisionError::PlatformFailed("uname returned no architecture".into())) +} + +fn remote_command(program: &str, arguments: &[OsString], via_sudo: bool) -> String { + let mut command = String::new(); + if via_sudo { + command.push_str("sudo "); + } + command.push_str(&shell_quote(program)); + for argument in arguments { + command.push(' '); + command.push_str(&shell_quote(&argument.to_string_lossy())); + } + command +} + +fn cleanup_remote_bootstrap(matches: &ArgMatches, path: &str) { + if let Ok((mut cleanup, destination)) = ssh_command(matches) { + let _ = cleanup + .arg(destination) + .arg(format!("rm -f -- {}", shell_quote(path))) + .status(); + } +} + pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), ProvisionError> { let (mut whoami, destination) = ssh_command(matches)?; let output = whoami @@ -221,49 +596,116 @@ pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), Pro } } - let encoded = encoded_installer(); + let architecture = remote_platform(matches)?; + let bootstrap = acquire_bootstrap(&architecture, bootstrap_source().as_deref())?; + let remote_bootstrap = format!("/tmp/ployz-bootstrap-{}", Uuid::new_v4()); + let (mut transfer, destination) = scp_command(matches)?; + let status = transfer + .arg(&bootstrap.daemon) + .arg(scp_destination(&destination, &remote_bootstrap)) + .status() + .map_err(ProvisionError::Transfer)?; + if !status.success() { + cleanup_remote_bootstrap(matches, &remote_bootstrap); + return Err(ProvisionError::TransferFailed { status }); + } + + let preflight = (|| { + let (mut verify, destination) = ssh_command(matches)?; + let command = format!( + "chmod 700 {path} && {path} version", + path = shell_quote(&remote_bootstrap) + ); + let output = verify + .arg(destination) + .arg(command) + .output() + .map_err(|source| ProvisionError::BootstrapIo { + stage: "preflight remote bootstrap daemon", + source, + })?; + if !output.status.success() { + return Err(ProvisionError::BootstrapCommand { + stage: "preflight remote bootstrap daemon", + status: output.status, + }); + } + let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if observed != env!("CARGO_PKG_VERSION") { + return Err(ProvisionError::BootstrapVerification(format!( + "remote bootstrap daemon reported version {observed:?}, expected {}", + env!("CARGO_PKG_VERSION") + ))); + } + Ok(()) + })(); + if let Err(error) = preflight { + cleanup_remote_bootstrap(matches, &remote_bootstrap); + return Err(error); + } + let version = matches .get_one::("version") .expect("version has a default"); let via_sudo = user != "root"; - let remote = format!( - "bash -c {}", - shell_quote(&pipefail(&install_command( - &encoded, - version, - via_sudo.then_some(user), - via_sudo, + let arguments = install_arguments( + version, + Preparation::Host { storage, - ))) + group_user: via_sudo.then_some(user), + }, + None, ); + let remote = remote_command(&remote_bootstrap, &arguments, via_sudo); let (mut install, destination) = ssh_command(matches)?; - installer_status(install.arg(destination).arg(remote).status()) + let result = installer_status(install.arg(destination).arg(remote).status()); + cleanup_remote_bootstrap(matches, &remote_bootstrap); + result } -/// Install and start local `ployzd` with the embedded Machine installer. +/// Install and start local `ployzd` through a verified temporary daemon. /// /// # Errors /// /// Returns [`ProvisionError::NotRoot`] when this process is not root. /// Returns [`ProvisionError::Install`] when the installer cannot be spawned, /// or [`ProvisionError::InstallFailed`] when it exits non-zero. -pub fn provision_local(storage: StorageChoice) -> Result<(), ProvisionError> { +pub fn provision_local(version: &str, storage: StorageChoice) -> Result<(), ProvisionError> { + let group_user = env::var("SUDO_USER").ok().filter(|user| !user.is_empty()); + provision_local_with( + version, + Preparation::Host { + storage, + group_user: group_user.as_deref(), + }, + ) +} + +/// Synchronize the local daemon binary without preparing host software or storage. +/// +/// # Errors +/// +/// Returns the first bootstrap acquisition, verification, or installation failure. +pub fn synchronize_local_daemon() -> Result<(), ProvisionError> { + provision_local_with(env!("CARGO_PKG_VERSION"), Preparation::SoftwareOnly) +} + +fn provision_local_with(version: &str, preparation: Preparation<'_>) -> Result<(), ProvisionError> { if !process_is_root() { return Err(ProvisionError::NotRoot); } - let group_user = std::env::var("SUDO_USER") - .ok() - .filter(|user| !user.is_empty()); + if env::consts::OS != "linux" { + return Err(ProvisionError::UnsupportedOs); + } + let bootstrap = acquire_bootstrap(env::consts::ARCH, bootstrap_source().as_deref())?; + let mut version_command = Command::new(&bootstrap.daemon); + version_command.arg("version"); + verify_bootstrap_version(&mut version_command)?; + let release_dir = (normalized_version(version) == env!("CARGO_PKG_VERSION")) + .then_some(bootstrap.directory.as_path()); installer_status( - Command::new("bash") - .arg("-c") - .arg(pipefail(&install_command( - &encoded_installer(), - env!("CARGO_PKG_VERSION"), - group_user.as_deref(), - false, - storage, - ))) + Command::new(&bootstrap.daemon) + .args(install_arguments(version, preparation, release_dir)) .status(), ) } @@ -309,18 +751,59 @@ mod tests { } #[test] - fn embedded_installer_command_preserves_root_sudo_and_local_group() { + fn scp_brackets_bare_ipv6_hosts() { + assert_eq!( + scp_destination("deploy@2001:db8::1", "/tmp/bootstrap"), + "deploy@[2001:db8::1]:/tmp/bootstrap" + ); + assert_eq!( + scp_destination("deploy@[2001:db8::1]", "/tmp/bootstrap"), + "deploy@[2001:db8::1]:/tmp/bootstrap" + ); + } + + #[test] + fn shared_installer_arguments_keep_bootstrap_and_target_distinct() { assert_eq!( - install_command("SCRIPT", "latest", None, false, StorageChoice::None), - "printf '%s' 'SCRIPT' | base64 -d | PLOYZ_STORAGE='none' PLOYZ_VERSION='latest' bash" + install_arguments( + "v1.2.3", + Preparation::Host { + storage: StorageChoice::None, + group_user: None, + }, + None, + ), + ["install", "--version", "1.2.3", "--storage", "none"] + .map(OsString::from) + .to_vec() ); + assert_eq!(normalized_version("latest"), "stable"); assert_eq!( - install_command("SCRIPT", "1.2.3", Some("deploy"), true, StorageChoice::Zfs,), - "printf '%s' 'SCRIPT' | base64 -d | sudo PLOYZ_GROUP_ADD_USER='deploy' PLOYZ_STORAGE='zfs' PLOYZ_VERSION='1.2.3' bash" + install_arguments( + "1.2.3", + Preparation::Host { + storage: StorageChoice::Zfs, + group_user: Some("deploy"), + }, + None, + ), + [ + "install", + "--version", + "1.2.3", + "--storage", + "zfs", + "--group-user", + "deploy", + ] + .map(OsString::from) + .to_vec() ); assert_eq!( - install_command("SCRIPT", "1.2.3", Some("nick"), false, StorageChoice::None,), - "printf '%s' 'SCRIPT' | base64 -d | PLOYZ_GROUP_ADD_USER='nick' PLOYZ_STORAGE='none' PLOYZ_VERSION='1.2.3' bash" + install_arguments("1.2.3", Preparation::SoftwareOnly, None), + ["install", "--version", "1.2.3", "--software-only"] + .map(OsString::from) + .to_vec() ); } @@ -383,7 +866,7 @@ mod tests { return; } assert!(matches!( - provision_local(StorageChoice::None), + provision_local(env!("CARGO_PKG_VERSION"), StorageChoice::None), Err(ProvisionError::NotRoot) )); } diff --git a/crates/ployz/tests/setup_cli.rs b/crates/ployz/tests/setup_cli.rs new file mode 100644 index 0000000000..122a311d40 --- /dev/null +++ b/crates/ployz/tests/setup_cli.rs @@ -0,0 +1,268 @@ +//! CLI setup adapters invoking the shared Machine installer. + +use std::{ + env, fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +struct Fixture(PathBuf); + +impl Fixture { + fn new() -> Self { + let path = env::temp_dir().join(format!("ployz-setup-cli-{}", Uuid::new_v4())); + fs::create_dir(&path).unwrap(); + Self(path) + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn write_executable(path: &Path, body: &str) { + fs::write(path, body).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +fn daemon_archive() -> &'static str { + match env::consts::ARCH { + "x86_64" => "ployzd_linux_amd64.tar.gz", + "aarch64" => "ployzd_linux_arm64.tar.gz", + architecture => panic!("unsupported test architecture {architecture}"), + } +} + +fn release_fixture(root: &Path, log: &Path) -> PathBuf { + let release = root.join("release"); + let payload = root.join("payload"); + fs::create_dir(&release).unwrap(); + fs::create_dir(&payload).unwrap(); + write_executable( + &payload.join("ployzd"), + &format!( + "#!/bin/sh\nif [ \"${{1:-}}\" = version ]; then printf '%s\\n' '{}'; else printf '%s\\n' \"$*\" >> '{}'; exit \"${{PLOYZ_FAKE_INSTALL_STATUS:-0}}\"; fi\n", + env!("CARGO_PKG_VERSION"), + log.display() + ), + ); + let archive = release.join(daemon_archive()); + assert!( + Command::new("tar") + .args(["-czf"]) + .arg(&archive) + .arg("-C") + .arg(&payload) + .arg("ployzd") + .status() + .unwrap() + .success() + ); + let hash = hex::encode(Sha256::digest(fs::read(&archive).unwrap())); + fs::write( + release.join("checksums.txt"), + format!("{hash} {}\n", daemon_archive()), + ) + .unwrap(); + release +} + +fn run(args: &[&str], root: &Path, release: &Path, bin: &Path) -> Output { + let inherited = env::var_os("PATH").unwrap_or_default(); + let path = + env::join_paths(std::iter::once(bin.to_path_buf()).chain(env::split_paths(&inherited))) + .unwrap(); + Command::new(env!("CARGO_BIN_EXE_ployz")) + .args(args) + .env("PATH", path) + .env("PLOYZ_RELEASE_DIR", release) + .env("PLOYZ_CONFIG", root.join("config.yaml")) + .env("PLOYZ_FAKE_INSTALL_STATUS", "17") + .env_remove("SUDO_USER") + .output() + .unwrap() +} + +#[test] +fn local_setup_runs_the_verified_bootstrap_installer() { + let fixture = Fixture::new(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &log); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("id"), + "#!/bin/sh\nif [ \"$1\" = -u ]; then echo 0; else exec /usr/bin/id \"$@\"; fi\n", + ); + + let output = run( + &[ + "machine", + "init", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-local", + ], + fixture.path(), + &release, + &bin, + ); + assert!(!output.status.success(), "fake daemon must not accept RPC"); + let log = fs::read_to_string(log).unwrap(); + assert!( + log.contains(&format!( + "install --version {} --storage none --release-dir", + env!("CARGO_PKG_VERSION") + )), + "{log}" + ); +} + +#[test] +fn corrupt_local_bootstrap_is_rejected_before_execution() { + let fixture = Fixture::new(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &log); + fs::write(release.join(daemon_archive()), b"corrupt").unwrap(); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("id"), + "#!/bin/sh\nif [ \"$1\" = -u ]; then echo 0; else exec /usr/bin/id \"$@\"; fi\n", + ); + + let output = run( + &[ + "machine", + "init", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-corrupt", + ], + fixture.path(), + &release, + &bin, + ); + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("bootstrap verification"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(!log.exists(), "corrupt bootstrap executed"); +} + +#[test] +fn ssh_setup_transfers_one_bootstrap_and_no_install_skips_it() { + let fixture = Fixture::new(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &log); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("ssh"), + &format!( + r#"#!/bin/sh +printf 'ssh %s\n' "$*" >> "$LOG" +case "$*" in + *' whoami') echo deploy ;; + *' sudo true') ;; + *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'chmod 700 '*' version') printf '%s\n' '{}' ;; + *" 'install' "*) ;; + *'rm -f -- '*) ;; + *' true') exit 23 ;; + *) echo "unexpected ssh invocation: $*" >&2; exit 24 ;; +esac +"#, + env::consts::ARCH, + env!("CARGO_PKG_VERSION") + ), + ); + write_executable( + &bin.join("scp"), + "#!/bin/sh\nprintf 'scp %s\\n' \"$*\" >> \"$LOG\"\n", + ); + + let common = [ + "machine", + "init", + "deploy@host", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-ssh", + ]; + let inherited = env::var_os("PATH").unwrap_or_default(); + let path = + env::join_paths(std::iter::once(bin.clone()).chain(env::split_paths(&inherited))).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_ployz")) + .args(common) + .env("PATH", &path) + .env("LOG", &log) + .env("PLOYZ_RELEASE_DIR", &release) + .env("PLOYZ_CONFIG", fixture.path().join("ssh-config.yaml")) + .output() + .unwrap(); + assert!(!output.status.success(), "fake daemon must not accept RPC"); + let setup_log = fs::read_to_string(&log).unwrap(); + assert!(setup_log.contains("scp "), "{setup_log}"); + assert!( + setup_log.contains(&format!( + "'install' '--version' '{}' '--storage' 'none' '--group-user' 'deploy'", + env!("CARGO_PKG_VERSION") + )), + "{setup_log}" + ); + assert!(!setup_log.contains("base64"), "{setup_log}"); + + fs::write(&log, "").unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_ployz")) + .args([ + "machine", + "init", + "deploy@host", + "--no-install", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-no-install", + ]) + .env("PATH", path) + .env("LOG", &log) + .env("PLOYZ_RELEASE_DIR", release) + .env( + "PLOYZ_CONFIG", + fixture.path().join("no-install-config.yaml"), + ) + .output() + .unwrap(); + assert!(!output.status.success(), "fake daemon must not accept RPC"); + let no_install_log = fs::read_to_string(log).unwrap(); + assert!(!no_install_log.contains("scp "), "{no_install_log}"); + assert!(!no_install_log.contains("'install'"), "{no_install_log}"); +} diff --git a/docs/RELEASE.md b/docs/RELEASE.md index bf561f0c85..309df66231 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -81,7 +81,7 @@ Goreleaser does not touch the tap (`--skip=homebrew`); `scripts/promote-release. ## Machine daemon -`ployzd install` on Linux installs or replaces a Machine daemon. It accepts `--version stable`, `--version beta`, or an exact version; use `--software-only` for ordinary replacement after the Machine has already been prepared. The bootstrap script remains available for the first executable during the migration. +`ployzd install` on Linux installs or replaces a Machine daemon. It accepts `--version stable`, `--version beta`, or an exact version; use `--software-only` for ordinary replacement after the Machine has already been prepared. Setup downloads and verifies the CLI release's daemon as a temporary bootstrap, then that daemon installs the selected Machine release through this interface. ## Cloud Relay diff --git a/evidence/product-paths.tsv b/evidence/product-paths.tsv index 9906d23fc1..30f3088ece 100644 --- a/evidence/product-paths.tsv +++ b/evidence/product-paths.tsv @@ -1,2 +1,3 @@ path rung1 rung2 rung3 rung4 rung5 machine-installer crates/ployzd/src/installer/mod.rs::tests::{installation_interface_contract,zfs_candidate_download_contract}; crates/ployzd/src/installer/release.rs::tests - - - ployzd install --software-only with a foreign-owned freshly built musl release on the retained Ubuntu systemd Machine +machine-setup crates/ployz/src/provisioning.rs::tests; scripts/test-{missing-ssh-client,qualify-release}.sh crates/ployz/tests/cloud_init/daemon_sync.rs crates/ployz/tests/setup_cli.rs - scripts/qualify-release.sh with verified musl CLI and daemon archives on real Machines diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100755 index f64cd1589f..0000000000 --- a/scripts/install.sh +++ /dev/null @@ -1,587 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -export LC_ALL="${LC_ALL:-C.UTF-8}" - -INSTALL_ONLY=${INSTALL_ONLY:-false} -INSTALL_BIN_DIR=${INSTALL_BIN_DIR:-/usr/local/bin} -INSTALL_SYSTEMD_DIR=${INSTALL_SYSTEMD_DIR:-/etc/systemd/system} -PLOYZ_GITHUB_URL=${PLOYZ_GITHUB_URL:-https://github.com/getployz/ployz2} -PLOYZ_CHANNEL_URL=${PLOYZ_CHANNEL_URL:-https://ployz.sh} -PLOYZ_VERSION=${PLOYZ_VERSION:-latest} -PLOYZ_VERSION=${PLOYZ_VERSION#v} -PLOYZ_USER=ployz -PLOYZ_GROUP_ADD_USER=${PLOYZ_GROUP_ADD_USER:-} -PLOYZ_DATA_DIR=${PLOYZ_DATA_DIR:-/var/lib/ployz} -PLOYZ_RUN_DIR=${PLOYZ_RUN_DIR:-/run/ployz} -PLOYZ_STORAGE=${PLOYZ_STORAGE:-none} -DOCKER_DAEMON_CONFIG_FILE=${DOCKER_DAEMON_CONFIG_FILE:-/etc/docker/daemon.json} -APT_LOCK_TIMEOUT_SECONDS=300 -PLOYZ_APT_CONFIG= -DAEMON_REPLACED=false -DOCKER_DAEMON_CONFIG='{ - "features": { "containerd-snapshotter": true }, - "live-restore": true, - "log-driver": "json-file", - "log-opts": { "max-size": "10m", "max-file": "3" } -}' - -log() { echo "$1"; } -warning() { echo "WARNING: $1" >&2; } -error() { echo "ERROR: $1" >&2; exit 1; } -command_exists() { command -v "$1" >/dev/null 2>&1; } - -configure_apt_lock_wait() { - local inherited_apt_config=${APT_CONFIG:-} - PLOYZ_APT_CONFIG=$(mktemp) - if [ -n "$inherited_apt_config" ]; then - cat "$inherited_apt_config" > "$PLOYZ_APT_CONFIG" - printf '\n' >> "$PLOYZ_APT_CONFIG" - fi - printf 'DPkg::Lock::Timeout "%s";\n' "$APT_LOCK_TIMEOUT_SECONDS" >> "$PLOYZ_APT_CONFIG" - export APT_CONFIG="$PLOYZ_APT_CONFIG" -} - -run_with_apt_lock_wait() { - local deadline=0 error_dir error_log status tee_pid - while true; do - error_dir=$(mktemp -d) - error_log=$error_dir/stderr - mkfifo "$error_dir/pipe" - tee "$error_log" < "$error_dir/pipe" >&2 & - tee_pid=$! - if LC_ALL=C "$@" 2> "$error_dir/pipe"; then - status=0 - else - status=$? - fi - wait "$tee_pid" - if [ "$status" -eq 0 ]; then - rm -rf "$error_dir" - return 0 - fi - if grep -Eq '^E: Could not get lock .*/(lists|archives)/lock\. It is held by process' "$error_log"; then - [ "$deadline" -ne 0 ] || deadline=$((SECONDS + APT_LOCK_TIMEOUT_SECONDS)) - if [ "$SECONDS" -lt "$deadline" ]; then - rm -rf "$error_dir" - sleep 1 - continue - fi - fi - break - done - if grep -Eq '^E: (Could not get lock|Unable to acquire .* lock)' "$error_log"; then - rm -rf "$error_dir" - error "The package-manager lock named above stayed busy for five minutes. Let its owner finish, then retry the installer." - fi - rm -rf "$error_dir" - return "$status" -} - -channel_version_from_file() { - local version - version=$(tr -d ' \t\r\n' < "$1") - echo "$version" | grep -Eq '^v?[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$' || return 1 - echo "$version" -} - -resolve_install() { - local requested=${1#v} dest resolved name - case "$requested" in - latest | stable | '') name=stable ;; - beta) name=beta ;; - *) - printf '%s\n' "$requested" - return 0 - ;; - esac - dest=$(mktemp) - if ! curl -fsSL -o "$dest" "$PLOYZ_CHANNEL_URL/$name" || ! resolved=$(channel_version_from_file "$dest"); then - rm -f "$dest" - error "$name channel is unavailable" - fi - rm -f "$dest" - printf '%s\n' "${resolved#v}" -} - -daemon_archive() { - case "$1" in - x86_64) echo ployzd_linux_amd64.tar.gz ;; - aarch64) echo ployzd_linux_arm64.tar.gz ;; - *) return 1 ;; - esac -} - -daemon_action() { - local installed=$1 target=$2 mode=$3 - if [ -z "$installed" ]; then - echo replace - elif [ "$mode" = pin ]; then - [ "$installed" = "$target" ] && echo keep || echo replace - elif [ "$installed" = "$target" ]; then - echo keep - else - local newest - newest=$(printf '%s\n%s\n' "${installed//-/\~}" "$target" | sort -V | tail -n1) - [ "$newest" = "$target" ] && echo replace || echo keep - fi -} - -verify_system() { - [ "$(uname -s)" = Linux ] || error "Ployz Machine must be Linux" - daemon_archive "$(uname -m)" >/dev/null || error "Unsupported architecture: $(uname -m)" - if [ ! -d /run/systemd/system ] && [ "$INSTALL_ONLY" != true ]; then - error "Ployz requires systemd" - fi -} - -install_prerequisites() { - command_exists curl && return - if command_exists apt-get; then - run_with_apt_lock_wait apt-get update -qq >/dev/null - run_with_apt_lock_wait env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl ca-certificates - elif command_exists dnf; then dnf install -y curl ca-certificates - elif command_exists yum; then yum install -y curl ca-certificates - elif command_exists pacman; then pacman -Sy --noconfirm curl ca-certificates - elif command_exists zypper; then zypper --non-interactive install curl ca-certificates - else error "curl is required and no supported package manager was found" - fi -} - -operating_system_id() { - local id - [ -r /etc/os-release ] || return 1 - id=$(sed -n 's/^ID=//p' /etc/os-release | head -n1) - id=${id#\"} - id=${id%\"} - [ -n "$id" ] || return 1 - printf '%s\n' "$id" -} - -container_virtualization() { - systemd-detect-virt --container 2>/dev/null || true -} - -lxc_is_unprivileged() { - local inside outside _length - read -r inside outside _length < /proc/self/uid_map || return 1 - [ "$inside" = 0 ] && [ "$outside" != 0 ] -} - -zfs_smoke_bytes() { - echo 134217728 -} - -require_host_root_reserve() { - local allocation=$1 root_size='' root_available='' reserve - read -r root_size root_available < <(df -B1 --output=size,avail / | tail -n1) || \ - error "Could not inspect host-root capacity for ZFS storage preparation" - case "$root_size:$root_available" in - *[!0-9:]* | :* | *:) error "Could not read host-root capacity for ZFS storage preparation" ;; - esac - reserve=$((root_size / 4)) - [ "$reserve" -ge 10737418240 ] || reserve=10737418240 - if [ "$root_available" -lt $((reserve + allocation)) ]; then - error "Host root has ${root_available} bytes available; ZFS validation needs ${allocation} bytes while preserving the ${reserve}-byte host-root reserve" - fi -} - -package_file_list_has_zfs_module() { - local kernel=$1 - grep -F "/lib/modules/$kernel/" | grep -E '/zfs\.ko(\.[^/]*)?$' >/dev/null -} - -ubuntu_zfs_module_package() { - local kernel=$1 package package_dir archive download_failure='' archive_failure='' - for package in \ - "linux-main-modules-zfs-$kernel" \ - "linux-modules-zfs-$kernel" \ - "linux-modules-$kernel" \ - "linux-modules-extra-$kernel"; do - apt-cache show "$package" >/dev/null 2>&1 || continue - if dpkg-query -L "$package" 2>/dev/null | package_file_list_has_zfs_module "$kernel"; then - printf '%s\n' "$package" - return - fi - package_dir=$(mktemp -d) - if (cd "$package_dir" && run_with_apt_lock_wait apt-get download "$package" >/dev/null); then - archive=$(find "$package_dir" -maxdepth 1 -type f -name '*.deb' -print -quit) - if [ -z "$archive" ] || ! dpkg-deb -c "$archive" > "$package_dir/files"; then - archive_failure=$package - elif package_file_list_has_zfs_module "$kernel" < "$package_dir/files"; then - rm -rf "$package_dir" - printf '%s\n' "$package" - return - fi - else - download_failure=$package - fi - rm -rf "$package_dir" - done - if [ -n "$archive_failure" ]; then - error "Could not inspect downloaded running-kernel module package $archive_failure for ZFS support" - fi - if [ -n "$download_failure" ]; then - error "Could not download running-kernel module package $download_failure to inspect it for ZFS support" - fi - error "Ubuntu has no packaged ZFS module for the running kernel $kernel; install a supported Ubuntu kernel and retry" -} - -install_zfs_packages_ubuntu() { - local kernel=$1 module_package - if ! run_with_apt_lock_wait apt-get update -qq >/dev/null; then - error "Could not refresh Ubuntu packages needed for ZFS" - fi - if ! module_package=$(ubuntu_zfs_module_package "$kernel"); then - return 1 - fi - if ! run_with_apt_lock_wait env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends zfsutils-linux "$module_package"; then - error "Could not install zfsutils-linux and running-kernel module package $module_package" - fi - if ! dpkg-query -L "$module_package" | package_file_list_has_zfs_module "$kernel"; then - error "Installed package $module_package does not supply the ZFS module for running kernel $kernel" - fi -} - -zfs_arc_max_for_memory_kib() { - local memory_kib=$1 cap - cap=$((memory_kib * 1024 / 4)) - [ "$cap" -ge 268435456 ] || cap=268435456 - [ "$cap" -le 1073741824 ] || cap=1073741824 - printf '%s\n' "$cap" -} - -zfs_arc_max() { - local memory_kib - memory_kib=$(awk '$1 == "MemTotal:" { print $2; exit }' /proc/meminfo) - case "$memory_kib" in '' | *[!0-9]*) error "Could not read total RAM for the ZFS ARC limit" ;; esac - zfs_arc_max_for_memory_kib "$memory_kib" -} - -persist_zfs_arc_max() { - local cap=$1 config - config=$(mktemp) - printf 'options zfs zfs_arc_max=%s\n' "$cap" > "$config" - if ! install -D -m 0644 "$config" /etc/modprobe.d/ployz-zfs.conf; then - rm -f "$config" - error "Could not persist the ZFS ARC limit in /etc/modprobe.d/ployz-zfs.conf" - fi - rm -f "$config" -} - -set_and_verify_zfs_arc_max() { - local cap=$1 parameter=/sys/module/zfs/parameters/zfs_arc_max observed - [ -w "$parameter" ] || error "Loaded ZFS module does not expose a writable zfs_arc_max parameter" - printf '%s\n' "$cap" > "$parameter" || error "Could not apply zfs_arc_max=$cap to the loaded ZFS module" - observed=$(cat "$parameter") || error "Could not read zfs_arc_max from the loaded ZFS module" - [ "$observed" = "$cap" ] || \ - error "Loaded ZFS module reports zfs_arc_max=$observed instead of the required $cap" -} - -cleanup_zfs_smoke() { - local pool=$1 backing=$2 pool_created=$3 failure=$4 - local pool_names dataset_names mount_sources - local cleanup_failure='' - - if [ "$pool_created" = true ]; then - if ! zpool destroy -f "$pool"; then - cleanup_failure="Could not destroy temporary ZFS smoke Pool $pool; destroy it, then remove $backing" - fi - elif ! pool_names=$(zpool list -H -o name 2>&1); then - cleanup_failure="Could not verify whether temporary ZFS smoke Pool $pool remains; inspect and destroy it, then remove $backing: ${pool_names:-zpool returned no diagnostic output}" - elif printf '%s\n' "$pool_names" | grep -Fxq "$pool" && ! zpool destroy -f "$pool"; then - cleanup_failure="Could not destroy temporary ZFS smoke Pool $pool; destroy it, then remove $backing" - fi - if [ -z "$cleanup_failure" ]; then - if ! pool_names=$(zpool list -H -o name 2>&1); then - cleanup_failure="Could not verify cleanup of temporary ZFS smoke Pool $pool; inspect and destroy it, then remove $backing: ${pool_names:-zpool returned no diagnostic output}" - elif printf '%s\n' "$pool_names" | grep -Fxq "$pool"; then - cleanup_failure="Temporary ZFS smoke Pool $pool remains after destroy; destroy it, then remove $backing" - elif ! dataset_names=$(zfs list -H -o name 2>&1); then - cleanup_failure="Could not verify cleanup of temporary ZFS smoke dataset $pool; inspect and destroy its Pool, then remove $backing: ${dataset_names:-zfs returned no diagnostic output}" - elif printf '%s\n' "$dataset_names" | awk -v pool="$pool" \ - '$0 == pool || index($0, pool "/") == 1 { found = 1 } END { exit !found }'; then - cleanup_failure="Temporary ZFS smoke dataset $pool remains after destroy; destroy its Pool, then remove $backing" - elif ! mount_sources=$(awk ' - { for (i = 1; i <= NF; i++) if ($i == "-" && $(i + 1) == "zfs") print $(i + 2) } - ' /proc/self/mountinfo 2>&1); then - cleanup_failure="Could not verify cleanup of temporary ZFS smoke mounts for $pool; inspect and unmount them, then remove $backing: ${mount_sources:-mount inspection returned no diagnostic output}" - elif printf '%s\n' "$mount_sources" | awk -v pool="$pool" \ - '$0 == pool || index($0, pool "/") == 1 { found = 1 } END { exit !found }'; then - cleanup_failure="Temporary ZFS smoke mount for $pool remains after destroy; unmount it, destroy the Pool, then remove $backing" - elif ! rm -f "$backing"; then - cleanup_failure="Could not remove temporary ZFS smoke backing file $backing" - fi - fi - if [ -z "$cleanup_failure" ] && [ -e "$backing" ]; then - cleanup_failure="Temporary ZFS smoke backing file $backing remains after cleanup" - fi - if [ -n "$cleanup_failure" ]; then - [ -z "$failure" ] || cleanup_failure="$cleanup_failure (after: $failure)" - error "$cleanup_failure" - fi - [ -z "$failure" ] || error "$failure" -} - -restore_zfs_smoke_trap() { - local saved=$1 signal=$2 - if [ -n "$saved" ]; then - eval "$saved" - else - trap - "$signal" - fi -} - -interrupt_zfs_smoke() { - local signal=$1 pool=$2 backing=$3 pool_created=$4 - local previous_int_trap=$5 previous_term_trap=$6 - trap - INT TERM - [ -z "$backing" ] || cleanup_zfs_smoke "$pool" "$backing" "$pool_created" '' - restore_zfs_smoke_trap "$previous_int_trap" INT - restore_zfs_smoke_trap "$previous_term_trap" TERM - kill -s "$signal" "${BASHPID:-$$}" -} - -validate_zfs() { - local bytes backing='' pool blocks block_size allocated mount_target - local previous_int_trap previous_term_trap - local pool_created=false failure='' - bytes=$(zfs_smoke_bytes) - require_host_root_reserve "$bytes" - pool="ployz-smoke-${BASHPID:-$$}-${RANDOM}" - previous_int_trap=$(trap -p INT) - previous_term_trap=$(trap -p TERM) - trap 'interrupt_zfs_smoke INT "$pool" "$backing" "$pool_created" "$previous_int_trap" "$previous_term_trap"' INT - trap 'interrupt_zfs_smoke TERM "$pool" "$backing" "$pool_created" "$previous_int_trap" "$previous_term_trap"' TERM - if ! backing=$(mktemp /var/tmp/ployz-zfs-smoke.XXXXXX); then - error "Could not create a temporary host-root backing file for ZFS validation" - fi - - if ! fallocate -l "$bytes" "$backing"; then - failure="Could not preallocate the non-sparse ZFS smoke backing file $backing" - elif ! read -r blocks block_size < <(stat -c '%b %B' "$backing"); then - failure="Could not verify allocation of ZFS smoke backing file $backing" - else - allocated=$((blocks * block_size)) - [ "$allocated" -ge "$bytes" ] || \ - failure="ZFS smoke backing file $backing is sparse: $allocated of $bytes bytes are allocated" - fi - if [ -z "$failure" ]; then - mount_target=$(findmnt -n -o TARGET -T "$backing" 2>/dev/null || true) - [ "$mount_target" = / ] || \ - failure="ZFS smoke backing file $backing is on $mount_target instead of the host root filesystem" - fi - if [ -z "$failure" ]; then - if zpool create -f -m none -o cachefile=none "$pool" "$backing"; then - pool_created=true - else - failure="Could not create temporary ZFS smoke Pool $pool on $backing" - fi - fi - if [ -z "$failure" ] && ! zpool list -Hp -o name,size,alloc,free "$pool" >/dev/null; then - failure="Could not query temporary ZFS smoke Pool $pool" - fi - if [ -z "$failure" ] && ! zfs list -Hp -o name,mountpoint "$pool" >/dev/null; then - failure="Could not query temporary ZFS smoke dataset $pool" - fi - - cleanup_zfs_smoke "$pool" "$backing" "$pool_created" "$failure" - restore_zfs_smoke_trap "$previous_int_trap" INT - restore_zfs_smoke_trap "$previous_term_trap" TERM -} - -prepare_zfs() { - local os_id container kernel cap - os_id=$(operating_system_id) || error "Could not identify the Linux distribution for ZFS storage preparation" - [ "$os_id" = ubuntu ] || error "ZFS storage preparation is not supported on $os_id yet; use a supported Ubuntu release" - container=$(container_virtualization) - if [ "$container" = openvz ] || { [ -d /proc/vz ] && [ ! -d /proc/bc ]; }; then - error "OpenVZ does not allow this Machine to load the host ZFS kernel module" - fi - if [ "$container" = lxc ] && lxc_is_unprivileged; then - error "Unprivileged LXC does not allow this Machine to load the host ZFS kernel module" - fi - command_exists apt-get || error "Ubuntu apt-get is required for ZFS storage preparation" - kernel=$(uname -r) - require_host_root_reserve "$(zfs_smoke_bytes)" - cap=$(zfs_arc_max) - persist_zfs_arc_max "$cap" - install_zfs_packages_ubuntu "$kernel" - modprobe zfs || error "modprobe zfs failed for running kernel $kernel; verify kernel module support and container privileges" - set_and_verify_zfs_arc_max "$cap" - validate_zfs - log "ZFS storage preparation validated; no Machine Pool was created" -} - -prepare_storage() { - case "$PLOYZ_STORAGE" in - none) return ;; - zfs) prepare_zfs ;; - *) error "Unsupported storage preparation '$PLOYZ_STORAGE'; expected none or zfs" ;; - esac -} - -install_docker() { - if command_exists dockerd; then - if [ "$INSTALL_ONLY" != true ] && ! docker info -f '{{ .DriverStatus }}' 2>/dev/null | grep -q io.containerd.snapshotter; then - warning "Docker is retained unchanged; enable its containerd image store for best results" - fi - return - fi - run_with_apt_lock_wait bash -o pipefail -c 'curl -fsSL https://get.docker.com | sh' - mkdir -p "$(dirname "$DOCKER_DAEMON_CONFIG_FILE")" - printf '%s\n' "$DOCKER_DAEMON_CONFIG" > "$DOCKER_DAEMON_CONFIG_FILE" - [ "$INSTALL_ONLY" = true ] || systemctl restart docker -} - -create_user_and_directories() { - if ! id "$PLOYZ_USER" >/dev/null 2>&1; then - useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin --user-group "$PLOYZ_USER" - fi - if [ -n "$PLOYZ_GROUP_ADD_USER" ]; then - gpasswd --add "$PLOYZ_GROUP_ADD_USER" "$PLOYZ_USER" >/dev/null - fi - install -d -m 0750 -o "$PLOYZ_USER" -g "$PLOYZ_USER" "$PLOYZ_DATA_DIR" "$PLOYZ_RUN_DIR" -} - -install_binaries() { - local archive installed_version target action base_url tmp_dir requested mode - archive=$(daemon_archive "$(uname -m)") || error "Unsupported architecture: $(uname -m)" - requested=$PLOYZ_VERSION - case "$requested" in - latest | stable | beta | '') mode=floating ;; - *) mode=pin ;; - esac - target=$(resolve_install "$requested") - installed_version="" - if [ -x "$INSTALL_BIN_DIR/ployzd" ]; then - installed_version=$("$INSTALL_BIN_DIR/ployzd" version 2>/dev/null || true) - fi - - base_url="$PLOYZ_GITHUB_URL/releases/download/v$target" - - action=$(daemon_action "$installed_version" "$target" "$mode") - if [ -n "${PLOYZ_RELEASE_DIR:-}" ]; then - action=replace - fi - if [ "$action" = keep ]; then - log "ployzd ${installed_version} retained" - return - fi - - tmp_dir=$(mktemp -d) - # shellcheck disable=SC2064 - trap "rm -rf '$tmp_dir'" EXIT - # Daemon downloads currently rely on TLS; published checksums are not verified here. - if [ -n "${PLOYZ_RELEASE_DIR:-}" ]; then - cp "$PLOYZ_RELEASE_DIR/$archive" "$tmp_dir/$archive" || error "PLOYZ_RELEASE_DIR is missing $archive" - else - curl -fsSL -o "$tmp_dir/$archive" "$base_url/$archive" || error "Failed to download $archive" - fi - tar -xzf "$tmp_dir/$archive" -C "$tmp_dir" - install -m 0755 "$tmp_dir/ployzd" "$INSTALL_BIN_DIR/ployzd" - install -m 0755 "$tmp_dir/ployz-uninstall" "$INSTALL_BIN_DIR/ployz-uninstall" - DAEMON_REPLACED=true - # Machines deliberately receive no CLI binary or alias. -} - -install_systemd() { - cat > "$INSTALL_SYSTEMD_DIR/ployz.service" < "$INSTALL_SYSTEMD_DIR/ployz-volume-plugin.socket" < "$INSTALL_SYSTEMD_DIR/ployz-volume-plugin.service" </dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +verify_archive() { + local archive=$1 expected actual + expected=$(awk -v archive="$archive" '$2 == archive || $2 == "*" archive { print $1; exit }' "$ARTIFACT_DIR/checksums.txt") + [ -n "$expected" ] || error "checksums.txt has no hash for $archive" + actual=$(sha256 "$ARTIFACT_DIR/$archive") + [ "$actual" = "$expected" ] || error "$archive checksum was $actual, expected $expected" +} + cli_archive() { case "$(uname -s):$(uname -m)" in Linux:x86_64) echo ployz_linux_amd64.tar.gz ;; @@ -84,6 +100,7 @@ need_archive ployz_linux_arm64.tar.gz need_archive ployzd_linux_amd64.tar.gz need_archive ployzd_linux_arm64.tar.gz need_archive "$(cli_archive)" +need_archive checksums.txt if [ "$DRY_RUN" != 0 ]; then echo "qualify dry-run" @@ -96,7 +113,7 @@ if [ "$DRY_RUN" != 0 ]; then else echo "reset: no (initialized hosts fail without PLOYZ_QUALIFY_RESET=1)" fi - echo "steps: install daemon from PLOYZ_RELEASE_DIR (always replace), machine init --no-install, machine add, deploy named volume qualify-data, volume ls" + echo "steps: verify and run temporary artifact ployzd, install daemon from --release-dir (always replace), machine init --no-install, machine add, deploy named volume qualify-data, volume ls" exit 0 fi @@ -110,17 +127,30 @@ CONFIG_DIR=$(mktemp -d) export PLOYZ_CONFIG=$CONFIG_DIR/config.yaml install_host() { - local host=$1 arch archive remote + local host=$1 arch archive remote bootstrap observed status arch=$(ssh_host "$host" uname -m) archive=$(daemon_archive "$arch") + verify_archive "$archive" + bootstrap=$(mktemp -d "$work/bootstrap.XXXXXX") + tar -xzf "$ARTIFACT_DIR/$archive" -C "$bootstrap" ployzd + [ -x "$bootstrap/ployzd" ] || error "$archive did not contain executable ployzd" remote=/tmp/ployz-qualify-$$ ssh_host "$host" mkdir -p "$remote" # shellcheck disable=SC2086 - scp $SSH_OPTS "$ROOT/scripts/install.sh" "$ARTIFACT_DIR/$archive" "$host:$remote/" - ssh_host "$host" sudo env \ - PLOYZ_RELEASE_DIR="$remote" \ - PLOYZ_VERSION="$version" \ - bash "$remote/install.sh" + if ! scp $SSH_OPTS "$bootstrap/ployzd" "$ARTIFACT_DIR/$archive" "$ARTIFACT_DIR/checksums.txt" "$host:$remote/"; then + ssh_host "$host" rm -rf "$remote" || true + return 1 + fi + if ! observed=$(ssh_host "$host" "$remote/ployzd" version) || [ "$observed" != "$version" ]; then + ssh_host "$host" rm -rf "$remote" || true + error "bootstrap daemon on $host reported ${observed:-no version}, expected $version" + fi + status=0 + ssh_host "$host" sudo "$remote/ployzd" install \ + --version "$version" \ + --release-dir "$remote" || status=$? + ssh_host "$host" rm -rf "$remote" || true + return "$status" } for host in "${HOST_LIST[@]}"; do diff --git a/scripts/test-daemon-lifecycle.sh b/scripts/test-daemon-lifecycle.sh index 4416b4ee71..ca84459f77 100755 --- a/scripts/test-daemon-lifecycle.sh +++ b/scripts/test-daemon-lifecycle.sh @@ -5,62 +5,27 @@ set -euo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT -mkdir -p "$TMP/bin" "$TMP/release" "$TMP/install" "$TMP/systemd" "$TMP/state" "$TMP/run" +mkdir -p "$TMP/bin" "$TMP/install" "$TMP/systemd" "$TMP/state" "$TMP/run" LOG=$TMP/commands.log export LOG : > "$LOG" -cat > "$TMP/ployzd" <<'EOF' -#!/bin/sh -[ "$1" = version ] && echo 1.2.3 -EOF -chmod 0755 "$TMP/ployzd" -tar -czf "$TMP/release/ployzd_linux_amd64.tar.gz" \ - -C "$TMP" ployzd -C "$ROOT/scripts" --transform='s|uninstall.sh|ployz-uninstall|' uninstall.sh -printf 'v1.2.3\n' > "$TMP/release/stable" - -cat > "$TMP/bin/uname" <<'EOF' -#!/bin/sh -case "$1" in -s) echo Linux ;; -m) echo x86_64 ;; esac -EOF -cat > "$TMP/bin/curl" <<'EOF' -#!/bin/bash -while [ "$#" -gt 0 ]; do - if [ "$1" = -o ]; then output=$2; shift 2; else url=$1; shift; fi -done -src="$FAKE_RELEASE/${url##*/}" -[ -f "$src" ] || exit 1 -cp "$src" "$output" -EOF -cat > "$TMP/bin/install" <<'EOF' -#!/bin/bash -args=() -while [ "$#" -gt 0 ]; do - case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac -done -exec /usr/bin/install "${args[@]}" -EOF -cat > "$TMP/bin/id" <<'EOF' -#!/bin/sh -exit 0 -EOF -cat > "$TMP/bin/dockerd" <<'EOF' -#!/bin/sh -exit 0 -EOF cat > "$TMP/bin/docker" <<'EOF' #!/bin/sh echo "docker $*" >> "$LOG" case "$*" in - 'info -f {{ .DriverStatus }}') echo io.containerd.snapshotter ;; 'ps -aq --filter label=ployz.managed') echo managed-container ;; 'ps -aq --filter name=^/ployz-corrosion$') echo corrosion-container ;; 'network ls -q --filter name=^ployz$') echo ployz-network ;; 'rm -f '*) - if grep -Eq '^systemctl stop .*ployz-volume-plugin\.(socket|service)' "$LOG" || [ ! -x "$INSTALL_BIN_DIR/ployzd" ]; then - echo "Docker could not unmount after volume plugin shutdown" >&2 + grep -Eq '^systemctl stop .*ployz-volume-plugin\.(socket|service)' "$LOG" && { + echo "volume plugin stopped before managed containers were removed" >&2 exit 1 - fi + } + [ -x "$INSTALL_BIN_DIR/ployzd" ] || { + echo "daemon removed before managed containers were removed" >&2 + exit 1 + } ;; esac EOF @@ -78,94 +43,14 @@ for command in getent userdel groupdel; do done chmod 0755 "$TMP/bin"/* -run_installer() { - sudo env PATH="$TMP/bin:$PATH" LOG="$LOG" FAKE_RELEASE="$TMP/release" PLOYZ_GITHUB_URL=https://example.invalid \ - PLOYZ_VERSION="$1" INSTALL_BIN_DIR="$TMP/install" INSTALL_SYSTEMD_DIR="$TMP/systemd" \ - PLOYZ_DATA_DIR="$TMP/state" PLOYZ_RUN_DIR="$TMP/run" bash "$ROOT/scripts/install.sh" -} +touch "$TMP/install/ployzd" "$TMP/install/ployz-uninstall" "$TMP/install/ployz-corrosion" +chmod 0755 "$TMP/install/ployzd" "$TMP/install/ployz-uninstall" +touch "$TMP/docker" "$TMP/images" "$TMP/volumes" "$TMP/docker-config" -set_installed_version() { - sudo tee "$TMP/install/ployzd" >/dev/null <&2 - exit 1 - fi -done -grep -Fq 'systemctl enable --now ployz-volume-plugin.socket' "$LOG" -if grep -Fq 'systemctl enable --now ployz-volume-plugin.service' "$LOG"; then - echo "Volume plugin service was enabled instead of socket-activated" >&2 - exit 1 -fi -grep -Fq 'systemctl restart ployz.service' "$LOG" -grep -Fq 'systemctl try-restart ployz-volume-plugin.service' "$LOG" -socket_enable_line=$(grep -nF 'systemctl enable --now ployz-volume-plugin.socket' "$LOG" | head -n 1 | cut -d: -f1) -docker_check_line=$(grep -nF 'docker info -f {{ .DriverStatus }}' "$LOG" | head -n 1 | cut -d: -f1) -if [ "$socket_enable_line" -ge "$docker_check_line" ]; then - echo "Volume plugin socket was activated after Docker installation" >&2 - exit 1 -fi - -: > "$LOG" -run_installer latest -if grep -Fq 'systemctl restart ployz.service' "$LOG"; then - echo "equal latest restarted ployz.service" >&2 - exit 1 -fi - -sudo rm -f "$TMP/systemd/ployz.service" -: > "$LOG" -run_installer latest -grep -Fq 'systemctl restart ployz.service' "$LOG" -[ -f "$TMP/systemd/ployz.service" ] - -: > "$LOG" -set_installed_version 1.2.2 -run_installer latest -grep -Fq 'systemctl restart ployz.service' "$LOG" -grep -Fxq 'RestartPreventExitStatus=78' "$TMP/systemd/ployz.service" -[ -f "$TMP/state/preserved" ] - -: > "$LOG" -set_installed_version 1.2.4 -run_installer latest -if grep -Fq 'systemctl restart ployz.service' "$LOG"; then - echo "newer-than-latest daemon was replaced" >&2 - exit 1 -fi -[ "$("$TMP/install/ployzd" version)" = 1.2.4 ] - -: > "$LOG" -run_installer 1.2.2 -grep -Fq 'systemctl restart ployz.service' "$LOG" -[ "$("$TMP/install/ployzd" version)" = 1.2.3 ] -[ -f "$TMP/state/preserved" ] - -touch "$TMP/docker" "$TMP/images" "$TMP/volumes" "$TMP/docker-config" "$TMP/install/ployz-corrosion" -: > "$LOG" sudo env PATH="$TMP/bin:$PATH" LOG="$LOG" PLOYZ_AUTO_CONFIRM=true INSTALL_BIN_DIR="$TMP/install" \ INSTALL_SYSTEMD_DIR="$TMP/systemd" PLOYZ_DATA_DIR="$TMP/state" PLOYZ_RUN_DIR="$TMP/run" \ bash "$ROOT/scripts/uninstall.sh" + [ ! -e "$TMP/install/ployzd" ] [ ! -e "$TMP/install/ployz-uninstall" ] [ -e "$TMP/install/ployz-corrosion" ] @@ -177,4 +62,4 @@ grep -Fq 'docker rm -f corrosion-container' "$LOG" grep -Fq 'docker network rm ployz-network' "$LOG" grep -Fq 'ip link delete ployz' "$LOG" -echo "daemon replacement and destructive uninstall contracts passed" +echo "destructive daemon uninstall contract passed" diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index c82a9e0bbe..5b26508090 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -23,7 +23,7 @@ fi TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT -mkdir -p "$TMP/bin" "$TMP/installed" +mkdir -p "$TMP/bin" export LOG=$TMP/calls.log cat > "$TMP/ployz" <<'CLI' #!/bin/sh @@ -74,24 +74,15 @@ chmod 0755 "$TMP/ployz" for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployz_macos_amd64.tar.gz ployz_macos_arm64.tar.gz ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do tar -czf "$TMP/$archive" -C "$TMP" ployz done - -# Same-version local archives must still replace the installed payload. -for payload in old new; do - printf '#!/bin/sh\nif [ "$1" = version ]; then echo 1.2.3; else echo %s; fi\n' "$payload" > "$TMP/$payload" - chmod 0755 "$TMP/$payload" -done -cp "$TMP/old" "$TMP/installed/ployzd" -cp "$TMP/new" "$TMP/ployzd" -cp "$TMP/new" "$TMP/ployz-uninstall" +cp "$TMP/ployz" "$TMP/ployzd" +cp "$TMP/ployz" "$TMP/ployz-uninstall" for archive in ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do tar -czf "$TMP/$archive" -C "$TMP" ployzd ployz-uninstall done ( - export PLOYZ_INSTALL_TEST_ONLY=true INSTALL_BIN_DIR="$TMP/installed" PLOYZ_RELEASE_DIR="$TMP" PLOYZ_VERSION=1.2.3 - source "$ROOT/scripts/install.sh" - install_binaries + cd "$TMP" + sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 > checksums.txt ) -[ "$("$TMP/installed/ployzd")" = new ] || fail "same-version local archive was not installed" export SSH_LOG=$TMP/ssh.log cat > "$TMP/bin/ssh" <<'SSH' @@ -107,7 +98,10 @@ while [ "$#" -gt 0 ]; do done [ "$key" = /tmp/qualify-key ] || { echo 'remote command lost SSH identity' >&2; exit 1; } printf '%s identity=%s\n' "${0##*/}" "$key" >> "$SSH_LOG" -case "$*" in *'uname -m') echo x86_64 ;; esac +case "$*" in + *'/ployzd version') echo 1.2.3 ;; + *'uname -m') echo x86_64 ;; +esac SSH chmod 0755 "$TMP/bin/ssh" ln -s ssh "$TMP/bin/scp" From 0da99056c1d5e9bf8a68c0ced8ff7d6c993592eb Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 01:50:31 +0000 Subject: [PATCH 06/30] fix(ployzd): tighten installer host boundaries --- Cargo.lock | 20 +++ crates/ployzd/Cargo.toml | 1 + crates/ployzd/src/installer/host.rs | 28 ++-- crates/ployzd/src/installer/mod.rs | 210 ++++++++++++++++++------- crates/ployzd/src/installer/release.rs | 147 ++++++----------- crates/ployzd/src/installer/storage.rs | 67 +++----- crates/ployzd/src/main.rs | 40 ++++- 7 files changed, 292 insertions(+), 221 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60e65149f5..01489d5da1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,6 +834,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -2275,6 +2281,7 @@ dependencies = [ "serde_json", "sha2", "shell-words", + "tempfile", "thiserror 2.0.20", "tokio", "tokio-stream", @@ -3166,6 +3173,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "termcolor" version = "1.4.1" diff --git a/crates/ployzd/Cargo.toml b/crates/ployzd/Cargo.toml index 2ac76ac3fa..ebc2788664 100644 --- a/crates/ployzd/Cargo.toml +++ b/crates/ployzd/Cargo.toml @@ -38,6 +38,7 @@ semver = "1.0.28" sha2 = "0.11.0" shell-words = "1.1.1" thiserror = "2.0.20" +tempfile = "3.27.0" tokio = { version = "1.53.1", features = ["io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = { version = "0.1.19", features = ["net"] } tokio-util = { version = "0.7.19", features = ["codec", "io"] } diff --git a/crates/ployzd/src/installer/host.rs b/crates/ployzd/src/installer/host.rs index ed29fde22e..2ca869412e 100644 --- a/crates/ployzd/src/installer/host.rs +++ b/crates/ployzd/src/installer/host.rs @@ -201,20 +201,18 @@ pub(super) fn write_file_atomically( atomic_write(path, content.as_bytes(), 0o644).map_err(|source| Error::Io { stage, source }) } -pub(super) async fn install_docker(paths: &InstallPaths, install_only: bool) -> Result<(), Error> { +pub(super) async fn install_docker(paths: &InstallPaths) -> Result<(), Error> { if command_exists("dockerd") { - if !install_only { - let mut command = Command::new("docker"); - command.args(["info", "-f", "{{ .DriverStatus }}"]); - let snapshotter = command.output().ok().is_some_and(|output| { - output.status.success() - && String::from_utf8_lossy(&output.stdout).contains("io.containerd.snapshotter") - }); - if !snapshotter { - eprintln!( - "WARNING: Docker is retained unchanged; enable its containerd image store for best results" - ); - } + let mut command = Command::new("docker"); + command.args(["info", "-f", "{{ .DriverStatus }}"]); + let snapshotter = command.output().ok().is_some_and(|output| { + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("io.containerd.snapshotter") + }); + if !snapshotter { + eprintln!( + "WARNING: Docker is retained unchanged; enable its containerd image store for best results" + ); } return Ok(()); } @@ -262,9 +260,7 @@ pub(super) async fn install_docker(paths: &InstallPaths, install_only: bool) -> DOCKER_DAEMON_CONFIG, "write Docker configuration", )?; - if !install_only { - systemctl("restart Docker", ["restart", "docker"])?; - } + systemctl("restart Docker", ["restart", "docker"])?; Ok(()) } diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index f5cd250156..ff6e7257c4 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -5,14 +5,13 @@ mod release; mod storage; use std::{ - fs::{self, File, OpenOptions}, + fs::{self, File, OpenOptions, TryLockError}, io, os::unix::fs::{OpenOptionsExt, PermissionsExt}, path::{Path, PathBuf}, process::{Command, Output}, }; -use fs2::FileExt; use ployz_core::StorageChoice; use thiserror::Error; @@ -146,7 +145,7 @@ impl Drop for InstallLock { fn drop(&mut self) { // Forked children can inherit this file descriptor before exec; unlock the shared lock // explicitly instead of waiting for every inherited descriptor to close. - let _ = self.0.unlock(); + let _ = File::unlock(&self.0); } } @@ -166,27 +165,29 @@ async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result verify_software_prerequisites(&paths)?, - Preparation::PrepareHost { - storage, - group_user, - } => { - prepare_storage(*storage, &paths)?; - install_prerequisites()?; - let inherited_group = sudo_user(); - create_user_and_directories( - group_user.as_deref().or(inherited_group.as_deref()), - &paths, - )?; + if !request.install_only { + match &request.preparation { + Preparation::SoftwareOnly => verify_software_prerequisites(&paths)?, + Preparation::PrepareHost { + storage, + group_user, + } => { + prepare_storage(*storage, &paths)?; + install_prerequisites()?; + let inherited_group = sudo_user(); + create_user_and_directories( + group_user.as_deref().or(inherited_group.as_deref()), + &paths, + )?; + } } } let mut restart_required = !paths.systemd_dir.join("ployz.service").is_file(); restart_required |= install_binaries(&request.source, &paths, &target).await?; install_systemd(&paths, request.install_only)?; - if matches!(request.preparation, Preparation::PrepareHost { .. }) { - install_docker(&paths, request.install_only).await?; + if !request.install_only && matches!(request.preparation, Preparation::PrepareHost { .. }) { + install_docker(&paths).await?; } let readiness = if request.install_only { @@ -240,16 +241,16 @@ fn claim_lock(run_dir: &Path) -> Result { stage: "open installation lock", source, })?; - file.try_lock_exclusive().map_err(|source| { - if source.kind() == io::ErrorKind::WouldBlock { - Error::Busy - } else { - Error::Io { + match file.try_lock() { + Ok(()) => {} + Err(TryLockError::WouldBlock) => return Err(Error::Busy), + Err(TryLockError::Error(source)) => { + return Err(Error::Io { stage: "lock installation", source, - } + }); } - })?; + } Ok(InstallLock(file)) } @@ -258,6 +259,12 @@ fn verify_system(install_only: bool) -> Result<(), Error> { return Err(Error::UnsupportedOs); } daemon_archive()?; + if !command_exists("tar") { + return Err(Error::Command { + stage: "preflight daemon archive extraction".into(), + message: "tar is required to extract Ployz release archives".into(), + }); + } if !install_only && !Path::new("/run/systemd/system").is_dir() { return Err(Error::SystemdRequired); } @@ -354,10 +361,10 @@ mod tests { io::Write, os::unix::fs::{PermissionsExt, symlink}, process::Command, - time::{SystemTime, UNIX_EPOCH}, }; use semver::Version; + use tempfile::TempDir; use super::*; @@ -372,22 +379,50 @@ mod tests { for case in [ "success", + "install-only", "same-target", "missing", + "missing-tar", "bad-checksum", "corrupt", "rejected-executable", "hung-executable", - "software-prerequisite", "busy", ] { let fixture = fixture(case); - create_installation_fixture(&fixture, case); - run_contract_child("installation_interface_contract", &fixture, case); - fs::remove_dir_all(fixture).unwrap(); + create_installation_fixture(fixture.path(), case); + run_contract_child("installation_interface_contract", fixture.path(), case); } } + #[test] + fn software_only_prerequisite_contract() { + if env::var_os("PLOYZ_SOFTWARE_PREREQUISITE_CONTRACT").is_some() { + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + let paths = InstallPaths::at(&root); + assert!(matches!( + verify_software_prerequisites(&paths), + Err(Error::Command { stage, message }) + if stage == "software-only preflight" + && message + == "Docker is not installed; run ployzd install without --software-only first" + )); + fs::write(root.join("child-completed"), "software-prerequisite").unwrap(); + return; + } + + let fixture = fixture("software-prerequisite"); + fs::create_dir_all(fixture.path().join("commands")).unwrap(); + run_contract_child_with_environment( + "software_only_prerequisite_contract", + fixture.path(), + OsString::from("PLOYZ_SOFTWARE_PREREQUISITE_CONTRACT"), + OsString::from("1"), + "software-prerequisite", + [], + ); + } + #[test] fn zfs_candidate_download_contract() { if env::var_os("PLOYZ_ZFS_CANDIDATE_CONTRACT").is_some() { @@ -398,9 +433,9 @@ mod tests { } let fixture = fixture("zfs-candidate"); - let commands = fixture.join("commands"); + let commands = fixture.path().join("commands"); fs::create_dir_all(&commands).unwrap(); - let marker = fixture.join("installed"); + let marker = fixture.path().join("installed"); write_script( &commands.join("apt-cache"), "[ \"$2\" = linux-main-modules-zfs-test-kernel ]", @@ -419,7 +454,7 @@ mod tests { ); run_contract_child_with_environment( "zfs_candidate_download_contract", - &fixture, + fixture.path(), OsString::from("PLOYZ_ZFS_CANDIDATE_CONTRACT"), OsString::from("1"), "zfs-candidate", @@ -428,27 +463,35 @@ mod tests { marker.into_os_string(), )], ); - assert!(fixture.join("installed").is_file()); - fs::remove_dir_all(fixture).unwrap(); + assert!(fixture.path().join("installed").is_file()); } async fn run_installation_case(case: &str) { let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); let paths = InstallPaths::at(&root); - fs::create_dir_all(&paths.data_dir).unwrap(); let existing = match case { - "success" => None, + "success" | "install-only" => None, "same-target" => Some(write_existing_daemon(&paths, "1.2.3")), _ => Some(write_existing_daemon(&paths, "1.2.2")), }; - if case == "software-prerequisite" { + if case == "install-only" { fs::remove_file(root.join("commands/dockerd")).unwrap(); } + if case == "missing-tar" { + fs::remove_file(root.join("commands/tar")).unwrap(); + } let request = InstallRequest { release: ReleaseRequest::Exact(Version::parse("1.2.3").unwrap()), source: ReleaseSource::Local(root.join("release")), - preparation: Preparation::SoftwareOnly, + preparation: if case == "install-only" { + Preparation::PrepareHost { + storage: StorageChoice::None, + group_user: None, + } + } else { + Preparation::SoftwareOnly + }, install_only: true, }; let result = if case == "busy" { @@ -459,12 +502,16 @@ mod tests { }; match case { - "success" => { + "success" | "install-only" => { let outcome = result.unwrap(); assert_eq!(outcome.target, "1.2.3"); assert_eq!(outcome.readiness, Readiness::InstallationOnly); assert!(paths.bin_dir.join("ployzd").is_file()); assert!(paths.systemd_dir.join("ployz.service").is_file()); + if case == "install-only" { + assert!(!root.join("forbidden-invocation").exists()); + assert!(!paths.data_dir.exists()); + } } "same-target" => { let outcome = result.unwrap(); @@ -477,7 +524,12 @@ mod tests { ); } "busy" => assert!(matches!(result, Err(Error::Busy))), - "software-prerequisite" => assert!(matches!(result, Err(Error::Command { .. }))), + "missing-tar" => assert!(matches!( + result, + Err(Error::Command { stage, message }) + if stage == "preflight daemon archive extraction" + && message == "tar is required to extract Ployz release archives" + )), "hung-executable" => { assert!(matches!( result, @@ -485,7 +537,20 @@ mod tests { if stage == "preflight staged daemon" && message == "timed out after 100ms" )); } - "missing" | "bad-checksum" | "corrupt" | "rejected-executable" => { + "bad-checksum" => assert!(matches!( + result, + Err(Error::Verification(message)) + if message + == format!("checksums.txt has no hash for {}", daemon_archive().unwrap()) + )), + "corrupt" => assert!(matches!( + result, + Err(Error::Verification(message)) + if message.contains(daemon_archive().unwrap()) + && message.contains("checksum was") + && message.contains("expected") + )), + "missing" | "rejected-executable" => { assert!(result.is_err(), "{case} artifact was accepted"); } other => panic!("unknown contract case {other}"), @@ -515,6 +580,27 @@ mod tests { }; write_script(&payload.join("ployzd"), daemon); write_script(&payload.join("ployz-uninstall"), "exit 0"); + if case == "install-only" { + write_script( + &commands.join("id"), + "if [ \"$1\" = -u ]; then echo 0; else echo id >> \"$PLOYZ_INSTALLER_FORBIDDEN\"; exit 97; fi", + ); + for command in [ + "apt-get", + "dnf", + "yum", + "pacman", + "zypper", + "bash", + "docker", + "systemctl", + ] { + write_script( + &commands.join(command), + "echo \"$0\" >> \"$PLOYZ_INSTALLER_FORBIDDEN\"; exit 97", + ); + } + } if case != "missing" { let archive = release.join(daemon_archive().unwrap()); let status = Command::new("tar") @@ -576,14 +662,24 @@ mod tests { } fn run_contract_child(test: &str, root: &Path, case: &str) { - run_contract_child_with_environment( - test, - root, - OsString::from("PLOYZ_INSTALLER_CONTRACT_CASE"), - OsString::from(case), - &format!("installation:{case}"), - [], - ); + let key = OsString::from("PLOYZ_INSTALLER_CONTRACT_CASE"); + let value = OsString::from(case); + let completion = format!("installation:{case}"); + if case == "install-only" { + run_contract_child_with_environment( + test, + root, + key, + value, + &completion, + [( + OsString::from("PLOYZ_INSTALLER_FORBIDDEN"), + root.join("forbidden-invocation").into_os_string(), + )], + ); + } else { + run_contract_child_with_environment(test, root, key, value, &completion, []); + } } fn run_contract_child_with_environment( @@ -623,16 +719,10 @@ mod tests { fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); } - fn fixture(name: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) + fn fixture(name: &str) -> TempDir { + tempfile::Builder::new() + .prefix(&format!("ployzd-installer-{name}-")) + .tempdir() .unwrap() - .as_nanos(); - let root = env::temp_dir().join(format!( - "ployzd-installer-{name}-{}-{nonce}", - std::process::id() - )); - fs::create_dir(&root).unwrap(); - root } } diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs index 71c614ac19..c590bd92ea 100644 --- a/crates/ployzd/src/installer/release.rs +++ b/crates/ployzd/src/installer/release.rs @@ -7,7 +7,6 @@ use std::{ os::unix::fs::{OpenOptionsExt, PermissionsExt}, path::{Path, PathBuf}, str::FromStr, - sync::atomic::{AtomicU64, Ordering}, time::Duration, }; @@ -15,21 +14,19 @@ use std::{ use std::os::unix::fs::chown; use semver::Version; -use serde::Deserialize; use sha2::{Digest, Sha256}; +use tempfile::TempDir; use tokio::{process::Command, time::timeout}; use super::{Error, InstallPaths, daemon_archive, run_command}; const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; const CHANNEL_URL: &str = "https://ployz.sh"; -const RELEASE_API: &str = "https://api.github.com/repos/getployz/ployz2"; const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); #[cfg(not(test))] const VERSION_COMMAND_TIMEOUT: Duration = Duration::from_secs(10); #[cfg(test)] const VERSION_COMMAND_TIMEOUT: Duration = Duration::from_millis(100); -static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); /// A trusted release target selected at the CLI boundary. #[derive(Clone, Debug, Eq, PartialEq)] @@ -103,17 +100,6 @@ impl ReleaseSource { } } -#[derive(Deserialize)] -struct GitHubRelease { - assets: Vec, -} - -#[derive(Deserialize)] -struct GitHubAsset { - name: String, - digest: Option, -} - pub(super) async fn resolve_release( request: &ReleaseRequest, source: &ReleaseSource, @@ -144,7 +130,8 @@ fn parse_release(value: &str) -> Result { pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { let client = reqwest::Client::builder() .https_only(true) - .timeout(DOWNLOAD_TIMEOUT) + .connect_timeout(DOWNLOAD_TIMEOUT) + .read_timeout(DOWNLOAD_TIMEOUT) .user_agent("ployzd-installer") .build() .map_err(|error| Error::ReleaseSelection(format!("build download client: {error}")))?; @@ -182,20 +169,20 @@ pub(super) async fn install_binaries( } let archive = daemon_archive()?; - let stage = Staging::new(&paths.bin_dir)?; - let archive_path = stage.path.join(archive); - let checksum = published_checksum(source, target, archive).await?; + let stage = staging_directory(&paths.bin_dir)?; + let archive_path = stage.path().join(archive); + let checksum = release_checksum(source, target, archive).await?; let archive_bytes = source .release_file(target, archive, "download daemon archive") .await?; + verify_checksum(&archive_bytes, archive, &checksum)?; write_private(&archive_path, &archive_bytes, "stage daemon archive")?; - verify_checksum(&archive_path, &checksum)?; - extract_archive(&archive_path, &stage.path)?; - let daemon = stage.path.join("ployzd"); - let uninstall = stage.path.join("ployz-uninstall"); + extract_archive(&archive_path, stage.path())?; + let daemon = stage.path().join("ployzd"); + let uninstall = stage.path().join("ployz-uninstall"); verify_executable(&daemon, target).await?; verify_uninstall(&uninstall)?; - sync_staged_files(&daemon, &uninstall, &stage.path)?; + sync_staged_files(&daemon, &uninstall, stage.path())?; activate(&daemon, &uninstall, paths)?; Ok(true) } @@ -224,7 +211,7 @@ pub(super) async fn installed_release(path: &Path) -> Result, Er .map_err(|_| Error::Verification("installed daemon reported an invalid version".into())) } -async fn published_checksum( +async fn release_checksum( source: &ReleaseSource, target: &Version, archive: &str, @@ -232,15 +219,8 @@ async fn published_checksum( let checksums = source .release_file(target, "checksums.txt", "download release checksums") .await?; - if let Some(checksum) = checksum_for(&checksums, archive) { - return Ok(checksum); - } - if source.is_local() { - return Err(Error::Verification(format!( - "checksums.txt has no hash for {archive}" - ))); - } - github_asset_digest(target, archive).await + checksum_for(&checksums, archive) + .ok_or_else(|| Error::Verification(format!("checksums.txt has no hash for {archive}"))) } pub(super) fn checksum_for(checksums: &[u8], archive: &str) -> Option { @@ -257,39 +237,15 @@ fn valid_checksum(value: &str) -> bool { value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) } -async fn github_asset_digest(target: &Version, archive: &str) -> Result { - let endpoint = format!("{RELEASE_API}/releases/tags/v{target}"); - let response = fetch(&endpoint, "read published daemon checksum").await?; - let release: GitHubRelease = serde_json::from_slice(&response).map_err(|error| { - Error::Verification(format!("published release metadata is invalid: {error}")) - })?; - let digest = release - .assets - .iter() - .find(|asset| asset.name == archive) - .and_then(|asset| asset.digest.as_deref()) - .and_then(|digest| digest.strip_prefix("sha256:")) - .filter(|digest| valid_checksum(digest)) - .ok_or_else(|| { - Error::Verification(format!( - "published release metadata has no SHA-256 digest for {archive}" - )) - })?; - Ok(digest.to_ascii_lowercase()) -} - -fn verify_checksum(path: &Path, expected: &str) -> Result<(), Error> { - let bytes = fs::read(path).map_err(|source| Error::Io { - stage: "read staged daemon archive", - source, - })?; +fn verify_checksum(bytes: &[u8], archive: &str, expected: &str) -> Result<(), Error> { let actual = hex::encode(Sha256::digest(bytes)); if actual == expected { Ok(()) } else { Err(Error::Verification(format!( "{} checksum was {actual}, expected {expected}", - path.file_name() + Path::new(archive) + .file_name() .and_then(OsStr::to_str) .unwrap_or("daemon archive") ))) @@ -381,48 +337,19 @@ fn sync_staged_files(daemon: &Path, uninstall: &Path, staging: &Path) -> Result< }) } -pub(super) struct Staging { - pub(super) path: PathBuf, -} - -impl Staging { - pub(super) fn new(parent: &Path) -> Result { - fs::create_dir_all(parent).map_err(|source| Error::Io { - stage: "create daemon installation directory", +pub(super) fn staging_directory(parent: &Path) -> Result { + fs::create_dir_all(parent).map_err(|source| Error::Io { + stage: "create daemon installation directory", + source, + })?; + tempfile::Builder::new() + .prefix(".ployz-stage-") + .permissions(fs::Permissions::from_mode(0o700)) + .tempdir_in(parent) + .map_err(|source| Error::Io { + stage: "create private daemon staging directory", source, - })?; - for _ in 0..100 { - let sequence = STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed); - let path = parent.join(format!(".ployz-stage-{}-{sequence}", std::process::id())); - match fs::create_dir(&path) { - Ok(()) => { - fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).map_err( - |source| Error::Io { - stage: "protect daemon staging directory", - source, - }, - )?; - return Ok(Self { path }); - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, - Err(source) => { - return Err(Error::Io { - stage: "create daemon staging directory", - source, - }); - } - } - } - Err(Error::Verification( - "could not create a private daemon staging directory".into(), - )) - } -} - -impl Drop for Staging { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.path); - } + }) } pub(super) fn write_private(path: &Path, bytes: &[u8], stage: &'static str) -> Result<(), Error> { @@ -496,6 +423,8 @@ fn root_ownership(path: &Path, stage: &'static str) -> Result<(), Error> { #[cfg(test)] mod tests { + use std::os::unix::fs::PermissionsExt; + use super::*; #[test] @@ -541,4 +470,18 @@ mod tests { &target )); } + + #[test] + fn staging_directory_is_private_and_removed_on_drop() { + let parent = tempfile::tempdir().unwrap(); + let path = { + let staging = staging_directory(parent.path()).unwrap(); + assert_eq!( + staging.path().metadata().unwrap().permissions().mode() & 0o777, + 0o700 + ); + staging.path().to_owned() + }; + assert!(!path.exists()); + } } diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs index 3548405136..5333fed415 100644 --- a/crates/ployzd/src/installer/storage.rs +++ b/crates/ployzd/src/installer/storage.rs @@ -4,6 +4,7 @@ use std::{ ffi::OsStr, fs::{self, OpenOptions}, io::Write, + os::unix::fs::MetadataExt, path::Path, process::Command, }; @@ -13,10 +14,11 @@ use ployz_core::StorageChoice; use super::{Error, InstallPaths, command_exists, run_apt, run_command, run_host}; use super::{ host::write_file_atomically, - release::{Staging, write_private}, + release::{staging_directory, write_private}, }; const ZFS_SMOKE_BYTES: u64 = 128 * 1024 * 1024; +const POSIX_STAT_BLOCK_BYTES: u64 = 512; pub(super) fn prepare_storage(storage: StorageChoice, paths: &InstallPaths) -> Result<(), Error> { match storage { @@ -121,26 +123,11 @@ fn uname(arg: &str, stage: &str) -> Result { } fn require_host_root_reserve(allocation: u64) -> Result<(), Error> { - let output = run_host( - "inspect host-root capacity for ZFS storage preparation", - "df", - ["-B1", "--output=size,avail", "/"], - )?; - let line = String::from_utf8_lossy(&output.stdout) - .lines() - .last() - .unwrap_or_default() - .to_owned(); - let mut fields = line.split_whitespace(); - let parse = |value: Option<&str>| value.and_then(|value| value.parse::().ok()); - let (Some(size), Some(available), None) = - (parse(fields.next()), parse(fields.next()), fields.next()) - else { - return Err(Error::Command { - stage: "prepare ZFS storage".into(), - message: "Could not read host-root capacity for ZFS storage preparation".into(), - }); - }; + let (size, available) = + crate::host_capacity::filesystem_space(Path::new("/")).map_err(|source| Error::Io { + stage: "inspect host-root capacity for ZFS storage preparation", + source, + })?; let reserve = (size / 4).max(10 * 1024 * 1024 * 1024); if available < reserve.saturating_add(allocation) { return Err(Error::Command { @@ -218,11 +205,12 @@ pub(super) fn install_zfs_packages(kernel: &str) -> Result<(), Error> { let mut show = Command::new("apt-cache"); show.args(["show", &candidate]); if !show - .status() + .output() .map_err(|source| Error::Io { stage: "inspect Ubuntu ZFS module packages", source, })? + .status .success() { continue; @@ -233,16 +221,16 @@ pub(super) fn install_zfs_packages(kernel: &str) -> Result<(), Error> { package = Some(candidate); break; } - let scratch = Staging::new(Path::new("/var/tmp"))?; + let scratch = staging_directory(Path::new("/var/tmp"))?; if let Err(error) = run_apt( "download Ubuntu ZFS module package", ["download", &candidate], - Some(&scratch.path), + Some(scratch.path()), ) { download_error = Some(error); continue; } - let archive = fs::read_dir(&scratch.path) + let archive = fs::read_dir(scratch.path()) .map_err(|source| Error::Io { stage: "inspect downloaded Ubuntu ZFS module package", source, @@ -341,8 +329,8 @@ fn set_and_verify_zfs_arc_max(cap: u64) -> Result<(), Error> { fn validate_zfs() -> Result<(), Error> { require_host_root_reserve(ZFS_SMOKE_BYTES)?; - let stage = Staging::new(Path::new("/var/tmp"))?; - let backing = stage.path.join("backing"); + let stage = staging_directory(Path::new("/var/tmp"))?; + let backing = stage.path().join("backing"); write_private(&backing, b"", "create ZFS smoke backing file")?; // The installer lock serializes smoke Pools, and the process ID is the shell installer's // existing collision boundary. @@ -354,26 +342,21 @@ fn validate_zfs() -> Result<(), Error> { .args(["-l", &ZFS_SMOKE_BYTES.to_string()]) .arg(&backing); run_command("preallocate ZFS smoke backing file", &mut allocate)?; - let mut stat = Command::new("stat"); - stat.args(["-c", "%b %B"]).arg(&backing); - let output = run_command("verify ZFS smoke backing allocation", &mut stat)?; - let stdout = String::from_utf8_lossy(&output.stdout); - let mut fields = stdout.split_whitespace(); - let blocks = fields.next().and_then(|value| value.parse::().ok()); - let block_size = fields.next().and_then(|value| value.parse::().ok()); - if blocks - .zip(block_size) - .is_none_or(|(blocks, size)| blocks.saturating_mul(size) < ZFS_SMOKE_BYTES) - { + let metadata = backing.metadata().map_err(|source| Error::Io { + stage: "inspect ZFS smoke backing file", + source, + })?; + if metadata.blocks().saturating_mul(POSIX_STAT_BLOCK_BYTES) < ZFS_SMOKE_BYTES { return Err(Error::Verification(format!( "ZFS smoke backing file {} is sparse", backing.display() ))); } - let mut mount = Command::new("findmnt"); - mount.args(["-n", "-o", "TARGET", "-T"]).arg(&backing); - let output = run_command("verify ZFS smoke backing filesystem", &mut mount)?; - if String::from_utf8_lossy(&output.stdout).trim() != "/" { + let root = fs::metadata("/").map_err(|source| Error::Io { + stage: "inspect host-root filesystem", + source, + })?; + if metadata.dev() != root.dev() { return Err(Error::Verification(format!( "ZFS smoke backing file {} is not on the host root filesystem", backing.display() diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index 3336c550ad..1ba1b7504b 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -182,6 +182,18 @@ fn install_request( let release = version .parse::() .map_err(io::Error::other)?; + if install_only && storage != StorageChoice::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--install-only cannot be combined with --storage", + )); + } + if install_only && group_user.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--install-only cannot be combined with --group-user", + )); + } let preparation = if software_only { if storage != StorageChoice::None { return Err(io::Error::new( @@ -273,6 +285,32 @@ mod tests { ); } + #[test] + fn install_cli_rejects_host_options_for_installation_only() { + let storage = install_request("stable".into(), StorageChoice::Zfs, false, true, None, None) + .unwrap_err(); + assert_eq!(storage.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + storage.to_string(), + "--install-only cannot be combined with --storage" + ); + + let group = install_request( + "stable".into(), + StorageChoice::None, + false, + true, + Some("operator".into()), + None, + ) + .unwrap_err(); + assert_eq!(group.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + group.to_string(), + "--install-only cannot be combined with --group-user" + ); + } + #[test] fn install_cli_builds_one_explicit_preparation_mode() { let replacement = @@ -283,7 +321,7 @@ mod tests { "1.2.3".into(), StorageChoice::Zfs, false, - true, + false, Some("operator".into()), None, ) From 5f672d360d79f9f2d0d7d36548cdd8e824a2510b Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 01:53:44 +0000 Subject: [PATCH 07/30] feat: add bounded machine upgrades --- Cargo.lock | 1 + crates/ployz-core/Cargo.toml | 1 + crates/ployz-core/src/rpc.rs | 86 ++++ crates/ployz-core/src/rpc_catalog.rs | 2 + crates/ployz-core/src/value.rs | 63 +++ crates/ployz-core/tests/capabilities.rs | 38 +- crates/ployz-core/tests/contract.rs | 68 ++- crates/ployz/src/cli/mod.rs | 28 ++ crates/ployz/src/handlers/machine.rs | 2 + crates/ployz/src/handlers/machine/upgrade.rs | 286 +++++++++++ crates/ployz/src/handlers/mod.rs | 2 + crates/ployz/tests/cli_shape.rs | 87 ++++ crates/ployz/tests/cloud_init/harness.rs | 14 + crates/ployz/tests/connect/support.rs | 14 + crates/ployz/tests/deploy_client/support.rs | 12 + crates/ployzd/src/daemon.rs | 12 +- crates/ployzd/src/installer/admission.rs | 149 ++++++ crates/ployzd/src/installer/mod.rs | 135 +++--- crates/ployzd/src/installer/release.rs | 6 +- crates/ployzd/src/installer/upgrade.rs | 449 ++++++++++++++++++ crates/ployzd/src/machine/local_machine.rs | 141 +++++- .../src/machine/local_machine/container.rs | 27 +- crates/ployzd/src/machine/mod.rs | 17 +- crates/ployzd/src/machine_api/local.rs | 213 +++++++-- crates/ployzd/src/main.rs | 25 +- crates/ployzd/src/volume_plugin.rs | 46 +- crates/ployzd/src/volume_plugin/capacity.rs | 4 +- crates/ployzd/src/volume_plugin/pool.rs | 2 +- crates/ployzd/src/volume_plugin/removal.rs | 2 +- .../volume_plugin/tests/first_pool_tests.rs | 6 + 30 files changed, 1759 insertions(+), 179 deletions(-) create mode 100644 crates/ployz/src/handlers/machine/upgrade.rs create mode 100644 crates/ployzd/src/installer/admission.rs create mode 100644 crates/ployzd/src/installer/upgrade.rs diff --git a/Cargo.lock b/Cargo.lock index 60e65149f5..614ca779dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2181,6 +2181,7 @@ dependencies = [ "base64", "ipnet", "prost", + "semver", "serde", "serde_json", "thiserror 2.0.20", diff --git a/crates/ployz-core/Cargo.toml b/crates/ployz-core/Cargo.toml index 8ed533f5e1..f77bf5784a 100644 --- a/crates/ployz-core/Cargo.toml +++ b/crates/ployz-core/Cargo.toml @@ -11,6 +11,7 @@ ipnet = { version = "2.12.1", features = ["serde"] } prost = "0.13.5" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" +semver = "1.0.28" thiserror = "2.0.20" ts-rs.workspace = true url = { version = "2.5.8", features = ["serde"] } diff --git a/crates/ployz-core/src/rpc.rs b/crates/ployz-core/src/rpc.rs index 6a391daaaf..6d69a87f51 100644 --- a/crates/ployz-core/src/rpc.rs +++ b/crates/ployz-core/src/rpc.rs @@ -457,6 +457,18 @@ pub struct UpdateMachineRequest { pub update: MachineUpdate, } +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RequestMachineUpgradeRequest { + pub attempt_id: crate::MachineUpgradeAttemptId, + pub release: crate::MachineRelease, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct InspectMachineUpgradeRequest { + #[serde(default)] + pub attempt_id: Option, +} + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct RemoveLocalMachineRequest { #[serde(default)] @@ -719,6 +731,79 @@ pub struct MachineUpdated { pub machine: Machine, } +crate::value::open_string_enum!(MachineUpgradeStage, Unknown { + Launching => "launching", + Preparing => "preparing", + Acquiring => "acquiring", + Verifying => "verifying", + Activating => "activating", + Restarting => "restarting", + Readiness => "readiness", +}); + +/// Durable local evidence for one bounded Machine upgrade attempt. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TS)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum MachineUpgradeAttempt { + Accepted { + attempt_id: crate::MachineUpgradeAttemptId, + target: crate::MachineVersion, + }, + Running { + attempt_id: crate::MachineUpgradeAttemptId, + target: crate::MachineVersion, + stage: MachineUpgradeStage, + }, + Succeeded { + attempt_id: crate::MachineUpgradeAttemptId, + /// The requested version observed running and locally ready. + version: crate::MachineVersion, + }, + Failed { + attempt_id: crate::MachineUpgradeAttemptId, + target: crate::MachineVersion, + stage: MachineUpgradeStage, + error: String, + }, + Interrupted { + attempt_id: crate::MachineUpgradeAttemptId, + target: crate::MachineVersion, + stage: MachineUpgradeStage, + }, +} + +impl MachineUpgradeAttempt { + #[must_use] + pub fn attempt_id(&self) -> crate::MachineUpgradeAttemptId { + match self { + Self::Accepted { attempt_id, .. } + | Self::Running { attempt_id, .. } + | Self::Succeeded { attempt_id, .. } + | Self::Failed { attempt_id, .. } + | Self::Interrupted { attempt_id, .. } => *attempt_id, + } + } + + #[must_use] + pub fn target(&self) -> &crate::MachineVersion { + match self { + Self::Accepted { target, .. } + | Self::Running { target, .. } + | Self::Failed { target, .. } + | Self::Interrupted { target, .. } => target, + Self::Succeeded { version, .. } => version, + } + } + + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!( + self, + Self::Succeeded { .. } | Self::Failed { .. } | Self::Interrupted { .. } + ) + } +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize, TS)] pub struct LocalMachineRemoved { #[serde(default)] @@ -829,6 +914,7 @@ define_responses! { Domain(Domain) => "domain"; DomainRecords(DomainRecords) => "domain_records"; MachineUpdated(MachineUpdated) => "machine_updated"; + MachineUpgradeAttempt(MachineUpgradeAttempt) => "machine_upgrade_attempt"; LocalMachineRemoved(LocalMachineRemoved) => "local_machine_removed"; MachineRemoved(MachineRemoved) => "machine_removed"; WireGuardInspected(WireGuardInspected) => "wireguard_inspected"; diff --git a/crates/ployz-core/src/rpc_catalog.rs b/crates/ployz-core/src/rpc_catalog.rs index 82f5b1dc30..5bda656876 100644 --- a/crates/ployz-core/src/rpc_catalog.rs +++ b/crates/ployz-core/src/rpc_catalog.rs @@ -44,6 +44,8 @@ macro_rules! rpc_catalog { SetCloudPairing: (set_cloud_pairing, "SetCloudPairing", SetCloudPairingRequest, "set_cloud_pairing", CloudPairingSet, SET_CLOUD_PAIRING_CAPABILITY, "ployz.machine.set-cloud-pairing.v1", Always), ListMachines: (list_machines, "ListMachines", ListMachinesRequest, "list_machines", MachineList, LIST_MACHINES_CAPABILITY, "ployz.machine.list.v1", Always), UpdateMachine: (update_machine, "UpdateMachine", UpdateMachineRequest, "update_machine", MachineUpdated, UPDATE_MACHINE_CAPABILITY, "ployz.machine.update.v1", Always), + RequestMachineUpgrade: (request_machine_upgrade, "RequestMachineUpgrade", RequestMachineUpgradeRequest, "request_machine_upgrade", MachineUpgradeAttempt, REQUEST_MACHINE_UPGRADE_CAPABILITY, "ployz.machine.upgrade.request.v1", Always), + InspectMachineUpgrade: (inspect_machine_upgrade, "InspectMachineUpgrade", InspectMachineUpgradeRequest, "inspect_machine_upgrade", MachineUpgradeAttempt, INSPECT_MACHINE_UPGRADE_CAPABILITY, "ployz.machine.upgrade.inspect.v1", Always), RemoveLocalMachine: (remove_local_machine, "RemoveLocalMachine", RemoveLocalMachineRequest, "remove_local_machine", LocalMachineRemoved, REMOVE_LOCAL_MACHINE_CAPABILITY, "ployz.machine.remove-local.v1", Always), RemoveMachine: (remove_machine, "RemoveMachine", RemoveMachineRequest, "remove_machine", MachineRemoved, REMOVE_MACHINE_CAPABILITY, "ployz.machine.remove.v1", Always), InspectWireguard: (inspect_wireguard, "InspectWireguard", InspectWireGuardRequest, "inspect_wireguard", WireGuardInspected, INSPECT_WIREGUARD_CAPABILITY, "ployz.wireguard.inspect.v1", Always), diff --git a/crates/ployz-core/src/value.rs b/crates/ployz-core/src/value.rs index 91f9b14275..9078e4de40 100644 --- a/crates/ployz-core/src/value.rs +++ b/crates/ployz-core/src/value.rs @@ -63,6 +63,18 @@ fn is_hostname(value: &str) -> bool { (1..=253).contains(&value.len()) && value.split('.').all(is_dns_label) } +fn is_machine_version(value: &str) -> bool { + let Ok(version) = semver::Version::parse(value) else { + return false; + }; + let pre = version.pre.as_str(); + version.build.is_empty() + && (pre.is_empty() + || pre.strip_prefix("beta.").is_some_and(|number| { + !number.is_empty() && number.bytes().all(|byte| byte.is_ascii_digit()) + })) +} + macro_rules! hex_id_newtype { ($(#[$attribute:meta])* $name:ident, $label:literal, $len:expr, $expected:literal) => { $(#[$attribute])* @@ -208,6 +220,13 @@ hex_id_newtype!( 32, "32 lowercase hexadecimal characters" ); +hex_id_newtype!( + /// Identity of one bounded Machine upgrade attempt. + MachineUpgradeAttemptId, + "Machine upgrade attempt ID", + 32, + "32 lowercase hexadecimal characters" +); hex_id_newtype!( ServiceId, "Service ID", @@ -237,6 +256,16 @@ impl MachineId { } } +impl MachineUpgradeAttemptId { + /// Random 32-character lowercase hexadecimal attempt ID. + #[must_use] + pub fn random() -> Self { + let mut hex = [0_u8; 32]; + uuid::Uuid::new_v4().simple().encode_lower(&mut hex); + Self(hex) + } +} + impl ServiceId { #[must_use] pub fn random() -> Self { @@ -287,6 +316,20 @@ validated_string_newtype!( "a 1-63 character lowercase DNS label", |value| is_dns_label(value) ); +validated_string_newtype!( + /// A trusted release selector: stable, beta, or an exact supported version. + MachineRelease, + "Machine release", + "stable, beta, X.Y.Z, or X.Y.Z-beta.N", + |value| matches!(value, "stable" | "beta") || is_machine_version(value) +); +validated_string_newtype!( + /// One exact supported Machine release version. + MachineVersion, + "Machine version", + "X.Y.Z or X.Y.Z-beta.N", + |value| is_machine_version(value) +); validated_string_newtype!( DockerVolumeName, "Docker Volume name", @@ -878,4 +921,24 @@ mod tests { assert_eq!(format!("{token:?}"), "CloudEnrollToken(..)"); assert!(!format!("{token:?}").contains("pmet_test")); } + + #[test] + fn machine_release_accepts_only_channels_and_supported_exact_versions() { + for release in ["stable", "beta", "1.2.3", "1.2.3-beta.4"] { + assert!(MachineRelease::parse(release).is_ok(), "{release}"); + } + for release in [ + "nightly", + "v1.2.3", + "1.2", + "1.2.3-alpha.1", + "1.2.3+build", + "1.2.3-beta.01", + "https://example.test/release", + ] { + assert!(MachineRelease::parse(release).is_err(), "{release}"); + assert!(MachineVersion::parse(release).is_err(), "{release}"); + } + assert!(MachineVersion::parse("stable").is_err()); + } } diff --git a/crates/ployz-core/tests/capabilities.rs b/crates/ployz-core/tests/capabilities.rs index a3754c9c57..613d39a041 100644 --- a/crates/ployz-core/tests/capabilities.rs +++ b/crates/ployz-core/tests/capabilities.rs @@ -4,16 +4,16 @@ use ployz_core::{ CapabilityName, DESCRIBE_CONTRACT_CAPABILITY, ENSURE_GLOBAL_SLOT_CAPABILITY, ENSURE_IMAGE_INGEST_CAPABILITY, EXEC_CONTAINER_CAPABILITY, GET_DOMAIN_CAPABILITY, GET_INGRESS_PROXY_CONFIG_CAPABILITY, INITIALIZE_MACHINE_CAPABILITY, - INSPECT_CONTAINER_CAPABILITY, INSPECT_MACHINE_CAPABILITY, INSPECT_STORAGE_CAPABILITY, - INSPECT_VOLUME_CAPABILITY, INSPECT_WIREGUARD_CAPABILITY, JOIN_MACHINE_CAPABILITY, - LIST_CONTAINERS_CAPABILITY, LIST_IMAGES_CAPABILITY, LIST_MACHINES_CAPABILITY, - LIST_VOLUMES_CAPABILITY, MACHINE_LOGS_CAPABILITY, MACHINE_STORAGE_OBSERVATION_CAPABILITY, - MACHINE_TOKEN_CAPABILITY, PREPARE_VOLUMES_CAPABILITY, PULL_IMAGE_FROM_MACHINE_CAPABILITY, - REGISTER_MACHINE_CAPABILITY, RELEASE_DOMAIN_CAPABILITY, REMOVE_CONTAINER_CAPABILITY, - REMOVE_LOCAL_MACHINE_CAPABILITY, REMOVE_MACHINE_CAPABILITY, REMOVE_VOLUME_CAPABILITY, - RESERVE_DOMAIN_CAPABILITY, RESET_MACHINE_CAPABILITY, RUNTIME_WATCH_CAPABILITY, Rpc, - SET_CLOUD_PAIRING_CAPABILITY, START_CONTAINER_CAPABILITY, STOP_CONTAINER_CAPABILITY, - UPDATE_MACHINE_CAPABILITY, op, + INSPECT_CONTAINER_CAPABILITY, INSPECT_MACHINE_CAPABILITY, INSPECT_MACHINE_UPGRADE_CAPABILITY, + INSPECT_STORAGE_CAPABILITY, INSPECT_VOLUME_CAPABILITY, INSPECT_WIREGUARD_CAPABILITY, + JOIN_MACHINE_CAPABILITY, LIST_CONTAINERS_CAPABILITY, LIST_IMAGES_CAPABILITY, + LIST_MACHINES_CAPABILITY, LIST_VOLUMES_CAPABILITY, MACHINE_LOGS_CAPABILITY, + MACHINE_STORAGE_OBSERVATION_CAPABILITY, MACHINE_TOKEN_CAPABILITY, PREPARE_VOLUMES_CAPABILITY, + PULL_IMAGE_FROM_MACHINE_CAPABILITY, REGISTER_MACHINE_CAPABILITY, RELEASE_DOMAIN_CAPABILITY, + REMOVE_CONTAINER_CAPABILITY, REMOVE_LOCAL_MACHINE_CAPABILITY, REMOVE_MACHINE_CAPABILITY, + REMOVE_VOLUME_CAPABILITY, REQUEST_MACHINE_UPGRADE_CAPABILITY, RESERVE_DOMAIN_CAPABILITY, + RESET_MACHINE_CAPABILITY, RUNTIME_WATCH_CAPABILITY, Rpc, SET_CLOUD_PAIRING_CAPABILITY, + START_CONTAINER_CAPABILITY, STOP_CONTAINER_CAPABILITY, UPDATE_MACHINE_CAPABILITY, op, }; /// Capability constants are generated from the catalog, so a typo would stay @@ -35,6 +35,14 @@ fn catalogued_capabilities_keep_stable_spellings() { "ployz.machine.set-cloud-pairing.v1", ), (LIST_MACHINES_CAPABILITY, "ployz.machine.list.v1"), + ( + REQUEST_MACHINE_UPGRADE_CAPABILITY, + "ployz.machine.upgrade.request.v1", + ), + ( + INSPECT_MACHINE_UPGRADE_CAPABILITY, + "ployz.machine.upgrade.inspect.v1", + ), ( MACHINE_STORAGE_OBSERVATION_CAPABILITY, "ployz.machine.storage-observation.v1", @@ -112,6 +120,8 @@ fn advertised_capability_groups_match_the_frozen_catalog() { "ployz.machine.set-cloud-pairing.v1", "ployz.machine.list.v1", "ployz.machine.update.v1", + "ployz.machine.upgrade.request.v1", + "ployz.machine.upgrade.inspect.v1", "ployz.machine.remove-local.v1", "ployz.machine.remove.v1", "ployz.wireguard.inspect.v1", @@ -199,4 +209,12 @@ fn unary_grpc_paths_stay_on_the_machine_rpc_service() { op::EnsureGlobalSlot::PATH, "/ployz.rpc.v1.MachineRpc/EnsureGlobalSlot" ); + assert_eq!( + op::RequestMachineUpgrade::PATH, + "/ployz.rpc.v1.MachineRpc/RequestMachineUpgrade" + ); + assert_eq!( + op::InspectMachineUpgrade::PATH, + "/ployz.rpc.v1.MachineRpc/InspectMachineUpgrade" + ); } diff --git a/crates/ployz-core/tests/contract.rs b/crates/ployz-core/tests/contract.rs index c8a8e61326..cdc6962fcf 100644 --- a/crates/ployz-core/tests/contract.rs +++ b/crates/ployz-core/tests/contract.rs @@ -14,16 +14,18 @@ use ployz_core::{ GetContainerObservationsRequest, GetIngressProxyConfigRequest, HealthObservation, HttpProtocol, ImageIngestDestination, ImageIngestOpened, ImageIngestReason, ImagePulled, ImageSummary, IngressHost, IngressHostname, IngressProxyConfig, IngressProxyFragment, - InspectWireGuardRequest, LIST_IMAGES_CAPABILITY, ListImagesRequest, MANAGED_LABEL, - MachineFailure, MachineGateway, MachineId, MachineImages, MachineName, MachineSubnet, - MachineSuccess, MachineTokenRequest, MachineUpdate, ManagementAddress, NameMatches, - OpaquePayload, PROJECT_NAME_LABEL, PROTOCOL_MAJOR, PULL_IMAGE_FROM_MACHINE_CAPABILITY, - PartialResult, PortPublication, ProjectName, PublicIpDiscovery, PublicIpUpdate, - PullImageFromMachineRequest, QualifiedService, RESET_MACHINE_CAPABILITY, - RemoveLocalMachineRequest, RemoveMachineRequest, RequestedServiceSpec, ReserveDomainRequest, - ResetAccepted, ResetRequest, ResolvedServiceSpec, ResponseKind, RpcError, RpcErrorCode, - RpcRequestBody, RpcResponse, RpcResponseBody, ServiceId, ServiceName, UpdateMachineRequest, - VolumeSource, encode_grpc_frame, grpc_frames, op, + InspectMachineUpgradeRequest, InspectWireGuardRequest, LIST_IMAGES_CAPABILITY, + ListImagesRequest, MANAGED_LABEL, MachineFailure, MachineGateway, MachineId, MachineImages, + MachineName, MachineRelease, MachineSubnet, MachineSuccess, MachineTokenRequest, MachineUpdate, + MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeStage, MachineVersion, + ManagementAddress, NameMatches, OpaquePayload, PROJECT_NAME_LABEL, PROTOCOL_MAJOR, + PULL_IMAGE_FROM_MACHINE_CAPABILITY, PartialResult, PortPublication, ProjectName, + PublicIpDiscovery, PublicIpUpdate, PullImageFromMachineRequest, QualifiedService, + RESET_MACHINE_CAPABILITY, RemoveLocalMachineRequest, RemoveMachineRequest, + RequestMachineUpgradeRequest, RequestedServiceSpec, ReserveDomainRequest, ResetAccepted, + ResetRequest, ResolvedServiceSpec, ResponseKind, RpcError, RpcErrorCode, RpcRequestBody, + RpcResponse, RpcResponseBody, ServiceId, ServiceName, UpdateMachineRequest, VolumeSource, + encode_grpc_frame, grpc_frames, op, }; use prost::Message; use serde_json::{Value, json}; @@ -1474,6 +1476,52 @@ fn machine_administration_requests_round_trip_as_typed_payloads() { } } +#[test] +fn machine_upgrade_request_and_receipt_have_one_typed_wire_contract() { + let attempt_id = MachineUpgradeAttemptId::parse(MACHINE_ID).unwrap(); + let request = RequestMachineUpgradeRequest { + attempt_id, + release: MachineRelease::parse("beta").unwrap(), + }; + assert_eq!( + op::RequestMachineUpgrade::into_request(request.clone()) + .encode() + .unwrap() + .decode_request() + .unwrap() + .body, + RpcRequestBody::RequestMachineUpgrade(request) + ); + let inspect = InspectMachineUpgradeRequest { + attempt_id: Some(attempt_id), + }; + assert_eq!( + op::InspectMachineUpgrade::into_request(inspect.clone()) + .encode() + .unwrap() + .decode_request() + .unwrap() + .body, + RpcRequestBody::InspectMachineUpgrade(inspect) + ); + + let attempt = MachineUpgradeAttempt::Failed { + attempt_id, + target: MachineVersion::parse("1.2.3-beta.4").unwrap(), + stage: MachineUpgradeStage::Verifying, + error: "checksum mismatch".into(), + }; + let response = RpcResponse::from(attempt.clone()); + assert_eq!( + response.decode::().unwrap(), + attempt + ); + assert_eq!( + response.decode::().unwrap(), + attempt + ); +} + #[test] fn service_ingress_proxy_fragment_is_a_validated_caddy_string() { let spec: RequestedServiceSpec = serde_json::from_value(json!({ diff --git a/crates/ployz/src/cli/mod.rs b/crates/ployz/src/cli/mod.rs index cd7816a5a6..569f56be4d 100644 --- a/crates/ployz/src/cli/mod.rs +++ b/crates/ployz/src/cli/mod.rs @@ -364,6 +364,7 @@ fn machine() -> Command { ), ) .subcommand(base("rtt", "Show round-trip times")) + .subcommand(machine_upgrade()) .subcommand( base("update", "Update machine configuration") .arg(value("name", None)) @@ -373,6 +374,33 @@ fn machine() -> Command { ) } +fn machine_upgrade() -> Command { + base("upgrade", "Upgrade explicitly selected machines") + .arg_required_else_help(true) + .subcommand_negates_reqs(true) + .args_conflicts_with_subcommands(true) + .subcommand_precedence_over_arg(true) + .arg( + positional("version", true) + .value_name("VERSION") + .value_parser(clap::value_parser!(ployz_core::MachineRelease)), + ) + .arg( + many("machine", Some('m')) + .required(true) + .help("Machine name or ID; repeat for an explicit sequence"), + ) + .subcommand( + base("inspect", "Inspect one Machine upgrade attempt") + .arg(positional("machine", true)) + .arg( + value("attempt", None) + .value_parser(clap::value_parser!(ployz_core::MachineUpgradeAttemptId)), + ) + .arg(json_output()), + ) +} + fn provisioning_flags(command: Command) -> Command { command .arg(value("name", Some('n'))) diff --git a/crates/ployz/src/handlers/machine.rs b/crates/ployz/src/handlers/machine.rs index 39bd984bab..999257afde 100644 --- a/crates/ployz/src/handlers/machine.rs +++ b/crates/ployz/src/handlers/machine.rs @@ -18,6 +18,7 @@ mod helpers; mod init; mod inspect; mod remove; +mod upgrade; pub(super) use add::add; pub(super) use helpers::{ @@ -26,6 +27,7 @@ pub(super) use helpers::{ pub(super) use init::init; pub(super) use inspect::{inspect, list, rtt, wireguard_show}; pub(super) use remove::remove; +pub(super) use upgrade::{inspect as inspect_upgrade, upgrade}; const DEFAULT_WIREGUARD_PORT: u16 = 51820; diff --git a/crates/ployz/src/handlers/machine/upgrade.rs b/crates/ployz/src/handlers/machine/upgrade.rs new file mode 100644 index 0000000000..929442f8e6 --- /dev/null +++ b/crates/ployz/src/handlers/machine/upgrade.rs @@ -0,0 +1,286 @@ +//! Explicit, bounded Machine upgrade sequencing and inspection. + +use std::{collections::BTreeSet, time::Duration}; + +use clap::ArgMatches; +use ployz_core::{ + InspectMachineUpgradeRequest, Machine, MachineRelease, MachineTarget, MachineUpgradeAttempt, + MachineUpgradeAttemptId, NameMatches, RequestMachineUpgradeRequest, op, +}; +use tokio::time::Instant; + +use crate::{cluster::Client, connect::ConnectError}; + +use super::super::{Error, leaf_matches, string_values, with_client}; + +const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(16 * 60); +const POLL_INTERVAL: Duration = Duration::from_secs(1); + +pub(in crate::handlers) fn upgrade(root: &ArgMatches) -> Result<(), Error> { + let matches = leaf_matches(root); + let release = matches + .get_one::("version") + .cloned() + .ok_or_else(|| Error::usage("upgrade version is required"))?; + let selectors = string_values(matches, "machine"); + with_client(root, |client| { + Box::pin(async move { + let machines = selected_machines(client, &selectors).await?; + for machine in machines { + let attempt_id = MachineUpgradeAttemptId::random(); + let attempt = run_one(client, &machine, release.clone(), attempt_id).await?; + print_attempt(&machine, &attempt); + match attempt { + MachineUpgradeAttempt::Succeeded { .. } => {} + MachineUpgradeAttempt::Failed { error, .. } => { + return Err(Error::usage(error)); + } + MachineUpgradeAttempt::Interrupted { .. } => { + return Err(Error::usage(format!( + "Machine {} upgrade was interrupted; {}", + machine.name, + journal_hint(attempt_id) + ))); + } + MachineUpgradeAttempt::Accepted { .. } + | MachineUpgradeAttempt::Running { .. } => { + unreachable!("run_one returns only terminal evidence") + } + } + } + Ok(()) + }) + }) +} + +pub(in crate::handlers) fn inspect(root: &ArgMatches) -> Result<(), Error> { + let matches = leaf_matches(root); + let target = MachineTarget::parse( + matches + .get_one::("machine") + .ok_or_else(|| Error::usage("machine is required"))?, + )?; + let attempt_id = matches + .get_one::("attempt") + .copied(); + let json = matches.get_one::("output").is_some(); + with_client(root, |client| { + Box::pin(async move { + let attempt = client + .call_repeatable::( + InspectMachineUpgradeRequest { attempt_id }, + Some(&target), + ) + .await?; + if json { + println!("{}", serde_json::to_string_pretty(&attempt)?); + } else { + print_attempt_target(target.as_str(), &attempt); + } + Ok(()) + }) + }) +} + +async fn selected_machines( + client: &mut Client, + selectors: &[String], +) -> Result, Error> { + let visible = client.machines().await?; + let mut selected = Vec::with_capacity(selectors.len()); + let mut ids = BTreeSet::new(); + for selector in selectors { + let target = MachineTarget::parse(selector)?; + let machine = match target.resolve(visible.iter().map(|entry| &entry.machine)) { + NameMatches::None => { + return Err(Error::usage(format!( + "Machine {} was not found", + selector.escape_debug() + ))); + } + NameMatches::One(machine) => machine, + matches @ NameMatches::Ambiguous { .. } => { + return Err(Error::usage(format!( + "Machine name {} is ambiguous: {}", + selector.escape_debug(), + matches + .iter() + .map(|machine| machine.id.as_str()) + .collect::>() + .join(", ") + ))); + } + }; + if !ids.insert(machine.id) { + return Err(Error::usage(format!( + "Machine {} was selected more than once", + machine.name + ))); + } + selected.push(machine.clone()); + } + Ok(selected) +} + +async fn run_one( + client: &mut Client, + machine: &Machine, + release: MachineRelease, + attempt_id: MachineUpgradeAttemptId, +) -> Result { + let target = MachineTarget::from(&machine.id); + let deadline = Instant::now() + OBSERVATION_TIMEOUT; + let request = RequestMachineUpgradeRequest { + attempt_id, + release, + }; + let accepted = request_until(deadline, client, &target, request) + .await + .map_err(|error| uncertain(machine, attempt_id, error))?; + print_attempt(machine, &accepted); + if accepted.is_terminal() { + return Ok(accepted); + } + + loop { + if tokio::time::timeout_at(deadline, tokio::time::sleep(POLL_INTERVAL)) + .await + .is_err() + { + return Err(uncertain_timeout(machine, attempt_id)); + } + let observed = inspect_until(deadline, client, &target, attempt_id) + .await + .map_err(|error| uncertain(machine, attempt_id, error))?; + if observed.is_terminal() { + return Ok(observed); + } + } +} + +async fn request_until( + deadline: Instant, + client: &mut Client, + target: &MachineTarget, + request: RequestMachineUpgradeRequest, +) -> Result { + loop { + let outcome = tokio::time::timeout_at( + deadline, + client.call_repeatable::(request.clone(), Some(target)), + ) + .await; + match outcome { + Ok(Ok(value)) => return Ok(value), + Ok(Err(error)) if !error.is_setup_retryable() => return Err(error), + Ok(Err(_)) => {} + Err(_) => { + return Err( + tonic::Status::deadline_exceeded("upgrade observation timed out").into(), + ); + } + } + } +} + +async fn inspect_until( + deadline: Instant, + client: &mut Client, + target: &MachineTarget, + attempt_id: MachineUpgradeAttemptId, +) -> Result { + loop { + let outcome = tokio::time::timeout_at( + deadline, + client.call_repeatable::( + InspectMachineUpgradeRequest { + attempt_id: Some(attempt_id), + }, + Some(target), + ), + ) + .await; + match outcome { + Ok(Ok(value)) => return Ok(value), + Ok(Err(error)) if !error.is_setup_retryable() => return Err(error), + Ok(Err(_)) => {} + Err(_) => { + return Err( + tonic::Status::deadline_exceeded("upgrade observation timed out").into(), + ); + } + } + } +} + +fn print_attempt(machine: &Machine, attempt: &MachineUpgradeAttempt) { + print_attempt_target(&format!("{} ({})", machine.name, machine.id), attempt); +} + +fn print_attempt_target(machine: &str, attempt: &MachineUpgradeAttempt) { + match attempt { + MachineUpgradeAttempt::Accepted { attempt_id, target } => println!( + "Machine {machine}: upgrade {attempt_id} accepted for {target}; {}", + journal_hint(*attempt_id) + ), + MachineUpgradeAttempt::Running { + attempt_id, + target, + stage, + } => println!( + "Machine {machine}: upgrade {attempt_id} is {} for {target}; {}", + stage.as_str(), + journal_hint(*attempt_id) + ), + MachineUpgradeAttempt::Succeeded { + attempt_id, + version, + } => { + println!("Machine {machine}: upgrade {attempt_id} succeeded; running version {version}") + } + MachineUpgradeAttempt::Failed { + attempt_id, + target, + stage, + error, + } => println!( + "Machine {machine}: upgrade {attempt_id} failed at {} for {target}: {error}; {}", + stage.as_str(), + journal_hint(*attempt_id) + ), + MachineUpgradeAttempt::Interrupted { + attempt_id, + target, + stage, + } => println!( + "Machine {machine}: upgrade {attempt_id} was interrupted at {} for {target}; {}", + stage.as_str(), + journal_hint(*attempt_id) + ), + } +} + +fn journal_hint(attempt_id: MachineUpgradeAttemptId) -> String { + format!("inspect locally with `journalctl -u ployz-upgrade-{attempt_id}.service`") +} + +fn uncertain(machine: &Machine, attempt_id: MachineUpgradeAttemptId, error: ConnectError) -> Error { + Error::usage(format!( + "Machine {} ({}) upgrade {attempt_id} outcome is uncertain: {error}; reconnect and run `ployz machine upgrade inspect {} --attempt {attempt_id}`; {}", + machine.name, + machine.id, + machine.id, + journal_hint(attempt_id) + )) +} + +fn uncertain_timeout(machine: &Machine, attempt_id: MachineUpgradeAttemptId) -> Error { + Error::usage(format!( + "Machine {} ({}) upgrade {attempt_id} outcome is uncertain after {} minutes; reconnect and run `ployz machine upgrade inspect {} --attempt {attempt_id}`; {}", + machine.name, + machine.id, + OBSERVATION_TIMEOUT.as_secs() / 60, + machine.id, + journal_hint(attempt_id) + )) +} diff --git a/crates/ployz/src/handlers/mod.rs b/crates/ployz/src/handlers/mod.rs index 3a9b418380..9c7e1237bd 100644 --- a/crates/ployz/src/handlers/mod.rs +++ b/crates/ployz/src/handlers/mod.rs @@ -262,6 +262,8 @@ fn handler_for(path: &str) -> Option { "machine rm" => machine::remove, "machine rtt" => machine::rtt, "machine update" => machine::update, + "machine upgrade" => machine::upgrade, + "machine upgrade inspect" => machine::inspect_upgrade, "proxy" => operator::proxy, "project ls" => project::list, "project rm" => project::remove, diff --git a/crates/ployz/tests/cli_shape.rs b/crates/ployz/tests/cli_shape.rs index 31bed43a6b..8eca61d108 100644 --- a/crates/ployz/tests/cli_shape.rs +++ b/crates/ployz/tests/cli_shape.rs @@ -61,6 +61,93 @@ fn native_completion_is_generated_for_every_supported_shell() { } } +#[test] +fn machine_upgrade_requires_explicit_targets_and_has_typed_inspection() { + let command = ployz::cli::command(); + let request = command + .clone() + .try_get_matches_from([ + "ployz", + "machine", + "upgrade", + "1.2.3-beta.4", + "--machine", + "edge-a", + "--machine", + "0123456789abcdef0123456789abcdef", + ]) + .unwrap(); + let upgrade = request + .subcommand_matches("machine") + .unwrap() + .subcommand_matches("upgrade") + .unwrap(); + assert_eq!( + upgrade + .get_one::("version") + .map(ployz_core::MachineRelease::as_str), + Some("1.2.3-beta.4") + ); + assert_eq!( + upgrade + .get_many::("machine") + .unwrap() + .map(String::as_str) + .collect::>(), + ["edge-a", "0123456789abcdef0123456789abcdef"] + ); + assert!( + command + .clone() + .try_get_matches_from(["ployz", "machine", "upgrade", "stable"]) + .is_err() + ); + assert!( + command + .clone() + .try_get_matches_from([ + "ployz", + "machine", + "upgrade", + "nightly", + "--machine", + "edge-a" + ]) + .is_err() + ); + + let inspect = command + .try_get_matches_from([ + "ployz", + "machine", + "upgrade", + "inspect", + "edge-a", + "--attempt", + "0123456789abcdef0123456789abcdef", + "-o", + "json", + ]) + .unwrap(); + let inspect = inspect + .subcommand_matches("machine") + .unwrap() + .subcommand_matches("upgrade") + .unwrap() + .subcommand_matches("inspect") + .unwrap(); + assert_eq!( + inspect + .get_one::("attempt") + .map(ToString::to_string), + Some("0123456789abcdef0123456789abcdef".into()) + ); + assert_eq!( + inspect.get_one::("output").map(String::as_str), + Some("json") + ); +} + #[cfg(unix)] #[test] fn completion_exits_on_sigpipe_when_the_reader_closes_after_one_line() { diff --git a/crates/ployz/tests/cloud_init/harness.rs b/crates/ployz/tests/cloud_init/harness.rs index bf0d5c8a76..8c5e54f562 100644 --- a/crates/ployz/tests/cloud_init/harness.rs +++ b/crates/ployz/tests/cloud_init/harness.rs @@ -494,6 +494,20 @@ impl MachineRpc for JoinDaemon { rpc_ok(CloudPairingSet {}) } + async fn request_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } + + async fn inspect_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } + async fn initialize( &self, request: Request, diff --git a/crates/ployz/tests/connect/support.rs b/crates/ployz/tests/connect/support.rs index 4873315ca7..a24a2c15a6 100644 --- a/crates/ployz/tests/connect/support.rs +++ b/crates/ployz/tests/connect/support.rs @@ -502,6 +502,20 @@ impl MachineRpc for DiscoveryService { Ok(Response::new(RpcResponse::from(report).encode().unwrap())) } + async fn request_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + + async fn inspect_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn inspect_container( &self, _request: Request, diff --git a/crates/ployz/tests/deploy_client/support.rs b/crates/ployz/tests/deploy_client/support.rs index f00074d7a6..3faea8a1cf 100644 --- a/crates/ployz/tests/deploy_client/support.rs +++ b/crates/ployz/tests/deploy_client/support.rs @@ -492,6 +492,18 @@ impl MachineRpc for DeployService { self.record_mutation(); unused() } + async fn request_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } + async fn inspect_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } async fn inspect_container( &self, request: Request, diff --git a/crates/ployzd/src/daemon.rs b/crates/ployzd/src/daemon.rs index 0ee56de050..3b9a68924d 100644 --- a/crates/ployzd/src/daemon.rs +++ b/crates/ployzd/src/daemon.rs @@ -99,7 +99,17 @@ impl Daemon { /// /// If construction, binding, or required planes fail. pub async fn start(config: DaemonConfig) -> Result { - let store = Arc::new(Mutex::new(LocalMachineStore::open(&config.data_dir)?)); + let run_dir = config + .socket + .parent() + .unwrap_or_else(|| Path::new("/run/ployz")); + crate::installer::upgrade::reconcile(&config.data_dir, run_dir) + .await + .map_err(io::Error::other)?; + let store = Arc::new(Mutex::new(LocalMachineStore::open_with_admission( + &config.data_dir, + run_dir, + )?)); let socket_lock = claim_socket(&config.socket)?; let local_record = store .lock() diff --git a/crates/ployzd/src/installer/admission.rs b/crates/ployzd/src/installer/admission.rs new file mode 100644 index 0000000000..f87cff9685 --- /dev/null +++ b/crates/ployzd/src/installer/admission.rs @@ -0,0 +1,149 @@ +//! Cross-process exclusion between installation and Machine-local mutations. + +use std::{ + fs::{self, File, OpenOptions, TryLockError}, + io, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::PathBuf, +}; + +use thiserror::Error; + +const LOCK_FILE: &str = ".install.lock"; +const ACTIVE_FILE: &str = ".upgrade-active"; + +#[derive(Clone, Debug)] +pub struct Admission { + lock_dir: PathBuf, + data_dir: PathBuf, +} + +#[derive(Debug, Error)] +pub enum Error { + #[error("a Ployz installation or upgrade is active")] + Busy, + #[error("Machine admission: {0}")] + Io(#[from] io::Error), +} + +pub struct MutationGuard(File); +pub struct InstallGuard(File); + +impl Admission { + #[must_use] + pub fn new(lock_dir: impl Into, data_dir: impl Into) -> Self { + Self { + lock_dir: lock_dir.into(), + data_dir: data_dir.into(), + } + } + + pub fn try_mutation(&self) -> Result { + let file = self.open_lock()?; + file.try_lock_shared().map_err(lock_error)?; + if self.active_path().is_file() { + let _ = file.unlock(); + return Err(Error::Busy); + } + Ok(MutationGuard(file)) + } + + #[must_use] + pub fn lock_dir(&self) -> &std::path::Path { + &self.lock_dir + } + + pub fn try_install(&self) -> Result { + let file = self.open_lock()?; + file.try_lock().map_err(lock_error)?; + Ok(InstallGuard(file)) + } + + pub fn lock_install(&self) -> Result { + let file = self.open_lock()?; + file.lock()?; + Ok(InstallGuard(file)) + } + + #[must_use] + pub fn active(&self) -> bool { + self.active_path().is_file() + } + + pub(super) fn mark_active(&self, attempt_id: &str) -> Result<(), Error> { + fs::create_dir_all(&self.data_dir)?; + crate::filesystem::atomic_write(&self.active_path(), attempt_id.as_bytes(), 0o600)?; + Ok(()) + } + + pub(super) fn clear_active(&self, attempt_id: &str) -> Result<(), Error> { + let path = self.active_path(); + match fs::read_to_string(&path) { + Ok(active) if active == attempt_id => fs::remove_file(path)?, + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + } + File::open(&self.data_dir)?.sync_all()?; + Ok(()) + } + + fn open_lock(&self) -> Result { + fs::create_dir_all(&self.lock_dir)?; + fs::set_permissions(&self.lock_dir, fs::Permissions::from_mode(0o750))?; + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .mode(0o600) + .open(self.lock_dir.join(LOCK_FILE)) + .map_err(Into::into) + } + + fn active_path(&self) -> PathBuf { + self.data_dir.join(ACTIVE_FILE) + } +} + +fn lock_error(error: TryLockError) -> Error { + match error { + TryLockError::WouldBlock => Error::Busy, + TryLockError::Error(error) => Error::Io(error), + } +} + +impl Drop for MutationGuard { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +impl Drop for InstallGuard { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installation_and_active_handoff_exclude_mutations_across_instances() { + let root = crate::test_dir::TestDir::new("installer-admission"); + let first = Admission::new(root.0.join("run"), root.0.join("data")); + let second = Admission::new(root.0.join("run"), root.0.join("data")); + + let install = first.try_install().unwrap(); + assert!(matches!(second.try_mutation(), Err(Error::Busy))); + first.mark_active("attempt").unwrap(); + drop(install); + assert!(matches!(second.try_mutation(), Err(Error::Busy))); + + let install = second.try_install().unwrap(); + second.clear_active("attempt").unwrap(); + drop(install); + assert!(second.try_mutation().is_ok()); + } +} diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index f5cd250156..f06f346820 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -1,18 +1,18 @@ //! Bounded local installation of a Ployz Machine release. +pub mod admission; mod host; mod release; mod storage; +pub mod upgrade; use std::{ - fs::{self, File, OpenOptions}, io, - os::unix::fs::{OpenOptionsExt, PermissionsExt}, + os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::{Command, Output}, }; -use fs2::FileExt; use ployz_core::StorageChoice; use thiserror::Error; @@ -28,6 +28,21 @@ use self::{ pub use self::release::{ReleaseRequest, ReleaseSource}; const PLOYZ_USER: &str = "ployz"; +const DEFAULT_BIN_DIR: &str = "/usr/local/bin"; +const DEFAULT_SYSTEMD_DIR: &str = "/etc/systemd/system"; +const DEFAULT_DATA_DIR: &str = "/var/lib/ployz"; +const DEFAULT_RUN_DIR: &str = "/run/ployz"; + +/// Observable stage of the shared installation implementation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum InstallStage { + Preparing, + Acquiring, + Verifying, + Activating, + Restarting, + Readiness, +} /// Explicit host work associated with one Machine installation attempt. #[derive(Clone, Debug, Eq, PartialEq)] @@ -116,12 +131,12 @@ pub(super) struct InstallPaths { } impl InstallPaths { - fn system() -> Self { + fn system(data_dir: impl Into, run_dir: impl Into) -> Self { Self { - bin_dir: PathBuf::from("/usr/local/bin"), - systemd_dir: PathBuf::from("/etc/systemd/system"), - data_dir: PathBuf::from("/var/lib/ployz"), - run_dir: PathBuf::from("/run/ployz"), + bin_dir: PathBuf::from(DEFAULT_BIN_DIR), + systemd_dir: PathBuf::from(DEFAULT_SYSTEMD_DIR), + data_dir: data_dir.into(), + run_dir: run_dir.into(), docker_config: PathBuf::from("/etc/docker/daemon.json"), modprobe_dir: PathBuf::from("/etc/modprobe.d"), } @@ -140,16 +155,6 @@ impl InstallPaths { } } -struct InstallLock(File); - -impl Drop for InstallLock { - fn drop(&mut self) { - // Forked children can inherit this file descriptor before exec; unlock the shared lock - // explicitly instead of waiting for every inherited descriptor to close. - let _ = self.0.unlock(); - } -} - /// Install the selected release through the one Machine-local installation seam. /// /// # Errors @@ -157,15 +162,42 @@ impl Drop for InstallLock { /// Returns the observed first failed installation stage. Acquisition and preflight failures /// happen before activation, so the installed daemon remains untouched. pub async fn install(request: InstallRequest) -> Result { - install_at(request, InstallPaths::system()).await + install_in(request, DEFAULT_DATA_DIR, DEFAULT_RUN_DIR).await +} + +/// Install using the daemon's configured data and runtime directories. +/// +/// # Errors +/// +/// Returns the observed first failed installation stage. +pub async fn install_in( + request: InstallRequest, + data_dir: impl Into, + run_dir: impl Into, +) -> Result { + install_at(request, InstallPaths::system(data_dir, run_dir)).await } async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result { require_root()?; - let _lock = claim_lock(&paths.run_dir)?; + let admission = admission::Admission::new(&paths.run_dir, &paths.data_dir); + let lock = admission.try_install().map_err(map_admission_error)?; + upgrade::reconcile_for_install(&admission, &paths.data_dir) + .await + .map_err(map_upgrade_reconciliation)?; + install_locked(request, paths, lock, |_| Ok(())).await +} + +async fn install_locked( + request: InstallRequest, + paths: InstallPaths, + _lock: admission::InstallGuard, + mut progress: impl FnMut(InstallStage) -> Result<(), Error>, +) -> Result { verify_system(request.install_only)?; let target = resolve_release(&request.release, &request.source).await?; + progress(InstallStage::Preparing)?; match &request.preparation { Preparation::SoftwareOnly => verify_software_prerequisites(&paths)?, Preparation::PrepareHost { @@ -183,7 +215,7 @@ async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result Result Result Error { + match error { + admission::Error::Busy => Error::Busy, + admission::Error::Io(source) => Error::Io { + stage: "claim Machine installation admission", + source, + }, + } +} + +fn map_upgrade_reconciliation(error: upgrade::Error) -> Error { + if matches!(error, upgrade::Error::Busy) { + Error::Busy + } else { + Error::Command { + stage: "reconcile previous Machine upgrade".into(), + message: error.to_string(), + } + } +} + fn require_root() -> Result<(), Error> { let mut command = Command::new("id"); command.arg("-u"); @@ -219,40 +274,6 @@ fn require_root() -> Result<(), Error> { } } -fn claim_lock(run_dir: &Path) -> Result { - fs::create_dir_all(run_dir).map_err(|source| Error::Io { - stage: "create installation lock directory", - source, - })?; - fs::set_permissions(run_dir, fs::Permissions::from_mode(0o750)).map_err(|source| { - Error::Io { - stage: "set installation lock directory permissions", - source, - } - })?; - let file = OpenOptions::new() - .create(true) - .truncate(false) - .write(true) - .mode(0o600) - .open(run_dir.join(".install.lock")) - .map_err(|source| Error::Io { - stage: "open installation lock", - source, - })?; - file.try_lock_exclusive().map_err(|source| { - if source.kind() == io::ErrorKind::WouldBlock { - Error::Busy - } else { - Error::Io { - stage: "lock installation", - source, - } - } - })?; - Ok(InstallLock(file)) -} - fn verify_system(install_only: bool) -> Result<(), Error> { if std::env::consts::OS != "linux" { return Err(Error::UnsupportedOs); @@ -351,6 +372,7 @@ mod tests { use std::{ env, ffi::OsString, + fs, io::Write, os::unix::fs::{PermissionsExt, symlink}, process::Command, @@ -452,7 +474,8 @@ mod tests { install_only: true, }; let result = if case == "busy" { - let _held = claim_lock(&paths.run_dir).unwrap(); + let admission = admission::Admission::new(&paths.run_dir, &paths.data_dir); + let _held = admission.try_install().unwrap(); install_at(request, paths.clone()).await } else { install_at(request, paths.clone()).await diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs index 71c614ac19..436685cfe8 100644 --- a/crates/ployzd/src/installer/release.rs +++ b/crates/ployzd/src/installer/release.rs @@ -19,7 +19,7 @@ use serde::Deserialize; use sha2::{Digest, Sha256}; use tokio::{process::Command, time::timeout}; -use super::{Error, InstallPaths, daemon_archive, run_command}; +use super::{Error, InstallPaths, InstallStage, daemon_archive, run_command}; const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; const CHANNEL_URL: &str = "https://ployz.sh"; @@ -170,6 +170,7 @@ pub(super) async fn install_binaries( source: &ReleaseSource, paths: &InstallPaths, target: &Version, + progress: &mut impl FnMut(InstallStage) -> Result<(), Error>, ) -> Result { let installed = installed_release(&paths.bin_dir.join("ployzd")).await?; let replace = replacement_required(source, installed.as_ref(), target); @@ -181,6 +182,7 @@ pub(super) async fn install_binaries( return Ok(false); } + progress(InstallStage::Acquiring)?; let archive = daemon_archive()?; let stage = Staging::new(&paths.bin_dir)?; let archive_path = stage.path.join(archive); @@ -189,6 +191,7 @@ pub(super) async fn install_binaries( .release_file(target, archive, "download daemon archive") .await?; write_private(&archive_path, &archive_bytes, "stage daemon archive")?; + progress(InstallStage::Verifying)?; verify_checksum(&archive_path, &checksum)?; extract_archive(&archive_path, &stage.path)?; let daemon = stage.path.join("ployzd"); @@ -196,6 +199,7 @@ pub(super) async fn install_binaries( verify_executable(&daemon, target).await?; verify_uninstall(&uninstall)?; sync_staged_files(&daemon, &uninstall, &stage.path)?; + progress(InstallStage::Activating)?; activate(&daemon, &uninstall, paths)?; Ok(true) } diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs new file mode 100644 index 0000000000..f00935a52c --- /dev/null +++ b/crates/ployzd/src/installer/upgrade.rs @@ -0,0 +1,449 @@ +//! Durable, bounded Machine upgrade attempts run by transient systemd services. + +use std::{ + env, fs, io, + path::{Path, PathBuf}, + time::Duration, +}; + +use ployz_core::{ + MachineRelease, MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeStage, + MachineVersion, RequestMachineUpgradeRequest, +}; +use semver::Version; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{process::Command, time::timeout}; + +use super::{ + Error as InstallError, InstallPaths, InstallRequest, InstallStage, Preparation, ReleaseRequest, + ReleaseSource, admission, +}; + +const RECEIPT_FILE: &str = "upgrade-attempt.json"; +const QUALIFICATION_RELEASE_DIR: &str = "PLOYZ_UPGRADE_RELEASE_DIR"; +const WORKER_RUNTIME: &str = "15min"; +const LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); + +#[derive(Debug, Error)] +pub enum Error { + #[error("no Machine upgrade attempt was found")] + NotFound, + #[error("Machine upgrade attempt {0} was reused with a different release")] + AttemptConflict(MachineUpgradeAttemptId), + #[error("a Machine upgrade or mutation is active")] + Busy, + #[error("read Machine upgrade receipt: {0}")] + Read(#[source] io::Error), + #[error("decode Machine upgrade receipt: {0}")] + Decode(#[source] serde_json::Error), + #[error("write Machine upgrade receipt: {0}")] + Write(#[source] io::Error), + #[error("encode Machine upgrade receipt: {0}")] + Encode(#[source] serde_json::Error), + #[error("invalid qualification release directory: {0}")] + QualificationSource(String), + #[error("resolve Machine release: {0}")] + Resolve(#[source] InstallError), + #[error("launch Machine upgrade worker: {0}")] + Launch(String), + #[error("inspect Machine upgrade worker: {0}")] + InspectWorker(#[source] io::Error), + #[error("Machine upgrade worker does not own active attempt {0}")] + NotActive(MachineUpgradeAttemptId), + #[error("Machine upgrade failed: {0}")] + Installation(#[source] InstallError), + #[error(transparent)] + Admission(#[from] admission::Error), +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct StoredAttempt { + requested: MachineRelease, + source: StoredReleaseSource, + attempt: MachineUpgradeAttempt, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum StoredReleaseSource { + Published, + Local { directory: PathBuf }, +} + +impl StoredReleaseSource { + fn current() -> Result { + let Some(directory) = env::var_os(QUALIFICATION_RELEASE_DIR) else { + return Ok(Self::Published); + }; + let directory = PathBuf::from(directory); + if !directory.is_absolute() { + return Err(Error::QualificationSource(format!( + "{QUALIFICATION_RELEASE_DIR} must be an absolute path" + ))); + } + Ok(Self::Local { directory }) + } + + fn installer(&self) -> ReleaseSource { + match self { + Self::Published => ReleaseSource::Published, + Self::Local { directory } => ReleaseSource::Local(directory.clone()), + } + } +} + +/// Persist an accepted attempt and launch its old-daemon worker while installation admission is +/// held. Reusing the same attempt ID returns the same attempt. +pub async fn request_locked( + request: RequestMachineUpgradeRequest, + data_dir: &Path, + run_dir: &Path, + _guard: &admission::InstallGuard, +) -> Result { + let admission = admission::Admission::new(run_dir, data_dir); + if let Some(mut stored) = read_optional(data_dir)? { + if stored.attempt.attempt_id() == request.attempt_id { + if stored.requested != request.release { + return Err(Error::AttemptConflict(request.attempt_id)); + } + reconcile_locked(&admission, data_dir, &mut stored).await?; + return Ok(stored.attempt); + } + reconcile_locked(&admission, data_dir, &mut stored).await?; + if !stored.attempt.is_terminal() { + return Err(Error::Busy); + } + } + + let source = StoredReleaseSource::current()?; + let release = request + .release + .as_str() + .parse::() + .map_err(Error::Resolve)?; + let target = super::release::resolve_release(&release, &source.installer()) + .await + .map_err(Error::Resolve)?; + let target = MachineVersion::parse(target.to_string()) + .expect("the installer accepts only supported Machine versions"); + let mut stored = StoredAttempt { + requested: request.release, + source, + attempt: MachineUpgradeAttempt::Accepted { + attempt_id: request.attempt_id, + target, + }, + }; + write(data_dir, &stored)?; + admission.mark_active(request.attempt_id.as_str())?; + + if let Err(error) = launch_worker(request.attempt_id, data_dir, run_dir).await { + stored.attempt = MachineUpgradeAttempt::Failed { + attempt_id: request.attempt_id, + target: stored.attempt.target().clone(), + stage: MachineUpgradeStage::Launching, + error: error.to_string(), + }; + write(data_dir, &stored)?; + admission.clear_active(request.attempt_id.as_str())?; + } + Ok(stored.attempt) +} + +/// Read an attempt, turning a stopped worker without terminal evidence into interruption. +pub async fn inspect( + attempt_id: Option, + data_dir: &Path, + run_dir: &Path, +) -> Result { + let admission = admission::Admission::new(run_dir, data_dir); + let mut stored = read(data_dir)?; + if attempt_id.is_some_and(|attempt_id| attempt_id != stored.attempt.attempt_id()) { + return Err(Error::NotFound); + } + if stored.attempt.is_terminal() { + if let Ok(_guard) = admission.try_install() { + admission.clear_active(stored.attempt.attempt_id().as_str())?; + } + return Ok(stored.attempt); + } + if worker_active(stored.attempt.attempt_id()).await? { + return Ok(stored.attempt); + } + let Ok(_guard) = admission.try_install() else { + return Ok(stored.attempt); + }; + stored = read(data_dir)?; + reconcile_locked(&admission, data_dir, &mut stored).await?; + Ok(stored.attempt) +} + +/// Run the accepted attempt from the transient systemd service. +pub async fn run_worker( + attempt_id: MachineUpgradeAttemptId, + data_dir: &Path, + run_dir: &Path, +) -> Result<(), Error> { + let admission = admission::Admission::new(run_dir, data_dir); + let guard = admission.lock_install()?; + let mut stored = read(data_dir)?; + if stored.attempt.attempt_id() != attempt_id || stored.attempt.is_terminal() { + return Err(Error::NotActive(attempt_id)); + } + let target = stored.attempt.target().clone(); + let target_version = Version::parse(target.as_str()) + .expect("a MachineVersion contains a supported semantic version"); + let mut stage = MachineUpgradeStage::Preparing; + stored.attempt = MachineUpgradeAttempt::Running { + attempt_id, + target: target.clone(), + stage: stage.clone(), + }; + write(data_dir, &stored)?; + + let result = super::install_locked( + InstallRequest { + release: ReleaseRequest::Exact(target_version), + source: stored.source.installer(), + preparation: Preparation::SoftwareOnly, + install_only: false, + }, + InstallPaths::system(data_dir, run_dir), + guard, + |install_stage| { + stage = upgrade_stage(install_stage); + stored.attempt = MachineUpgradeAttempt::Running { + attempt_id, + target: target.clone(), + stage: stage.clone(), + }; + write(data_dir, &stored).map_err(|error| InstallError::Io { + stage: "record Machine upgrade progress", + source: io::Error::other(error), + }) + }, + ) + .await; + + match result { + Ok(_) => { + stored.attempt = MachineUpgradeAttempt::Succeeded { + attempt_id, + version: target, + }; + write(data_dir, &stored)?; + admission.clear_active(attempt_id.as_str())?; + Ok(()) + } + Err(error) => { + stored.attempt = MachineUpgradeAttempt::Failed { + attempt_id, + target, + stage, + error: error.to_string(), + }; + write(data_dir, &stored)?; + admission.clear_active(attempt_id.as_str())?; + Err(Error::Installation(error)) + } + } +} + +/// Reconcile a retained receipt before serving requests after daemon restart. +pub async fn reconcile(data_dir: &Path, run_dir: &Path) -> Result<(), Error> { + let admission = admission::Admission::new(run_dir, data_dir); + let Some(mut stored) = read_optional(data_dir)? else { + return Ok(()); + }; + if stored.attempt.is_terminal() { + if let Ok(_guard) = admission.try_install() { + admission.clear_active(stored.attempt.attempt_id().as_str())?; + } + return Ok(()); + } + let Ok(_guard) = admission.try_install() else { + return Ok(()); + }; + reconcile_locked(&admission, data_dir, &mut stored).await +} + +pub(super) async fn reconcile_for_install( + admission: &admission::Admission, + data_dir: &Path, +) -> Result<(), Error> { + let Some(mut stored) = read_optional(data_dir)? else { + return Ok(()); + }; + reconcile_locked(admission, data_dir, &mut stored).await?; + if stored.attempt.is_terminal() { + Ok(()) + } else { + Err(Error::Busy) + } +} + +async fn reconcile_locked( + admission: &admission::Admission, + data_dir: &Path, + stored: &mut StoredAttempt, +) -> Result<(), Error> { + if stored.attempt.is_terminal() { + admission.clear_active(stored.attempt.attempt_id().as_str())?; + return Ok(()); + } + if worker_active(stored.attempt.attempt_id()).await? { + return Ok(()); + } + let stage = match &stored.attempt { + MachineUpgradeAttempt::Accepted { .. } => MachineUpgradeStage::Launching, + MachineUpgradeAttempt::Running { stage, .. } => stage.clone(), + MachineUpgradeAttempt::Succeeded { .. } + | MachineUpgradeAttempt::Failed { .. } + | MachineUpgradeAttempt::Interrupted { .. } => return Ok(()), + }; + stored.attempt = MachineUpgradeAttempt::Interrupted { + attempt_id: stored.attempt.attempt_id(), + target: stored.attempt.target().clone(), + stage, + }; + write(data_dir, stored)?; + admission.clear_active(stored.attempt.attempt_id().as_str())?; + Ok(()) +} + +async fn launch_worker( + attempt_id: MachineUpgradeAttemptId, + data_dir: &Path, + run_dir: &Path, +) -> Result<(), Error> { + let executable = env::current_exe().map_err(Error::InspectWorker)?; + let unit = unit_name(attempt_id); + let writable = format!( + "ReadWritePaths=/usr/local/bin /etc/systemd/system {} {}", + data_dir.display(), + run_dir.display() + ); + let mut command = Command::new("systemd-run"); + command + .arg(format!("--unit={unit}")) + .args(["--collect", "--quiet"]) + .arg("--property=Type=exec") + .arg(format!("--property=RuntimeMaxSec={WORKER_RUNTIME}")) + .arg("--property=TimeoutStopSec=15s") + .arg("--property=KillMode=control-group") + .arg("--property=NoNewPrivileges=yes") + .arg("--property=ProtectSystem=full") + .arg("--property=ProtectHome=yes") + .arg("--property=ProtectControlGroups=yes") + .arg("--property=ProtectKernelTunables=yes") + .arg("--property=PrivateTmp=yes") + .arg("--property=RestrictNamespaces=yes") + .arg("--property=RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX") + .arg(format!("--property={writable}")) + .arg(executable) + .arg("--data-dir") + .arg(data_dir) + .arg("--socket") + .arg(run_dir.join("ployz.sock")) + .arg("upgrade-worker") + .arg("--attempt") + .arg(attempt_id.as_str()) + .kill_on_drop(true); + let output = timeout(LAUNCH_TIMEOUT, command.output()) + .await + .map_err(|_| { + Error::Launch(format!( + "systemd-run timed out after {}s", + LAUNCH_TIMEOUT.as_secs() + )) + })? + .map_err(Error::InspectWorker)?; + if output.status.success() { + Ok(()) + } else { + let detail = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + Err(Error::Launch(if detail.is_empty() { + format!("systemd-run exited with {}", output.status) + } else { + detail + })) + } +} + +async fn worker_active(attempt_id: MachineUpgradeAttemptId) -> Result { + let output = Command::new("systemctl") + .args(["is-active", "--quiet", &unit_name(attempt_id)]) + .output() + .await + .map_err(Error::InspectWorker)?; + Ok(output.status.success()) +} + +fn unit_name(attempt_id: MachineUpgradeAttemptId) -> String { + format!("ployz-upgrade-{attempt_id}.service") +} + +fn upgrade_stage(stage: InstallStage) -> MachineUpgradeStage { + match stage { + InstallStage::Preparing => MachineUpgradeStage::Preparing, + InstallStage::Acquiring => MachineUpgradeStage::Acquiring, + InstallStage::Verifying => MachineUpgradeStage::Verifying, + InstallStage::Activating => MachineUpgradeStage::Activating, + InstallStage::Restarting => MachineUpgradeStage::Restarting, + InstallStage::Readiness => MachineUpgradeStage::Readiness, + } +} + +fn read(data_dir: &Path) -> Result { + read_optional(data_dir)?.ok_or(Error::NotFound) +} + +fn read_optional(data_dir: &Path) -> Result, Error> { + let bytes = match fs::read(data_dir.join(RECEIPT_FILE)) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(Error::Read(error)), + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(Error::Decode) +} + +fn write(data_dir: &Path, stored: &StoredAttempt) -> Result<(), Error> { + fs::create_dir_all(data_dir).map_err(Error::Write)?; + let bytes = serde_json::to_vec(stored).map_err(Error::Encode)?; + crate::filesystem::atomic_write(&data_dir.join(RECEIPT_FILE), &bytes, 0o600) + .map_err(Error::Write) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stored_attempt_round_trips_without_exposing_local_source_in_response() { + let attempt_id = MachineUpgradeAttemptId::random(); + let stored = StoredAttempt { + requested: MachineRelease::parse("beta").unwrap(), + source: StoredReleaseSource::Local { + directory: "/root/qualification".into(), + }, + attempt: MachineUpgradeAttempt::Accepted { + attempt_id, + target: MachineVersion::parse("1.2.3-beta.4").unwrap(), + }, + }; + let encoded = serde_json::to_string(&stored).unwrap(); + assert!(encoded.contains("/root/qualification")); + assert!( + !serde_json::to_string(&stored.attempt) + .unwrap() + .contains("qualification") + ); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + stored + ); + } +} diff --git a/crates/ployzd/src/machine/local_machine.rs b/crates/ployzd/src/machine/local_machine.rs index 6014870719..76f6f83e13 100644 --- a/crates/ployzd/src/machine/local_machine.rs +++ b/crates/ployzd/src/machine/local_machine.rs @@ -18,7 +18,7 @@ use ployz_core::{ synthesize_membership, }; use thiserror::Error; -use tokio::sync::watch; +use tokio::sync::{OwnedMutexGuard, watch}; use super::{FoundingCluster, LocalMachineRecord, LocalMachineStore, StoreError, local_runtime}; @@ -120,6 +120,20 @@ pub enum Error { NotAllocator, #[error("this Machine is isolation-locked")] IsolationLocked, + #[error(transparent)] + Admission(#[from] crate::installer::admission::Error), +} + +pub(crate) struct UpgradeAdmission { + pub(crate) data_dir: std::path::PathBuf, + pub(crate) run_dir: std::path::PathBuf, + pub(crate) install: crate::installer::admission::InstallGuard, + _local: OwnedMutexGuard<()>, +} + +struct MutationAdmission { + _local: OwnedMutexGuard<()>, + _installation: crate::installer::admission::MutationGuard, } impl LocalMachine { @@ -180,6 +194,65 @@ impl LocalMachine { self.store.lock().map_err(|_| Error::LockPoisoned) } + async fn admit_mutation(&self) -> Result { + let (local, installation) = { + let store = self.lock_store()?; + ( + store.admission_lock.clone(), + store.installation_admission.clone(), + ) + }; + let local = local.lock_owned().await; + let installation = installation.try_mutation()?; + Ok(MutationAdmission { + _local: local, + _installation: installation, + }) + } + + pub(crate) async fn run_mutation(&self, work: F) -> Result + where + T: Send + 'static, + F: std::future::Future> + Send + 'static, + { + let admission = self.admit_mutation().await?; + tokio::spawn(async move { + let _admission = admission; + work.await + }) + .await? + } + + pub(crate) fn try_upgrade_admission(&self) -> Result { + let (data_dir, run_dir, local, installation) = { + let store = self.lock_store()?; + ( + store.data_dir.clone(), + store.installation_admission.lock_dir().to_owned(), + store.admission_lock.clone(), + store.installation_admission.clone(), + ) + }; + let local = local + .try_lock_owned() + .map_err(|_| crate::installer::admission::Error::Busy)?; + let install = installation.try_install()?; + Ok(UpgradeAdmission { + data_dir, + run_dir, + install, + _local: local, + }) + } + + pub(crate) fn upgrade_paths(&self) -> Result<(std::path::PathBuf, std::path::PathBuf), Error> { + let store = self.lock_store()?; + Ok(( + store.data_dir.clone(), + store.installation_admission.lock_dir().to_owned(), + )) + } + pub(crate) fn replicated(&self) -> Result<&ReplicatedStore, Error> { self.cluster .as_ref() @@ -367,6 +440,15 @@ impl LocalMachine { Ok(Initialized { machine }) } + pub(crate) async fn initialize_mutation( + &self, + request: InitializeRequest, + ) -> Result { + let local = self.clone(); + self.run_mutation(async move { local.initialize(request) }) + .await + } + /// Assign a new Machine into the Cluster from this participating Machine /// when cluster KV names this Machine as Allocator. A replica row with the /// same public key and name is returned as-is, without publishing. Replay @@ -387,6 +469,12 @@ impl LocalMachine { /// another Machine or is missing, [`Error::Network`] when subnet allocation /// fails, and [`Error::Cluster`] when replicated I/O fails. pub async fn register(&self, request: RegisterRequest) -> Result { + let local = self.clone(); + self.run_mutation(async move { local.register_admitted(request).await }) + .await + } + + async fn register_admitted(&self, request: RegisterRequest) -> Result { let me = { let record = self.record()?; if record.phase() != LocalMachinePhase::Participating { @@ -484,7 +572,6 @@ impl LocalMachine { advertised_endpoints: request.advertised_endpoints, runtime: request.runtime, }; - // TODO: cross-process registration stays unfenced and has no rollback. publication.publish(&assigned_machine).await?; assigned_machine } @@ -525,6 +612,11 @@ impl LocalMachine { Ok(JoinAccepted {}) } + pub(crate) async fn join_mutation(&self, request: JoinRequest) -> Result { + let local = self.clone(); + self.run_mutation(async move { local.join(request) }).await + } + /// Persist Cloud Pairing so this Machine can hold Relay Register, or clear /// it so Register is dropped. /// @@ -542,6 +634,15 @@ impl LocalMachine { Ok(()) } + pub(crate) async fn set_cloud_pairing_mutation( + &self, + pairing: Option, + ) -> Result<(), Error> { + let local = self.clone(); + self.run_mutation(async move { local.set_cloud_pairing(pairing) }) + .await + } + /// Membership Observation of Machines visible from this participating Machine. /// /// # Errors @@ -583,6 +684,15 @@ impl LocalMachine { /// [`Error::Store`] when the update is not legal, and [`Error::Cluster`] /// when listing visible Machines fails. pub async fn update(&self, request: UpdateMachineRequest) -> Result { + let local = self.clone(); + self.run_mutation(async move { local.update_admitted(request).await }) + .await + } + + async fn update_admitted( + &self, + request: UpdateMachineRequest, + ) -> Result { if request.update.is_empty() { return Err(Error::EmptyUpdate); } @@ -607,6 +717,15 @@ impl LocalMachine { pub async fn remove_peer( &self, request: RemoveMachineRequest, + ) -> Result { + let local = self.clone(); + self.run_mutation(async move { local.remove_peer_admitted(request).await }) + .await + } + + async fn remove_peer_admitted( + &self, + request: RemoveMachineRequest, ) -> Result { self.replicated()? .remove_machine(&request.machine_id) @@ -634,14 +753,9 @@ impl LocalMachine { &self, request: RemoveLocalMachineRequest, ) -> Result { - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let local = self.clone(); - tokio::spawn(async move { - let _guard = guard; - local.remove_local_admitted(request).await - }) - .await? + self.run_mutation(async move { local.remove_local_admitted(request).await }) + .await } async fn remove_local_admitted( @@ -690,14 +804,9 @@ impl LocalMachine { /// [`Error::Store`] when reset is not legal in the current phase, and /// [`Error::OperationTask`] if the admitted operation task fails. pub async fn reset(&self) -> Result { - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let local = self.clone(); - tokio::spawn(async move { - let _guard = guard; - local.reset_admitted().await - }) - .await? + self.run_mutation(async move { local.reset_admitted().await }) + .await } async fn reset_admitted(&self) -> Result { diff --git a/crates/ployzd/src/machine/local_machine/container.rs b/crates/ployzd/src/machine/local_machine/container.rs index d7d33c340b..fbf960f6fd 100644 --- a/crates/ployzd/src/machine/local_machine/container.rs +++ b/crates/ployzd/src/machine/local_machine/container.rs @@ -24,11 +24,8 @@ impl LocalMachine { specs: Vec, ) -> Result { use ployz_core::{RawVolumeSource, RpcError}; - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let local = self.clone(); - tokio::spawn(async move { - let _guard = guard; + self.run_mutation(async move { let containers = local.containers.as_ref().ok_or(Error::DockerUnavailable)?; let record = local.record()?; if !matches!( @@ -79,7 +76,7 @@ impl LocalMachine { })?; Ok(ployz_core::PreparedVolumes { names }) }) - .await? + .await } /// Create a container after storage admission and deferred Machine-local validation. @@ -95,14 +92,11 @@ impl LocalMachine { spec: &ResolvedServiceSpec, ) -> Result { // Once admitted, caller cancellation must not let reset overtake a Docker request. - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let (local, project, spec) = (self.clone(), project.clone(), spec.clone()); - tokio::spawn(async move { - let _guard = guard; - local.create_container_admitted(kind, &project, &spec).await - }) - .await? + self.run_mutation( + async move { local.create_container_admitted(kind, &project, &spec).await }, + ) + .await } async fn create_container_admitted( @@ -145,14 +139,9 @@ impl LocalMachine { project: &ProjectName, spec: &ResolvedServiceSpec, ) -> Result { - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let (local, project, spec) = (self.clone(), project.clone(), spec.clone()); - tokio::spawn(async move { - let _guard = guard; - local.converge_global_slot_admitted(&project, &spec).await - }) - .await? + self.run_mutation(async move { local.converge_global_slot_admitted(&project, &spec).await }) + .await } async fn converge_global_slot_admitted( diff --git a/crates/ployzd/src/machine/mod.rs b/crates/ployzd/src/machine/mod.rs index 645d8b010b..b65e7ff7f2 100644 --- a/crates/ployzd/src/machine/mod.rs +++ b/crates/ployzd/src/machine/mod.rs @@ -363,6 +363,7 @@ pub struct LocalMachineStore { // Reset waits for admitted operations, including Docker streams with no total deadline. // ponytail: serialize local creates; use shared admission reads if throughput requires it. admission_lock: Arc>, + installation_admission: crate::installer::admission::Admission, } pub(crate) struct PreparedReset { @@ -407,7 +408,16 @@ impl Drop for LocalMachineStore { impl LocalMachineStore { pub fn open(data_dir: impl AsRef) -> Result { + let data_dir = data_dir.as_ref(); + Self::open_with_admission(data_dir, data_dir.join(".run")) + } + + pub(crate) fn open_with_admission( + data_dir: impl AsRef, + run_dir: impl AsRef, + ) -> Result { let data_dir = data_dir.as_ref().to_owned(); + let run_dir = run_dir.as_ref().to_owned(); validate_data_dir(&data_dir)?; claim_data_dir(&data_dir)?; fs::create_dir_all(&data_dir)?; @@ -450,16 +460,19 @@ impl LocalMachineStore { }; let mut store = Self { - data_dir, + data_dir: data_dir.clone(), record, _lock: lock, admission_lock: Arc::new(tokio::sync::Mutex::new(())), + installation_admission: crate::installer::admission::Admission::new( + &run_dir, &data_dir, + ), }; if store.record.phase() == LocalMachinePhase::Resetting { let data_dir = store.data_dir.clone(); store.complete_reset()?; drop(store); - return Self::open(data_dir); + return Self::open_with_admission(data_dir, run_dir); } store.refresh_runtime()?; Ok(store) diff --git a/crates/ployzd/src/machine_api/local.rs b/crates/ployzd/src/machine_api/local.rs index 55d44d06c4..8cdae79a50 100644 --- a/crates/ployzd/src/machine_api/local.rs +++ b/crates/ployzd/src/machine_api/local.rs @@ -302,7 +302,11 @@ impl MachineRpc for MachineService { &self, request: Request, ) -> Result, Status> { - finish(self.local.initialize(expect::(request)?)) + finish( + self.local + .initialize_mutation(expect::(request)?) + .await, + ) } async fn register( @@ -331,7 +335,7 @@ impl MachineRpc for MachineService { &self, request: Request, ) -> Result, Status> { - finish(self.local.join(expect::(request)?)) + finish(self.local.join_mutation(expect::(request)?).await) } async fn set_cloud_pairing( @@ -339,7 +343,11 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - if let Err(error) = self.local.set_cloud_pairing(request.cloud_pairing.clone()) { + if let Err(error) = self + .local + .set_cloud_pairing_mutation(request.cloud_pairing.clone()) + .await + { return local_error(error); } if let Some(sender) = &self.cloud_pairing { @@ -448,15 +456,18 @@ impl MachineRpc for MachineService { ) -> Result, Status> { let request = expect::(request)?; let containers = match self.containers() { - Ok(containers) => containers, + Ok(containers) => containers.clone(), Err(error) => return respond(error), }; - match containers.start(&request.container_id).await { - Ok(()) => respond(ContainerChanged { - container_id: request.container_id, - }), - Err(error) => respond(RpcError::from(&error)), - } + let container_id = request.container_id; + finish( + self.local + .run_mutation(async move { + containers.start(&container_id).await?; + Ok(ContainerChanged { container_id }) + }) + .await, + ) } async fn stop_container( @@ -465,22 +476,24 @@ impl MachineRpc for MachineService { ) -> Result, Status> { let request = expect::(request)?; let containers = match self.containers() { - Ok(containers) => containers, + Ok(containers) => containers.clone(), Err(error) => return respond(error), }; - match containers - .stop( - &request.container_id, - request.signal.as_deref(), - request.grace_period_seconds, - ) - .await - { - Ok(()) => respond(ContainerChanged { - container_id: request.container_id, - }), - Err(error) => respond(RpcError::from(&error)), - } + let container_id = request.container_id; + finish( + self.local + .run_mutation(async move { + containers + .stop( + &container_id, + request.signal.as_deref(), + request.grace_period_seconds, + ) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await, + ) } async fn remove_container( @@ -489,18 +502,20 @@ impl MachineRpc for MachineService { ) -> Result, Status> { let request = expect::(request)?; let containers = match self.containers() { - Ok(containers) => containers, + Ok(containers) => containers.clone(), Err(error) => return respond(error), }; - match containers - .remove(&request.container_id, request.remove_volumes, request.force) - .await - { - Ok(()) => respond(ContainerChanged { - container_id: request.container_id, - }), - Err(error) => respond(RpcError::from(&error)), - } + let container_id = request.container_id; + finish( + self.local + .run_mutation(async move { + containers + .remove(&container_id, request.remove_volumes, request.force) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await, + ) } async fn create_volume( @@ -510,13 +525,16 @@ impl MachineRpc for MachineService { let request = expect::(request)?; let machine_id = self.local_record()?.id(); let containers = match self.containers() { - Ok(containers) => containers, + Ok(containers) => containers.clone(), Err(error) => return respond(error), }; - match containers.create_volume(&machine_id, request).await { - Ok(volume) => respond(volume), - Err(error) => respond(RpcError::from(&error)), - } + finish( + self.local + .run_mutation( + async move { Ok(containers.create_volume(&machine_id, request).await?) }, + ) + .await, + ) } async fn inspect_storage( @@ -576,13 +594,19 @@ impl MachineRpc for MachineService { ) -> Result, Status> { let request = expect::(request)?; let containers = match self.containers() { - Ok(containers) => containers, + Ok(containers) => containers.clone(), Err(error) => return respond(error), }; - match containers.remove_volume(&request.name, request.force).await { - Ok(()) => respond(VolumeRemoved {}), - Err(error) => respond(RpcError::from(&error)), - } + finish( + self.local + .run_mutation(async move { + containers + .remove_volume(&request.name, request.force) + .await?; + Ok(VolumeRemoved {}) + }) + .await, + ) } async fn exec( @@ -680,6 +704,43 @@ impl MachineRpc for MachineService { ) } + async fn request_machine_upgrade( + &self, + request: Request, + ) -> Result, Status> { + let request = expect::(request)?; + let admission = match self.local.try_upgrade_admission() { + Ok(admission) => admission, + Err(error) => return local_error(error), + }; + match crate::installer::upgrade::request_locked( + request, + &admission.data_dir, + &admission.run_dir, + &admission.install, + ) + .await + { + Ok(attempt) => respond(attempt), + Err(error) => respond(upgrade_error(error)), + } + } + + async fn inspect_machine_upgrade( + &self, + request: Request, + ) -> Result, Status> { + let request = expect::(request)?; + let (data_dir, run_dir) = match self.local.upgrade_paths() { + Ok(paths) => paths, + Err(error) => return local_error(error), + }; + match crate::installer::upgrade::inspect(request.attempt_id, &data_dir, &run_dir).await { + Ok(attempt) => respond(attempt), + Err(error) => respond(upgrade_error(error)), + } + } + async fn remove_local_machine( &self, request: Request, @@ -740,10 +801,18 @@ impl MachineRpc for MachineService { ImageIngestReason::NotParticipating.rpc_error("Machine is not participating"), ); }; - match self.ingest.open(machine.management_address()).await { - Ok(opened) => respond(opened), - Err(error) => respond(error), - } + let address = machine.management_address(); + let ingest = Arc::clone(&self.ingest); + finish( + self.local + .run_mutation(async move { + ingest + .open(address) + .await + .map_err(LocalMachineError::StoragePreparation) + }) + .await, + ) } async fn pull_image_from_machine( @@ -761,10 +830,14 @@ impl MachineRpc for MachineService { if self.containers().is_err() { return respond(unavailable("Docker is not available")); } - match crate::docker::pull_from_ingest(&request.image, request.source).await { - Ok(()) => respond(ImagePulled {}), - Err(error) => respond(RpcError::from(&error)), - } + finish( + self.local + .run_mutation(async move { + crate::docker::pull_from_ingest(&request.image, request.source).await?; + Ok(ImagePulled {}) + }) + .await, + ) } async fn get_ingress_proxy_config( @@ -955,6 +1028,42 @@ fn local_error(error: LocalMachineError) -> Result, Stat LocalMachineError::AllocatorNotQuiet | LocalMachineError::NotAllocator | LocalMachineError::IsolationLocked => respond(unavailable(&error.to_string())), + LocalMachineError::Admission(crate::installer::admission::Error::Busy) => { + respond(RpcError { + code: RpcErrorCode::Conflict, + message: "a Ployz installation or upgrade is active".into(), + details: Value::Null, + }) + } + LocalMachineError::Admission(crate::installer::admission::Error::Io(error)) => { + Err(Status::internal(error.to_string())) + } + } +} + +fn upgrade_error(error: crate::installer::upgrade::Error) -> RpcError { + use crate::installer::{admission, upgrade::Error}; + let code = match &error { + Error::NotFound => RpcErrorCode::NotFound, + Error::AttemptConflict(_) | Error::Busy | Error::Admission(admission::Error::Busy) => { + RpcErrorCode::Conflict + } + Error::Resolve(_) => RpcErrorCode::InvalidArgument, + Error::Read(_) + | Error::Decode(_) + | Error::Write(_) + | Error::Encode(_) + | Error::QualificationSource(_) + | Error::Launch(_) + | Error::InspectWorker(_) + | Error::NotActive(_) + | Error::Installation(_) + | Error::Admission(admission::Error::Io(_)) => RpcErrorCode::Internal, + }; + RpcError { + code, + message: error.to_string(), + details: Value::Null, } } diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index 3336c550ad..40c68bd530 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -12,7 +12,7 @@ use std::{ }; use clap::{Parser, Subcommand}; -use ployz_core::{DOCKER_NETWORK_CONFLICT_EXIT_STATUS, StorageChoice}; +use ployz_core::{DOCKER_NETWORK_CONFLICT_EXIT_STATUS, MachineUpgradeAttemptId, StorageChoice}; use ployzd::{ daemon::{ContainerMode, Daemon, DaemonConfig, Error, wait_until_socket_accepts}, diag, @@ -54,6 +54,12 @@ enum Command { DialStdio, /// Serve the Docker Volume plugin on its systemd socket. VolumePlugin, + /// Execute one accepted Machine upgrade from its transient systemd service. + #[command(hide = true)] + UpgradeWorker { + #[arg(long)] + attempt: MachineUpgradeAttemptId, + }, /// Install or replace this Machine's daemon release. Install { /// Release channel (stable or beta) or exact published version. @@ -119,6 +125,17 @@ async fn run(args: Args) -> Result<(), Error> { if matches!(args.command, Some(Command::DialStdio)) { return dial_stdio(&args.socket).await.map_err(Error::from); } + let run_dir = args + .socket + .parent() + .unwrap_or_else(|| Path::new("/run/ployz")) + .to_owned(); + if let Some(Command::UpgradeWorker { attempt }) = args.command { + return ployzd::installer::upgrade::run_worker(attempt, &args.data_dir, &run_dir) + .await + .map_err(io::Error::other) + .map_err(Error::from); + } if let Some(Command::Install { version, storage, @@ -137,7 +154,7 @@ async fn run(args: Args) -> Result<(), Error> { release_dir, ) .map_err(Error::from)?; - let outcome = ployzd::installer::install(request) + let outcome = ployzd::installer::install_in(request, &args.data_dir, &run_dir) .await .map_err(io::Error::other)?; match outcome.readiness { @@ -157,7 +174,9 @@ async fn run(args: Args) -> Result<(), Error> { .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; if matches!(args.command, Some(Command::VolumePlugin)) { let listener = volume_plugin::inherited_listener()?; - return volume_plugin::run(listener).await.map_err(Error::from); + return volume_plugin::run(listener, &args.data_dir, &run_dir) + .await + .map_err(Error::from); } let daemon = Daemon::start(DaemonConfig { data_dir: args.data_dir, diff --git a/crates/ployzd/src/volume_plugin.rs b/crates/ployzd/src/volume_plugin.rs index 9cd7d04741..f99069ee4f 100644 --- a/crates/ployzd/src/volume_plugin.rs +++ b/crates/ployzd/src/volume_plugin.rs @@ -15,7 +15,11 @@ use axum::{ }; use ployzd::machine_pool::MachinePool; use serde::{Deserialize, Serialize}; -use tokio::{net::UnixListener, process::Command, sync::Mutex}; +use tokio::{ + net::UnixListener, + process::Command, + sync::{Mutex, OwnedMutexGuard}, +}; mod capacity; mod pool; @@ -82,6 +86,7 @@ struct VolumeStorage { pool: PoolStorage, zfs: PathBuf, mutation: Arc>, + installation: ployzd::installer::admission::Admission, } enum CapacityAdmission { @@ -90,11 +95,12 @@ enum CapacityAdmission { } impl VolumeStorage { - fn new() -> Self { + fn new(data_dir: impl Into, run_dir: impl Into) -> Self { Self { pool: PoolStorage::new("zpool"), zfs: "zfs".into(), mutation: Arc::new(Mutex::new(())), + installation: ployzd::installer::admission::Admission::new(run_dir, data_dir), } } @@ -102,13 +108,35 @@ impl VolumeStorage { fn with_programs(zpool: impl Into, zfs: impl Into) -> Self { let zpool = zpool.into(); let backing = zpool.with_file_name("machine-pool"); + let fixture = zpool + .parent() + .expect("test command has a fixture directory") + .to_owned(); Self { pool: PoolStorage::new(zpool).with_backing(backing), zfs: zfs.into(), mutation: Arc::new(Mutex::new(())), + installation: ployzd::installer::admission::Admission::new( + fixture.join("admission-run"), + fixture.join("admission-data"), + ), } } + async fn admit_mutation( + &self, + ) -> Result<( + OwnedMutexGuard<()>, + ployzd::installer::admission::MutationGuard, + )> { + let local = Arc::clone(&self.mutation).lock_owned().await; + let installation = self + .installation + .try_mutation() + .map_err(|error| VolumeError::from(error.to_string()))?; + Ok((local, installation)) + } + async fn create_volume( &self, pool: &MachinePool, @@ -175,7 +203,7 @@ impl VolumeStorage { } async fn mountpoint(&self, name: &DockerVolumeName) -> Result { - let _guard = self.mutation.lock().await; + let _guard = self.admit_mutation().await?; let pool = self.one_pool().await?; let datasets = self.datasets(&pool).await?; let dataset = Self::dataset(&datasets, &pool, name)? @@ -475,9 +503,17 @@ pub(super) fn inherited_listener() -> io::Result { /// # Errors /// /// Returns an error when the listener cannot become asynchronous or serving fails. -pub(super) async fn run(listener: StdUnixListener) -> io::Result<()> { +pub(super) async fn run( + listener: StdUnixListener, + data_dir: &std::path::Path, + run_dir: &std::path::Path, +) -> io::Result<()> { listener.set_nonblocking(true)?; - serve(UnixListener::from_std(listener)?, VolumeStorage::new()).await + serve( + UnixListener::from_std(listener)?, + VolumeStorage::new(data_dir, run_dir), + ) + .await } async fn serve(listener: UnixListener, storage: VolumeStorage) -> io::Result<()> { diff --git a/crates/ployzd/src/volume_plugin/capacity.rs b/crates/ployzd/src/volume_plugin/capacity.rs index b091bac650..9119fdf973 100644 --- a/crates/ployzd/src/volume_plugin/capacity.rs +++ b/crates/ployzd/src/volume_plugin/capacity.rs @@ -69,7 +69,7 @@ impl VolumeStorage { &self, requested: &Volumes, ) -> Result, ployz_core::RpcError> { - let _guard = self.mutation.lock().await; + let _guard = self.admit_mutation().await.map_err(storage_error)?; let _pool_guard = self.pool.lock_mutation().await.map_err(storage_error)?; let capacity = self.capacity().await.map_err(unknown)?; let budget = capacity @@ -133,7 +133,7 @@ pub(super) async fn inspect( // Finish import recovery under the locks even if the observer disconnects. Json( tokio::spawn(async move { - let _guard = storage.mutation.lock().await; + let _guard = storage.admit_mutation().await.map_err(unknown)?; let _pool_guard = storage.pool.lock_mutation().await.map_err(unknown)?; storage.capacity().await.map_err(unknown) }) diff --git a/crates/ployzd/src/volume_plugin/pool.rs b/crates/ployzd/src/volume_plugin/pool.rs index cd1030d176..3dd750cded 100644 --- a/crates/ployzd/src/volume_plugin/pool.rs +++ b/crates/ployzd/src/volume_plugin/pool.rs @@ -70,7 +70,7 @@ impl VolumeStorage { options: &BTreeMap, ) -> Result<()> { let requested = parse_size(options)?; - let _guard = self.mutation.lock().await; + let _guard = self.admit_mutation().await?; let _pool_guard = self.pool.lock_mutation().await?; let existing = match self.pool.one_usable().await? { Some(pool) => Some(pool), diff --git a/crates/ployzd/src/volume_plugin/removal.rs b/crates/ployzd/src/volume_plugin/removal.rs index 8a055cfe39..b01f8af6c3 100644 --- a/crates/ployzd/src/volume_plugin/removal.rs +++ b/crates/ployzd/src/volume_plugin/removal.rs @@ -11,7 +11,7 @@ use super::{ impl VolumeStorage { async fn remove(&self, name: &DockerVolumeName) -> Result<()> { - let _guard = self.mutation.lock().await; + let _guard = self.admit_mutation().await?; let Some(pool) = self.pool.one_usable().await? else { return Ok(()); }; diff --git a/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs b/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs index b5493b8979..336bddb8df 100644 --- a/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs +++ b/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs @@ -426,6 +426,7 @@ async fn concurrent_and_retried_first_creates_converge_on_one_pool() { pool: first.pool.clone(), zfs: first.zfs.clone(), mutation: Arc::new(Mutex::new(())), + installation: first.installation.clone(), }; let first_socket = test.0.join("first-plugin.sock"); let second_socket = test.0.join("second-plugin.sock"); @@ -491,6 +492,7 @@ async fn a_second_process_cannot_use_a_pool_before_its_owner_finishes() { pool: first.pool.clone(), zfs: first.zfs.clone(), mutation: Arc::new(Mutex::new(())), + installation: first.installation.clone(), }; let first_socket = test.0.join("first-plugin.sock"); let second_socket = test.0.join("second-plugin.sock"); @@ -956,5 +958,9 @@ esac ), zfs: program("zfs"), mutation: Arc::new(Mutex::new(())), + installation: ployzd::installer::admission::Admission::new( + directory.join("admission-run"), + directory.join("admission-data"), + ), } } From bb4d30e394293666bd0359117c7c401e69b8a71a Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:35:43 +0000 Subject: [PATCH 08/30] fix(setup): address adapter review findings --- Cargo.lock | 1 + crates/ployz/Cargo.toml | 1 + crates/ployz/src/context.rs | 13 + crates/ployz/src/handlers/cloud.rs | 86 +- crates/ployz/src/handlers/machine/add.rs | 19 +- crates/ployz/src/handlers/machine/init.rs | 24 +- crates/ployz/src/provisioning.rs | 820 +++++++------------ crates/ployz/src/provisioning/bootstrap.rs | 294 +++++++ crates/ployz/tests/cloud_init/daemon_sync.rs | 45 +- crates/ployz/tests/setup_cli.rs | 137 +++- docs/RELEASE.md | 2 +- scripts/qualify-release.sh | 77 +- scripts/test-qualify-release.sh | 37 +- 13 files changed, 897 insertions(+), 659 deletions(-) create mode 100644 crates/ployz/src/provisioning/bootstrap.rs diff --git a/Cargo.lock b/Cargo.lock index 01489d5da1..766b9746ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2149,6 +2149,7 @@ dependencies = [ "sha2", "shell-words", "sigpipe", + "tempfile", "thiserror 2.0.20", "tokio", "tokio-stream", diff --git a/crates/ployz/Cargo.toml b/crates/ployz/Cargo.toml index 4725faee48..cdd1dbe67d 100644 --- a/crates/ployz/Cargo.toml +++ b/crates/ployz/Cargo.toml @@ -30,6 +30,7 @@ shell-words = "1.1.1" rustix = { version = "1.1.4", features = ["fs", "process", "termios"] } sigpipe = "0.1.3" thiserror = "2.0.20" +tempfile = "3.27.0" tokio = { version = "1.53.1", features = ["io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = "0.1.19" tokio-util = "0.7.19" diff --git a/crates/ployz/src/context.rs b/crates/ployz/src/context.rs index 8bc053bebd..5104d0a6df 100644 --- a/crates/ployz/src/context.rs +++ b/crates/ployz/src/context.rs @@ -543,6 +543,7 @@ impl<'de> Deserialize<'de> for Connection { pub struct SshDestination { value: String, target: String, + copy_target: String, port: Option, } @@ -565,9 +566,15 @@ impl SshDestination { return Err(ConnectionError::SshDestination(value)); } let target = format!("{user}@{host}"); + let copy_target = if host.contains(':') && !host.starts_with('[') { + format!("{user}@[{host}]") + } else { + target.clone() + }; Ok(Self { value, target, + copy_target, port, }) } @@ -582,6 +589,12 @@ impl SshDestination { &self.target } + /// Return the SCP target, with IPv6 hosts bracketed to distinguish its path separator. + #[must_use] + pub(crate) fn copy_target(&self) -> &str { + &self.copy_target + } + #[must_use] pub fn port(&self) -> Option { self.port diff --git a/crates/ployz/src/handlers/cloud.rs b/crates/ployz/src/handlers/cloud.rs index 15d69d8bac..1e1620a863 100644 --- a/crates/ployz/src/handlers/cloud.rs +++ b/crates/ployz/src/handlers/cloud.rs @@ -1,6 +1,6 @@ //! `ployz cloud enroll`: enroll `initialize` or `join` on this Machine. -use std::time::Duration; +use std::{future::Future, time::Duration}; use clap::ArgMatches; use ipnet::Ipv4Net; @@ -16,24 +16,33 @@ use crate::connect::{Client, ConnectError}; use crate::context::{ContextError, Transport}; pub(super) fn enroll(root: &ArgMatches) -> Result<(), Error> { - enroll_with_installer(root, &|| { - crate::provisioning::synchronize_local_daemon().map_err(Into::into) + enroll_with_installer(root, &|storage| async move { + if storage == StorageChoice::Zfs { + crate::provisioning::provision_local(env!("CARGO_PKG_VERSION"), storage).await?; + } else { + crate::provisioning::synchronize_local_daemon().await?; + } + Ok(()) }) } /// Run Cloud enrollment, installing this CLI's daemon version with `install`. /// -/// `install` prepares no storage; tests substitute it to avoid provisioning a -/// real daemon. +/// `install` receives `none` for software-only synchronization and `zfs` for +/// storage preparation; tests substitute it to avoid provisioning a real daemon. /// /// # Errors /// /// Returns the same CLI failure as [`enroll`]. #[doc(hidden)] -pub fn enroll_with_installer( +pub fn enroll_with_installer( root: &ArgMatches, - install: &dyn Fn() -> Result<(), Error>, -) -> Result<(), Error> { + install: &Install, +) -> Result<(), Error> +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let matches = leaf_matches(root); let token = CloudEnrollToken::parse(required(matches, "token")?)?; let cloud_url = matches @@ -60,7 +69,7 @@ pub fn enroll_with_installer( let (details, machine_token, name, outcome) = enroll_current_identity(&mut client, requested_name, requested_storage, &url).await?; match outcome { - Outcome::Join(join) => enroll_join(matches, client, details, *join).await, + Outcome::Join(join) => enroll_join(matches, client, details, *join, install).await, Outcome::Initialize { mode, pairing, @@ -78,6 +87,7 @@ pub fn enroll_with_installer( storage, cloud_url, &token, + install, ) .await } @@ -112,12 +122,17 @@ async fn enroll_current_identity( Ok((details, machine_token, name, outcome)) } -async fn enroll_join( +async fn enroll_join( matches: &ArgMatches, mut client: Client, details: MachineDetails, join: Join, -) -> Result<(), Error> { + install: &Install, +) -> Result<(), Error> +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let assigned = join.registration.assigned_machine.clone(); if already_assigned(&details, &assigned) { println!("Initialised Machine {} ({})", assigned.name, assigned.id); @@ -131,7 +146,7 @@ async fn enroll_join( client, ) .await?; - provision_storage(&client, join.storage)?; + client = provision_storage(matches, client, join.storage, install).await?; crate::handlers::machine::join( &mut client, JoinRequest { @@ -171,7 +186,7 @@ enum FounderLocalState { clippy::too_many_arguments, reason = "the founder tail consumes the existing cloud-enroll command interface" )] -async fn enroll_founder( +async fn enroll_founder( matches: &ArgMatches, mut client: Client, details: MachineDetails, @@ -183,7 +198,12 @@ async fn enroll_founder( storage: StorageChoice, cloud_url: &str, token: &CloudEnrollToken, -) -> Result<(), Error> { + install: &Install, +) -> Result<(), Error> +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let state = match (mode, details.phase) { (InitializeMode::Resume, LocalMachinePhase::Participating) => FounderLocalState::Resume { machine: Box::new(details.machine.ok_or_else(|| { @@ -223,7 +243,7 @@ async fn enroll_founder( client, ) .await?; - provision_storage(&client, storage)?; + client = provision_storage(matches, client, storage, install).await?; let initialized = crate::handlers::machine::initialize( &mut client, InitializeRequest { @@ -277,10 +297,19 @@ async fn enroll_founder( Ok(()) } -fn provision_storage(client: &Client, storage: StorageChoice) -> Result<(), Error> { +async fn provision_storage( + matches: &ArgMatches, + client: Client, + storage: StorageChoice, + install: &Install, +) -> Result +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ crate::provisioning::announce_storage(storage); if storage != StorageChoice::Zfs { - return Ok(()); + return Ok(client); } if !matches!(client.connection().transport(), Transport::Unix(_)) { return Err(Error::usage(format!( @@ -288,15 +317,19 @@ fn provision_storage(client: &Client, storage: StorageChoice) -> Result<(), Erro client.connection() ))); } - crate::provisioning::provision_local(env!("CARGO_PKG_VERSION"), storage)?; - Ok(()) + install(storage).await?; + wait_matching_daemon(matches).await } -async fn synchronize_daemon( +async fn synchronize_daemon( matches: &ArgMatches, mut client: Client, - install: &dyn Fn() -> Result<(), Error>, -) -> Result { + install: &Install, +) -> Result +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let daemon = client .call_repeatable::(DescribeContractRequest {}, None) .await?; @@ -309,7 +342,11 @@ async fn synchronize_daemon( client.connection() ))); } - install()?; + install(StorageChoice::None).await?; + wait_matching_daemon(matches).await +} + +async fn wait_matching_daemon(matches: &ArgMatches) -> Result { let mut client = wait_client(matches).await?; let daemon = client .call_repeatable::(DescribeContractRequest {}, None) @@ -340,7 +377,8 @@ async fn connect_machine(matches: &ArgMatches) -> Result { crate::provisioning::provision_local( env!("CARGO_PKG_VERSION"), ployz_core::StorageChoice::None, - )?; + ) + .await?; wait_client(matches).await } Err(error) => Err(error.into()), diff --git a/crates/ployz/src/handlers/machine/add.rs b/crates/ployz/src/handlers/machine/add.rs index cee88c789a..f63d7d4c9d 100644 --- a/crates/ployz/src/handlers/machine/add.rs +++ b/crates/ployz/src/handlers/machine/add.rs @@ -27,14 +27,15 @@ pub(in crate::handlers) fn add(root: &ArgMatches) -> Result<(), Error> { let wireguard_mtu = matches.get_one::("wg-mtu").copied(); let yes = matches.get_flag("yes"); let storage = crate::provisioning::resolve_storage(matches)?; - if !matches.get_flag("no-install") { - crate::provisioning::provision(matches, storage)?; - } - - let assigned = runtime()?.block_on(async { + let no_install = matches.get_flag("no-install"); + let runtime = runtime()?; + let assigned = runtime.block_on(async { + if !no_install { + crate::provisioning::provision(matches, storage).await?; + } let mut entry = connect_client(matches, options.context()).await?; let visible = entry.machines().await?; - let mut target_client = if matches.get_flag("no-install") { + let mut target_client = if no_install { helpers::connect_direct(matches, &connection).await? } else { helpers::reconnect_direct(matches, &connection).await? @@ -100,13 +101,13 @@ pub(in crate::handlers) fn add(root: &ArgMatches) -> Result<(), Error> { config.save()?; println!("{}", added_machine_line(&assigned)); - runtime()?.block_on(helpers::wait_direct_participating( + runtime.block_on(helpers::wait_direct_participating( matches, &connection, "added Machine did not become ready", ))?; - let catch_up = runtime()?.block_on(async { + let catch_up = runtime.block_on(async { let mut entry = super::super::reconnect_client(matches, options.context()).await?; Ok::<_, Error>( crate::global_catch_up::catch_up_globals(&mut entry, &assigned, !deploy_ingress).await, @@ -120,7 +121,7 @@ pub(in crate::handlers) fn add(root: &ArgMatches) -> Result<(), Error> { crate::global_catch_up::joined_catch_up_error(error) ))); } - let dns_result = runtime()?.block_on(async { + let dns_result = runtime.block_on(async { let mut entry = super::super::reconnect_client(matches, options.context()).await?; crate::dns::update_records_if_reserved(&mut entry).await?; Ok::<_, Error>(()) diff --git a/crates/ployz/src/handlers/machine/init.rs b/crates/ployz/src/handlers/machine/init.rs index 344d24ae79..f3876a82d7 100644 --- a/crates/ployz/src/handlers/machine/init.rs +++ b/crates/ployz/src/handlers/machine/init.rs @@ -50,18 +50,18 @@ pub(in crate::handlers) fn init(root: &ArgMatches) -> Result<(), Error> { let yes = matches.get_flag("yes"); let no_install = matches.get_flag("no-install"); let storage = crate::provisioning::resolve_storage(matches)?; - if !no_install { - let version = matches - .get_one::("version") - .expect("version has a default"); - if local { - crate::provisioning::provision_local(version, storage)?; - } else { - crate::provisioning::provision(matches, storage)?; + let version = matches + .get_one::("version") + .expect("version has a default"); + let runtime = runtime()?; + let (machine, connection) = runtime.block_on(async { + if !no_install { + if local { + crate::provisioning::provision_local(version, storage).await?; + } else { + crate::provisioning::provision(matches, storage).await?; + } } - } - - let (machine, connection) = runtime()?.block_on(async { let mut target = if !no_install { helpers::reconnect_direct(matches, &connection).await? } else { @@ -120,7 +120,7 @@ pub(in crate::handlers) fn init(root: &ArgMatches) -> Result<(), Error> { &context_name, &["machine", "inspect", machine.name.as_str()], ); - runtime()?.block_on(async { + runtime.block_on(async { let mut ready = helpers::wait_direct_participating(matches, &connection, "initial Machine did not become ready") .await.map_err(|error| Error::usage(format!("Machine initialized; startup incomplete: {error}\nInspect with: {inspect_recovery}")))?; diff --git a/crates/ployz/src/provisioning.rs b/crates/ployz/src/provisioning.rs index d93a561172..7252556fce 100644 --- a/crates/ployz/src/provisioning.rs +++ b/crates/ployz/src/provisioning.rs @@ -1,85 +1,127 @@ use std::{ env, ffi::OsString, - fs, io::{self, IsTerminal, Write}, - os::unix::fs::{DirBuilderExt, PermissionsExt}, path::{Path, PathBuf}, - process::{Command, Output, Stdio}, - thread, - time::{Duration, Instant}, + process::{Command, Stdio}, }; use clap::ArgMatches; use ployz_core::StorageChoice; -use sha2::{Digest, Sha256}; use thiserror::Error; use uuid::Uuid; -const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; -const BOOTSTRAP_VERSION_TIMEOUT: Duration = Duration::from_secs(10); +use crate::context::{Connection, ConnectionError, SshDestination, Transport}; +mod bootstrap; + +use bootstrap::Bootstrap; + +/// A failure while preparing a Machine through the shared daemon installer. #[derive(Debug, Error)] pub enum ProvisionError { - #[error("ssh+go provisioning is not implemented; use system ssh")] - SshGo, - #[error("remote machine destination is empty")] - EmptyDestination, + /// The remote setup command did not include a destination. #[error("remote machine destination is required")] MissingDestination, + /// The requested connection is not an SSH connection. + #[error("Machine provisioning requires an SSH destination, not {0}")] + RemoteTransport(String), + /// The SSH destination is malformed or uses a removed transport spelling. + #[error(transparent)] + Connection(#[from] ConnectionError), + /// The local OpenSSH client is unavailable. #[error("local ssh client not found; install an ssh client")] SshClientMissing(#[source] io::Error), + /// The initial remote identity command could not be run. #[error("run ssh whoami: {0}")] Whoami(#[source] io::Error), + /// The initial remote identity command exited unsuccessfully. #[error("ssh whoami failed: {0}")] WhoamiFailed(String), + /// The remote identity was not valid UTF-8. #[error("ssh whoami returned non-UTF-8 output")] WhoamiUtf8, + /// The remote identity command returned no user. #[error("ssh whoami returned an empty user")] EmptyUser, + /// The remote sudo preflight could not be run. #[error("check remote sudo: {0}")] Sudo(#[source] io::Error), + /// A non-root remote user could not authenticate with sudo. #[error("remote user {user} could not authenticate or obtain sudo privileges to install Ployz")] SudoRequired { user: String }, + /// The remote platform inspection command could not be run. #[error("inspect remote Machine platform: {0}")] Platform(#[source] io::Error), + /// The remote platform inspection command exited unsuccessfully. #[error("remote Machine platform inspection failed: {0}")] PlatformFailed(String), + /// The remote platform response was not valid UTF-8. #[error("remote Machine platform inspection returned non-UTF-8 output")] PlatformUtf8, + /// The target host cannot run the Linux Machine daemon. #[error("Ployz Machine must be Linux")] UnsupportedOs, + /// No daemon release archive exists for the target architecture. #[error("unsupported Machine architecture: {0}")] UnsupportedArchitecture(String), + /// A bootstrap filesystem or process operation failed. #[error("{stage}: {source}")] BootstrapIo { stage: &'static str, #[source] source: io::Error, }, + /// A bootstrap process exited unsuccessfully. #[error("{stage} exited with {status}")] BootstrapCommand { stage: &'static str, status: std::process::ExitStatus, }, + /// A bootstrap checksum or version did not match the current CLI release. #[error("bootstrap verification: {0}")] BootstrapVerification(String), - #[error("bootstrap acquisition: {0}")] - BootstrapDownload(String), - #[error("transfer bootstrap daemon: {0}")] + /// A published bootstrap release could not be downloaded. + #[error("{stage}: {source}")] + BootstrapDownload { + stage: &'static str, + #[source] + source: reqwest::Error, + }, + /// The bootstrap or its local release files could not be copied to the Machine. + #[error("transfer bootstrap release: {0}")] Transfer(#[source] io::Error), - #[error("bootstrap daemon transfer exited with {status}")] + /// Remote staging or transfer exited unsuccessfully. + #[error("bootstrap release transfer exited with {status}")] TransferFailed { status: std::process::ExitStatus }, + /// The shared Machine installer could not be spawned. #[error("run Ployz installer: {0}")] Install(#[source] io::Error), + /// The shared Machine installer exited unsuccessfully. #[error("Ployz installer exited with {status}")] InstallFailed { status: std::process::ExitStatus }, + /// Remote bootstrap cleanup could not be run. + #[error("remove remote bootstrap: {0}")] + Cleanup(#[source] io::Error), + /// Remote bootstrap cleanup exited unsuccessfully. + #[error("remote bootstrap cleanup exited with {status}")] + CleanupFailed { status: std::process::ExitStatus }, + /// Setup failed and the subsequent remote cleanup also failed. + #[error("{primary}; cleanup: {cleanup}")] + CleanupAfter { + primary: Box, + cleanup: Box, + }, + /// Local Machine installation requires root privileges. #[error("run this command with sudo")] NotRoot, + /// Reading an interactive storage selection failed. #[error("read storage choice: {0}")] StorageInput(#[source] io::Error), + /// The selected storage value is invalid. #[error(transparent)] StorageChoice(#[from] ployz_core::ValueError), + /// ZFS was selected while installation was explicitly disabled. #[error("zfs storage preparation requires the installer; remove --no-install")] ZfsWithoutInstaller, } @@ -124,249 +166,6 @@ pub(crate) fn announce_storage(storage: StorageChoice) { } } -fn shell_quote(value: &str) -> String { - format!("'{}'", value.replace('\'', "'\"'\"'")) -} - -fn daemon_archive(architecture: &str) -> Result<&'static str, ProvisionError> { - match architecture { - "x86_64" => Ok("ployzd_linux_amd64.tar.gz"), - "aarch64" => Ok("ployzd_linux_arm64.tar.gz"), - architecture => Err(ProvisionError::UnsupportedArchitecture( - architecture.to_owned(), - )), - } -} - -struct Bootstrap { - directory: PathBuf, - daemon: PathBuf, -} - -impl Bootstrap { - fn new() -> Result { - let directory = env::temp_dir().join(format!("ployz-bootstrap-{}", Uuid::new_v4())); - fs::DirBuilder::new() - .mode(0o700) - .create(&directory) - .map_err(|source| ProvisionError::BootstrapIo { - stage: "create bootstrap directory", - source, - })?; - let daemon = directory.join("ployzd"); - Ok(Self { directory, daemon }) - } -} - -impl Drop for Bootstrap { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.directory); - } -} - -fn bootstrap_source() -> Option { - env::var_os("PLOYZ_RELEASE_DIR").map(PathBuf::from) -} - -fn acquire_bootstrap( - architecture: &str, - source: Option<&Path>, -) -> Result { - let archive = daemon_archive(architecture)?; - let bootstrap = Bootstrap::new()?; - let archive_path = bootstrap.directory.join(archive); - let checksums_path = bootstrap.directory.join("checksums.txt"); - if let Some(source) = source { - copy_bootstrap_file( - &source.join(archive), - &archive_path, - "copy bootstrap daemon archive", - )?; - copy_bootstrap_file( - &source.join("checksums.txt"), - &checksums_path, - "copy bootstrap release checksums", - )?; - } else { - let repository = env::var("PLOYZ_GITHUB_URL").unwrap_or_else(|_| RELEASE_REPOSITORY.into()); - let base = format!( - "{repository}/releases/download/v{}", - env!("CARGO_PKG_VERSION") - ); - download_bootstrap( - &format!("{base}/{archive}"), - &archive_path, - "download bootstrap daemon archive", - )?; - download_bootstrap( - &format!("{base}/checksums.txt"), - &checksums_path, - "download bootstrap release checksums", - )?; - } - verify_bootstrap_checksum(&archive_path, &checksums_path, archive)?; - let mut extract = Command::new("tar"); - extract - .arg("-xzf") - .arg(&archive_path) - .arg("-C") - .arg(&bootstrap.directory) - .arg("ployzd"); - bootstrap_status("extract bootstrap daemon", extract.status())?; - fs::set_permissions(&bootstrap.daemon, fs::Permissions::from_mode(0o700)).map_err( - |source| ProvisionError::BootstrapIo { - stage: "mark bootstrap daemon executable", - source, - }, - )?; - Ok(bootstrap) -} - -fn copy_bootstrap_file( - source: &Path, - destination: &Path, - stage: &'static str, -) -> Result<(), ProvisionError> { - fs::copy(source, destination) - .map(|_| ()) - .map_err(|source| ProvisionError::BootstrapIo { stage, source }) -} - -fn download_bootstrap( - url: &str, - destination: &Path, - stage: &'static str, -) -> Result<(), ProvisionError> { - let url = url.to_owned(); - let bytes = thread::spawn(move || { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| format!("{stage}: build download runtime: {error}"))?; - runtime.block_on(async { - let client = reqwest::Client::builder() - .https_only(true) - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(120)) - .user_agent("ployz-bootstrap") - .build() - .map_err(|error| format!("{stage}: build download client: {error}"))?; - client - .get(url) - .send() - .await - .map_err(|error| format!("{stage}: {error}"))? - .error_for_status() - .map_err(|error| format!("{stage}: {error}"))? - .bytes() - .await - .map(|bytes| bytes.to_vec()) - .map_err(|error| format!("{stage}: {error}")) - }) - }) - .join() - .map_err(|_| ProvisionError::BootstrapDownload(format!("{stage}: worker panicked")))? - .map_err(ProvisionError::BootstrapDownload)?; - fs::write(destination, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) -} - -fn bootstrap_status( - stage: &'static str, - result: io::Result, -) -> Result<(), ProvisionError> { - let status = result.map_err(|source| ProvisionError::BootstrapIo { stage, source })?; - if status.success() { - Ok(()) - } else { - Err(ProvisionError::BootstrapCommand { stage, status }) - } -} - -fn verify_bootstrap_checksum( - archive: &Path, - checksums: &Path, - archive_name: &str, -) -> Result<(), ProvisionError> { - let checksums = - fs::read_to_string(checksums).map_err(|source| ProvisionError::BootstrapIo { - stage: "read bootstrap release checksums", - source, - })?; - let expected = checksums - .lines() - .find_map(|line| { - let mut fields = line.split_whitespace(); - let hash = fields.next()?; - let name = fields.next()?.trim_start_matches('*'); - (name == archive_name - && hash.len() == 64 - && hash.bytes().all(|byte| byte.is_ascii_hexdigit())) - .then(|| hash.to_ascii_lowercase()) - }) - .ok_or_else(|| { - ProvisionError::BootstrapVerification(format!( - "checksums.txt has no SHA-256 hash for {archive_name}" - )) - })?; - let bytes = fs::read(archive).map_err(|source| ProvisionError::BootstrapIo { - stage: "read bootstrap daemon archive", - source, - })?; - let actual = hex::encode(Sha256::digest(bytes)); - if actual == expected { - Ok(()) - } else { - Err(ProvisionError::BootstrapVerification(format!( - "{archive_name} checksum was {actual}, expected {expected}" - ))) - } -} - -fn command_output_with_timeout(command: &mut Command, timeout: Duration) -> io::Result { - let mut child = command - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - let deadline = Instant::now() + timeout; - while Instant::now() < deadline { - if child.try_wait()?.is_some() { - return child.wait_with_output(); - } - thread::sleep(Duration::from_millis(10)); - } - child.kill()?; - let _ = child.wait(); - Err(io::Error::new( - io::ErrorKind::TimedOut, - format!("timed out after {} seconds", timeout.as_secs()), - )) -} - -fn verify_bootstrap_version(command: &mut Command) -> Result<(), ProvisionError> { - let output = - command_output_with_timeout(command, BOOTSTRAP_VERSION_TIMEOUT).map_err(|source| { - ProvisionError::BootstrapIo { - stage: "preflight bootstrap daemon", - source, - } - })?; - if !output.status.success() { - return Err(ProvisionError::BootstrapCommand { - stage: "preflight bootstrap daemon", - status: output.status, - }); - } - let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if observed == env!("CARGO_PKG_VERSION") { - Ok(()) - } else { - Err(ProvisionError::BootstrapVerification(format!( - "bootstrap daemon reported version {observed:?}, expected {}", - env!("CARGO_PKG_VERSION") - ))) - } -} - enum Preparation<'user> { Host { storage: StorageChoice, @@ -375,24 +174,12 @@ enum Preparation<'user> { SoftwareOnly, } -fn normalized_version(version: &str) -> &str { - let version = version.strip_prefix('v').unwrap_or(version); - match version { - "" | "latest" => "stable", - version => version, - } -} - fn install_arguments( version: &str, preparation: Preparation<'_>, release_dir: Option<&Path>, ) -> Vec { - let mut arguments = vec![ - "install".into(), - "--version".into(), - normalized_version(version).into(), - ]; + let mut arguments = vec!["install".into(), "--version".into(), version.into()]; match preparation { Preparation::Host { storage, @@ -411,33 +198,146 @@ fn install_arguments( arguments } -fn installer_status(result: io::Result) -> Result<(), ProvisionError> { - let status = result.map_err(ProvisionError::Install)?; - if status.success() { - Ok(()) - } else { - Err(ProvisionError::InstallFailed { status }) +struct Remote { + destination: SshDestination, + key: PathBuf, + timeout: std::time::Duration, + control_path: Option, +} + +impl Remote { + fn from_matches(matches: &ArgMatches) -> Result { + let value = matches + .get_one::("destination") + .ok_or(ProvisionError::MissingDestination)?; + let connection = value.parse::()?; + let Transport::Ssh { destination, .. } = connection.transport() else { + return Err(ProvisionError::RemoteTransport(connection.to_string())); + }; + Ok(Self { + destination: destination.clone(), + key: ssh_key(matches), + timeout: crate::cli::ssh_timeout(matches), + control_path: crate::connect::control_path(), + }) } -} -fn ssh_parts(destination: &str) -> Result<(String, Option), ProvisionError> { - if destination.starts_with("ssh+go://") { - return Err(ProvisionError::SshGo); + fn ssh(&self) -> Command { + let mut command = Command::new("ssh"); + command.args(self.common_arguments()); + if let Some(port) = self.destination.port() { + command.args([OsString::from("-p"), port.to_string().into()]); + } + // Provisioning may wait for human authentication; only network setup is timed. + command.args(["-tt", self.destination.target()]); + command.stdin(Stdio::inherit()); + command } - let destination = destination - .strip_prefix("ssh://") - .or_else(|| destination.strip_prefix("ssh+cli://")) - .unwrap_or(destination); - if destination.is_empty() { - return Err(ProvisionError::EmptyDestination); + + fn scp(&self) -> Command { + let mut command = Command::new("scp"); + command.args(self.common_arguments()); + if let Some(port) = self.destination.port() { + command.args([OsString::from("-P"), port.to_string().into()]); + } + command + } + + fn common_arguments(&self) -> Vec { + let mut arguments: Vec = + crate::connect::ssh_control_args(self.control_path.as_deref()) + .into_iter() + .map(Into::into) + .collect(); + arguments.extend([ + "-o".into(), + format!("ConnectTimeout={}", self.timeout.as_secs().max(1)).into(), + "-o".into(), + "BatchMode=no".into(), + "-i".into(), + self.key.as_os_str().to_owned(), + ]); + arguments + } + + async fn platform(&self) -> Result { + let mut command = self.ssh(); + command.arg("uname -s; uname -m"); + let output = tokio::process::Command::from(command) + .output() + .await + .map_err(ProvisionError::Platform)?; + if !output.status.success() { + return Err(ProvisionError::PlatformFailed( + String::from_utf8_lossy(&output.stderr).trim().to_owned(), + )); + } + let output = String::from_utf8(output.stdout).map_err(|_| ProvisionError::PlatformUtf8)?; + let mut lines = output + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()); + if lines.next() != Some("Linux") { + return Err(ProvisionError::UnsupportedOs); + } + lines + .next() + .map(str::to_owned) + .ok_or_else(|| ProvisionError::PlatformFailed("uname returned no architecture".into())) + } + + async fn stage( + &self, + bootstrap: &Bootstrap, + version: &str, + directory: &str, + ) -> Result<(), ProvisionError> { + let mut create = self.ssh(); + create.arg(format!("mkdir -m 700 -- {}", shell_quote(directory))); + transfer_status(tokio::process::Command::from(create).status().await)?; + + let mut transfer = self.scp(); + transfer.arg(bootstrap.daemon()); + if let Some(files) = bootstrap.release_files(version) { + transfer.args(files); + } + transfer.arg(format!("{}:{}/", self.destination.copy_target(), directory)); + transfer_status(tokio::process::Command::from(transfer).status().await) } - if let Some((host, port)) = destination.rsplit_once(':') - && !port.is_empty() - && port.chars().all(|character| character.is_ascii_digit()) - { - return Ok((host.to_owned(), Some(port.to_owned()))); + + async fn verify(&self, daemon: &str) -> Result<(), ProvisionError> { + let mut command = self.ssh(); + command.arg(format!( + "chmod 700 {path} && {path} version", + path = shell_quote(daemon) + )); + bootstrap::verify_command_version("preflight remote bootstrap daemon", command).await + } + + async fn install( + &self, + daemon: &str, + arguments: &[OsString], + via_sudo: bool, + ) -> Result<(), ProvisionError> { + let mut command = self.ssh(); + command.arg(remote_command(daemon, arguments, via_sudo)); + installer_status(tokio::process::Command::from(command).status().await) + } + + async fn cleanup(&self, directory: &str) -> Result<(), ProvisionError> { + let mut cleanup = self.ssh(); + cleanup.arg(format!("rm -rf -- {}", shell_quote(directory))); + let status = tokio::process::Command::from(cleanup) + .status() + .await + .map_err(ProvisionError::Cleanup)?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::CleanupFailed { status }) + } } - Ok((destination.to_owned(), None)) } fn ssh_key(matches: &ArgMatches) -> PathBuf { @@ -447,7 +347,7 @@ fn ssh_key(matches: &ArgMatches) -> PathBuf { key.strip_prefix("~/").map_or_else( || PathBuf::from(key), |relative| { - std::env::var_os("HOME") + env::var_os("HOME") .map(PathBuf::from) .unwrap_or_default() .join(relative) @@ -455,85 +355,8 @@ fn ssh_key(matches: &ArgMatches) -> PathBuf { ) } -fn ssh_command(matches: &ArgMatches) -> Result<(Command, String), ProvisionError> { - let destination = matches - .get_one::("destination") - .ok_or(ProvisionError::MissingDestination)?; - let (destination, port) = ssh_parts(destination)?; - let mut command = Command::new("ssh"); - command.arg("-o").arg(format!( - "ConnectTimeout={}", - crate::cli::ssh_timeout(matches).as_secs() - )); - // Provisioning may wait for human authentication; only network setup is timed. - command.args(["-o", "BatchMode=no", "-tt"]); - command.args(crate::connect::ssh_control_args( - crate::connect::control_path().as_deref(), - )); - command.stdin(Stdio::inherit()); - command.arg("-i").arg(ssh_key(matches)); - if let Some(port) = port { - command.arg("-p").arg(port); - } - Ok((command, destination)) -} - -fn scp_command(matches: &ArgMatches) -> Result<(Command, String), ProvisionError> { - let destination = matches - .get_one::("destination") - .ok_or(ProvisionError::MissingDestination)?; - let (destination, port) = ssh_parts(destination)?; - let mut command = Command::new("scp"); - command.arg("-o").arg(format!( - "ConnectTimeout={}", - crate::cli::ssh_timeout(matches).as_secs() - )); - command.args(["-o", "BatchMode=no"]); - command.args(crate::connect::ssh_control_args( - crate::connect::control_path().as_deref(), - )); - command.arg("-i").arg(ssh_key(matches)); - if let Some(port) = port { - command.arg("-P").arg(port); - } - Ok((command, destination)) -} - -fn scp_destination(destination: &str, path: &str) -> String { - let (user, host) = destination - .split_once('@') - .expect("validated SSH destinations include a user"); - if host.contains(':') && !host.starts_with('[') { - format!("{user}@[{host}]:{path}") - } else { - format!("{destination}:{path}") - } -} - -fn remote_platform(matches: &ArgMatches) -> Result { - let (mut platform, destination) = ssh_command(matches)?; - let output = platform - .arg(destination) - .arg("uname -s; uname -m") - .output() - .map_err(ProvisionError::Platform)?; - if !output.status.success() { - return Err(ProvisionError::PlatformFailed( - String::from_utf8_lossy(&output.stderr).trim().to_owned(), - )); - } - let output = String::from_utf8(output.stdout).map_err(|_| ProvisionError::PlatformUtf8)?; - let mut lines = output - .lines() - .map(str::trim) - .filter(|line| !line.is_empty()); - if lines.next() != Some("Linux") { - return Err(ProvisionError::UnsupportedOs); - } - lines - .next() - .map(str::to_owned) - .ok_or_else(|| ProvisionError::PlatformFailed("uname returned no architecture".into())) +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\"'\"'")) } fn remote_command(program: &str, arguments: &[OsString], via_sudo: bool) -> String { @@ -549,21 +372,54 @@ fn remote_command(program: &str, arguments: &[OsString], via_sudo: bool) -> Stri command } -fn cleanup_remote_bootstrap(matches: &ArgMatches, path: &str) { - if let Ok((mut cleanup, destination)) = ssh_command(matches) { - let _ = cleanup - .arg(destination) - .arg(format!("rm -f -- {}", shell_quote(path))) - .status(); +fn transfer_status(result: io::Result) -> Result<(), ProvisionError> { + let status = result.map_err(ProvisionError::Transfer)?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::TransferFailed { status }) + } +} + +fn installer_status(result: io::Result) -> Result<(), ProvisionError> { + let status = result.map_err(ProvisionError::Install)?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::InstallFailed { status }) + } +} + +fn finish_remote( + primary: Result<(), ProvisionError>, + cleanup: Result<(), ProvisionError>, +) -> Result<(), ProvisionError> { + match (primary, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(primary), Ok(())) => Err(primary), + (Ok(()), Err(cleanup)) => Err(cleanup), + (Err(primary), Err(cleanup)) => Err(ProvisionError::CleanupAfter { + primary: Box::new(primary), + cleanup: Box::new(cleanup), + }), } } -pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), ProvisionError> { - let (mut whoami, destination) = ssh_command(matches)?; - let output = whoami - .arg(&destination) - .arg("whoami") +/// Prepare a remote Linux Machine through its verified bootstrap daemon. +/// +/// # Errors +/// +/// Returns a [`ProvisionError`] when the SSH target or privilege preflight is +/// invalid, when platform inspection, bootstrap acquisition, checksum or version +/// verification fails, when staging or installation fails, or when the staged +/// bootstrap cannot be removed. +pub async fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), ProvisionError> { + let remote = Remote::from_matches(matches)?; + let mut whoami = remote.ssh(); + whoami.arg("whoami"); + let output = tokio::process::Command::from(whoami) .output() + .await .map_err(|error| { if error.kind() == io::ErrorKind::NotFound { ProvisionError::SshClientMissing(error) @@ -582,12 +438,13 @@ pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), Pro return Err(ProvisionError::EmptyUser); } - if user != "root" { - let (mut sudo, destination) = ssh_command(matches)?; - let status = sudo - .arg(destination) - .arg("sudo true") + let via_sudo = user != "root"; + if via_sudo { + let mut sudo = remote.ssh(); + sudo.arg("sudo true"); + let status = tokio::process::Command::from(sudo) .status() + .await .map_err(ProvisionError::Sudo)?; if !status.success() { return Err(ProvisionError::SudoRequired { @@ -596,81 +453,42 @@ pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), Pro } } - let architecture = remote_platform(matches)?; - let bootstrap = acquire_bootstrap(&architecture, bootstrap_source().as_deref())?; - let remote_bootstrap = format!("/tmp/ployz-bootstrap-{}", Uuid::new_v4()); - let (mut transfer, destination) = scp_command(matches)?; - let status = transfer - .arg(&bootstrap.daemon) - .arg(scp_destination(&destination, &remote_bootstrap)) - .status() - .map_err(ProvisionError::Transfer)?; - if !status.success() { - cleanup_remote_bootstrap(matches, &remote_bootstrap); - return Err(ProvisionError::TransferFailed { status }); - } - - let preflight = (|| { - let (mut verify, destination) = ssh_command(matches)?; - let command = format!( - "chmod 700 {path} && {path} version", - path = shell_quote(&remote_bootstrap) - ); - let output = verify - .arg(destination) - .arg(command) - .output() - .map_err(|source| ProvisionError::BootstrapIo { - stage: "preflight remote bootstrap daemon", - source, - })?; - if !output.status.success() { - return Err(ProvisionError::BootstrapCommand { - stage: "preflight remote bootstrap daemon", - status: output.status, - }); - } - let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if observed != env!("CARGO_PKG_VERSION") { - return Err(ProvisionError::BootstrapVerification(format!( - "remote bootstrap daemon reported version {observed:?}, expected {}", - env!("CARGO_PKG_VERSION") - ))); - } - Ok(()) - })(); - if let Err(error) = preflight { - cleanup_remote_bootstrap(matches, &remote_bootstrap); - return Err(error); - } - + let architecture = remote.platform().await?; + let bootstrap = Bootstrap::acquire(&architecture).await?; let version = matches .get_one::("version") .expect("version has a default"); - let via_sudo = user != "root"; + let remote_directory = format!("/tmp/ployz-bootstrap-{}", Uuid::new_v4()); + let remote_daemon = format!("{remote_directory}/ployzd"); + let release_dir = bootstrap + .release_dir(version) + .map(|_| Path::new(&remote_directory)); let arguments = install_arguments( version, Preparation::Host { storage, group_user: via_sudo.then_some(user), }, - None, + release_dir, ); - let remote = remote_command(&remote_bootstrap, &arguments, via_sudo); - let (mut install, destination) = ssh_command(matches)?; - let result = installer_status(install.arg(destination).arg(remote).status()); - cleanup_remote_bootstrap(matches, &remote_bootstrap); - result + let primary = async { + remote.stage(&bootstrap, version, &remote_directory).await?; + remote.verify(&remote_daemon).await?; + remote.install(&remote_daemon, &arguments, via_sudo).await + } + .await; + finish_remote(primary, remote.cleanup(&remote_directory).await) } /// Install and start local `ployzd` through a verified temporary daemon. /// /// # Errors /// -/// Returns [`ProvisionError::NotRoot`] when this process is not root. -/// Returns [`ProvisionError::Install`] when the installer cannot be spawned, -/// or [`ProvisionError::InstallFailed`] when it exits non-zero. -pub fn provision_local(version: &str, storage: StorageChoice) -> Result<(), ProvisionError> { +/// Returns [`ProvisionError::NotRoot`] without local root privileges. It also +/// reports unsupported platforms, release acquisition and filesystem failures, +/// checksum or executable-version verification failures, and installer spawn or +/// non-zero exit failures. +pub async fn provision_local(version: &str, storage: StorageChoice) -> Result<(), ProvisionError> { let group_user = env::var("SUDO_USER").ok().filter(|user| !user.is_empty()); provision_local_with( version, @@ -679,34 +497,37 @@ pub fn provision_local(version: &str, storage: StorageChoice) -> Result<(), Prov group_user: group_user.as_deref(), }, ) + .await } /// Synchronize the local daemon binary without preparing host software or storage. /// /// # Errors /// -/// Returns the first bootstrap acquisition, verification, or installation failure. -pub fn synchronize_local_daemon() -> Result<(), ProvisionError> { - provision_local_with(env!("CARGO_PKG_VERSION"), Preparation::SoftwareOnly) +/// Returns the first privilege, platform, bootstrap acquisition, checksum, +/// executable-version, process, or installation failure. +pub async fn synchronize_local_daemon() -> Result<(), ProvisionError> { + provision_local_with(env!("CARGO_PKG_VERSION"), Preparation::SoftwareOnly).await } -fn provision_local_with(version: &str, preparation: Preparation<'_>) -> Result<(), ProvisionError> { +async fn provision_local_with( + version: &str, + preparation: Preparation<'_>, +) -> Result<(), ProvisionError> { if !process_is_root() { return Err(ProvisionError::NotRoot); } if env::consts::OS != "linux" { return Err(ProvisionError::UnsupportedOs); } - let bootstrap = acquire_bootstrap(env::consts::ARCH, bootstrap_source().as_deref())?; - let mut version_command = Command::new(&bootstrap.daemon); - version_command.arg("version"); - verify_bootstrap_version(&mut version_command)?; - let release_dir = (normalized_version(version) == env!("CARGO_PKG_VERSION")) - .then_some(bootstrap.directory.as_path()); + let bootstrap = Bootstrap::acquire(env::consts::ARCH).await?; + bootstrap.verify_local_version().await?; + let arguments = install_arguments(version, preparation, bootstrap.release_dir(version)); installer_status( - Command::new(&bootstrap.daemon) - .args(install_arguments(version, preparation, release_dir)) - .status(), + tokio::process::Command::new(bootstrap.daemon()) + .args(arguments) + .status() + .await, ) } @@ -722,10 +543,9 @@ pub(crate) fn process_is_root() -> bool { #[cfg(test)] mod tests { use super::*; - use ployz_core::StorageChoice; #[test] - fn provisioning_ssh_allows_interactive_authentication_with_a_network_timeout() { + fn provisioning_ssh_allows_interactive_authentication_with_shared_options() { for (extra, seconds) in [(vec![], "5"), (vec!["--ssh-timeout", "17"], "17")] { let mut args = vec!["ployz", "machine", "add", "root@host"]; args.extend(extra); @@ -735,49 +555,29 @@ mod tests { .unwrap() .subcommand_matches("add") .unwrap(); - let (command, _) = ssh_command(matches).unwrap(); - let args: Vec<_> = command.get_args().collect(); - assert!(args.contains(&std::ffi::OsStr::new("BatchMode=no"))); - assert!(args.contains(&std::ffi::OsStr::new("-tt"))); - for arg in crate::connect::ssh_control_args(crate::connect::control_path().as_deref()) { - assert!(args.contains(&std::ffi::OsStr::new(&arg))); + let remote = Remote::from_matches(matches).unwrap(); + let ssh = remote.ssh(); + let scp = remote.scp(); + for command in [&ssh, &scp] { + let args: Vec<_> = command.get_args().collect(); + assert!(args.contains(&std::ffi::OsStr::new("BatchMode=no"))); + for arg in + crate::connect::ssh_control_args(crate::connect::control_path().as_deref()) + { + assert!(args.contains(&std::ffi::OsStr::new(&arg))); + } + assert!( + command + .get_args() + .any(|arg| arg == format!("ConnectTimeout={seconds}").as_str()) + ); } - assert!( - command - .get_args() - .any(|arg| arg == format!("ConnectTimeout={seconds}").as_str()) - ); + assert!(ssh.get_args().any(|arg| arg == "-tt")); } } - #[test] - fn scp_brackets_bare_ipv6_hosts() { - assert_eq!( - scp_destination("deploy@2001:db8::1", "/tmp/bootstrap"), - "deploy@[2001:db8::1]:/tmp/bootstrap" - ); - assert_eq!( - scp_destination("deploy@[2001:db8::1]", "/tmp/bootstrap"), - "deploy@[2001:db8::1]:/tmp/bootstrap" - ); - } - #[test] fn shared_installer_arguments_keep_bootstrap_and_target_distinct() { - assert_eq!( - install_arguments( - "v1.2.3", - Preparation::Host { - storage: StorageChoice::None, - group_user: None, - }, - None, - ), - ["install", "--version", "1.2.3", "--storage", "none"] - .map(OsString::from) - .to_vec() - ); - assert_eq!(normalized_version("latest"), "stable"); assert_eq!( install_arguments( "1.2.3", @@ -807,6 +607,18 @@ mod tests { ); } + #[test] + fn cleanup_failures_preserve_primary_evidence() { + let primary = ProvisionError::BootstrapVerification("bad version".into()); + let cleanup = ProvisionError::Cleanup(io::Error::other("ssh failed")); + let error = finish_remote(Err(primary), Err(cleanup)).unwrap_err(); + + assert_eq!( + error.to_string(), + "bootstrap verification: bad version; cleanup: remove remote bootstrap: ssh failed" + ); + } + #[test] fn storage_resolution_honors_explicit_and_safe_noninteractive_choices() { assert_eq!( @@ -860,13 +672,13 @@ mod tests { resolve_storage(matches).unwrap() } - #[test] - fn local_provision_requires_root() { + #[tokio::test] + async fn local_provision_requires_root() { if process_is_root() { return; } assert!(matches!( - provision_local(env!("CARGO_PKG_VERSION"), StorageChoice::None), + provision_local(env!("CARGO_PKG_VERSION"), StorageChoice::None).await, Err(ProvisionError::NotRoot) )); } diff --git a/crates/ployz/src/provisioning/bootstrap.rs b/crates/ployz/src/provisioning/bootstrap.rs new file mode 100644 index 0000000000..f5b7cac179 --- /dev/null +++ b/crates/ployz/src/provisioning/bootstrap.rs @@ -0,0 +1,294 @@ +use std::{ + env, + fs::{self, File}, + io::{self, Read, Seek, SeekFrom}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Command, Output, Stdio}, + time::Duration, +}; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; +use tokio::time::timeout; + +use super::ProvisionError; + +const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; +const VERSION_TIMEOUT: Duration = Duration::from_secs(10); + +/// A verified copy of this CLI release's daemon, owned by a private directory. +pub(super) struct Bootstrap { + directory: TempDir, + daemon: PathBuf, + archive: PathBuf, + checksums: PathBuf, + local_source: bool, +} + +impl Bootstrap { + /// Acquire, checksum, and extract the bootstrap for a Linux architecture. + pub(super) async fn acquire(architecture: &str) -> Result { + let archive_name = daemon_archive(architecture)?; + let directory = tempfile::Builder::new() + .prefix("ployz-bootstrap-") + .tempdir() + .map_err(|source| ProvisionError::BootstrapIo { + stage: "create bootstrap directory", + source, + })?; + let archive = directory.path().join(archive_name); + let checksums = directory.path().join("checksums.txt"); + let local_source = if let Some(source) = env::var_os("PLOYZ_RELEASE_DIR") { + let source = PathBuf::from(source); + copy_file( + &source.join(archive_name), + &archive, + "copy bootstrap daemon archive", + )?; + copy_file( + &source.join("checksums.txt"), + &checksums, + "copy bootstrap release checksums", + )?; + true + } else { + let base = format!( + "{RELEASE_REPOSITORY}/releases/download/v{}", + env!("CARGO_PKG_VERSION") + ); + let client = reqwest::Client::builder() + .https_only(true) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(120)) + .user_agent("ployz-bootstrap") + .build() + .map_err(|source| ProvisionError::BootstrapDownload { + stage: "build bootstrap download client", + source, + })?; + download( + &client, + &format!("{base}/{archive_name}"), + &archive, + "download bootstrap daemon archive", + ) + .await?; + download( + &client, + &format!("{base}/checksums.txt"), + &checksums, + "download bootstrap release checksums", + ) + .await?; + false + }; + + verify_checksum(&archive, &checksums, archive_name)?; + let daemon = directory.path().join("ployzd"); + let mut extract = Command::new("tar"); + extract + .arg("-xzf") + .arg(&archive) + .arg("-C") + .arg(directory.path()) + .arg("ployzd"); + command_status("extract bootstrap daemon", extract).await?; + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o700)).map_err(|source| { + ProvisionError::BootstrapIo { + stage: "mark bootstrap daemon executable", + source, + } + })?; + + Ok(Self { + directory, + daemon, + archive, + checksums, + local_source, + }) + } + + pub(super) fn daemon(&self) -> &Path { + &self.daemon + } + + /// Return the local release directory only when it describes the selected target. + pub(super) fn release_dir(&self, version: &str) -> Option<&Path> { + (self.local_source && version == env!("CARGO_PKG_VERSION")).then(|| self.directory.path()) + } + + pub(super) fn release_files(&self, version: &str) -> Option<[&Path; 2]> { + self.release_dir(version) + .map(|_| [self.archive.as_path(), self.checksums.as_path()]) + } + + pub(super) async fn verify_local_version(&self) -> Result<(), ProvisionError> { + let mut command = Command::new(&self.daemon); + command.arg("version"); + verify_command_version("preflight bootstrap daemon", command).await + } +} + +pub(super) async fn verify_command_version( + stage: &'static str, + command: Command, +) -> Result<(), ProvisionError> { + let output = command_output(command, VERSION_TIMEOUT) + .await + .map_err(|source| ProvisionError::BootstrapIo { stage, source })?; + if !output.status.success() { + return Err(ProvisionError::BootstrapCommand { + stage, + status: output.status, + }); + } + let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if observed == env!("CARGO_PKG_VERSION") { + Ok(()) + } else { + Err(ProvisionError::BootstrapVerification(format!( + "bootstrap daemon reported version {observed:?}, expected {}", + env!("CARGO_PKG_VERSION") + ))) + } +} + +async fn download( + client: &reqwest::Client, + url: &str, + destination: &Path, + stage: &'static str, +) -> Result<(), ProvisionError> { + let bytes = client + .get(url) + .send() + .await + .map_err(|source| ProvisionError::BootstrapDownload { stage, source })? + .error_for_status() + .map_err(|source| ProvisionError::BootstrapDownload { stage, source })? + .bytes() + .await + .map_err(|source| ProvisionError::BootstrapDownload { stage, source })?; + fs::write(destination, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + +fn copy_file(source: &Path, destination: &Path, stage: &'static str) -> Result<(), ProvisionError> { + fs::copy(source, destination) + .map(|_| ()) + .map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + +fn verify_checksum( + archive: &Path, + checksums: &Path, + archive_name: &str, +) -> Result<(), ProvisionError> { + let checksums = + fs::read_to_string(checksums).map_err(|source| ProvisionError::BootstrapIo { + stage: "read bootstrap release checksums", + source, + })?; + let expected = checksums + .lines() + .find_map(|line| { + let mut fields = line.split_whitespace(); + let hash = fields.next()?; + let name = fields.next()?.trim_start_matches('*'); + (name == archive_name + && hash.len() == 64 + && hash.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then(|| hash.to_ascii_lowercase()) + }) + .ok_or_else(|| { + ProvisionError::BootstrapVerification(format!( + "checksums.txt has no SHA-256 hash for {archive_name}" + )) + })?; + let bytes = fs::read(archive).map_err(|source| ProvisionError::BootstrapIo { + stage: "read bootstrap daemon archive", + source, + })?; + let actual = hex::encode(Sha256::digest(bytes)); + if actual == expected { + Ok(()) + } else { + Err(ProvisionError::BootstrapVerification(format!( + "{archive_name} checksum was {actual}, expected {expected}" + ))) + } +} + +async fn command_status(stage: &'static str, command: Command) -> Result<(), ProvisionError> { + let status = tokio::process::Command::from(command) + .status() + .await + .map_err(|source| ProvisionError::BootstrapIo { stage, source })?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::BootstrapCommand { stage, status }) + } +} + +/// Capture bounded command output and explicitly kill and reap a timed-out child. +async fn command_output(mut command: Command, budget: Duration) -> io::Result { + let mut stdout = tempfile::tempfile()?; + let mut stderr = tempfile::tempfile()?; + command + .stdout(Stdio::from(stdout.try_clone()?)) + .stderr(Stdio::from(stderr.try_clone()?)); + let mut child = tokio::process::Command::from(command).spawn()?; + let status = match timeout(budget, child.wait()).await { + Ok(status) => status?, + Err(_) => { + if child.try_wait()?.is_none() { + child.kill().await?; + } + let _ = child.wait().await; + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("timed out after {} seconds", budget.as_secs()), + )); + } + }; + Ok(Output { + status, + stdout: read_from_start(&mut stdout)?, + stderr: read_from_start(&mut stderr)?, + }) +} + +fn read_from_start(file: &mut File) -> io::Result> { + file.seek(SeekFrom::Start(0))?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + Ok(bytes) +} + +fn daemon_archive(architecture: &str) -> Result<&'static str, ProvisionError> { + match architecture { + "x86_64" => Ok("ployzd_linux_amd64.tar.gz"), + "aarch64" => Ok("ployzd_linux_arm64.tar.gz"), + architecture => Err(ProvisionError::UnsupportedArchitecture( + architecture.to_owned(), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn bounded_output_kills_and_reaps_a_hung_child() { + let mut command = Command::new("sh"); + command.args(["-c", "exec sleep 30"]); + let error = command_output(command, Duration::from_millis(20)) + .await + .unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + } +} diff --git a/crates/ployz/tests/cloud_init/daemon_sync.rs b/crates/ployz/tests/cloud_init/daemon_sync.rs index 2c9d6f3ae0..0798ff7d7b 100644 --- a/crates/ployz/tests/cloud_init/daemon_sync.rs +++ b/crates/ployz/tests/cloud_init/daemon_sync.rs @@ -26,17 +26,17 @@ fn recording_installer( daemon: JoinDaemon, outcome: InstallOutcome, calls: Arc, -) -> impl Fn() -> Result<(), ployz::handlers::Error> { - move || { +) -> impl Fn(ployz_core::StorageChoice) -> std::future::Ready> { + move |_| { calls.fetch_add(1, Ordering::SeqCst); - match outcome { + std::future::ready(match outcome { InstallOutcome::UpdateDaemon => { daemon.set_daemon_version(env!("CARGO_PKG_VERSION")); Ok(()) } InstallOutcome::LeaveStale => Ok(()), InstallOutcome::Fail(message) => Err(ployz::handlers::Error::usage(message)), - } + }) } } @@ -62,7 +62,11 @@ fn enroll_matches(connect: &str, cloud_url: &str) -> ArgMatches { async fn run_enroll( matches: ArgMatches, - install: impl Fn() -> Result<(), ployz::handlers::Error> + Send + 'static, + install: impl Fn( + ployz_core::StorageChoice, + ) -> std::future::Ready> + + Send + + 'static, ) -> Result<(), ployz::handlers::Error> { tokio::task::spawn_blocking(move || { ployz::handlers::cloud_enroll_with_installer(&matches, &install) @@ -78,6 +82,18 @@ async fn enroll_locally( Result<(), ployz::handlers::Error>, Arc, Arc, +) { + enroll_locally_with_storage(daemon_version, outcome, "none").await +} + +async fn enroll_locally_with_storage( + daemon_version: &str, + outcome: InstallOutcome, + storage: &str, +) -> ( + Result<(), ployz::handlers::Error>, + Arc, + Arc, ) { let registration = registration(); let relay = RelayListen::start().await; @@ -85,7 +101,7 @@ async fn enroll_locally( CloudPairing::parse(&relay.url, PairingCredential::parse(PAIRING).unwrap()).unwrap(); let enroll = EnrollListen::start(json!({ "kind": "join", - "storage": "none", + "storage": storage, "pairing": pairing, "registration": registration, })) @@ -100,6 +116,23 @@ async fn enroll_locally( (result, calls, connections) } +#[tokio::test] +async fn zfs_preparation_reconnects_after_restarting_a_matching_daemon() { + let (result, calls, connections) = enroll_locally_with_storage( + env!("CARGO_PKG_VERSION"), + InstallOutcome::UpdateDaemon, + "zfs", + ) + .await; + + assert!(result.is_ok(), "{result:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!( + connections.load(Ordering::SeqCst) >= 2, + "enrollment must reconnect after storage preparation restarts the daemon" + ); +} + #[tokio::test] async fn matching_daemon_does_not_run_the_installer() { let (result, calls, _) = diff --git a/crates/ployz/tests/setup_cli.rs b/crates/ployz/tests/setup_cli.rs index 122a311d40..049581b064 100644 --- a/crates/ployz/tests/setup_cli.rs +++ b/crates/ployz/tests/setup_cli.rs @@ -88,6 +88,8 @@ fn run(args: &[&str], root: &Path, release: &Path, bin: &Path) -> Output { .env("PLOYZ_RELEASE_DIR", release) .env("PLOYZ_CONFIG", root.join("config.yaml")) .env("PLOYZ_FAKE_INSTALL_STATUS", "17") + .env("LOG", root.join("commands.log")) + .env("PLOYZ_TEST_PID", root.join("preflight.pid")) .env_remove("SUDO_USER") .output() .unwrap() @@ -186,9 +188,10 @@ case "$*" in *' whoami') echo deploy ;; *' sudo true') ;; *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'mkdir -m 700 -- '*) ;; *'chmod 700 '*' version') printf '%s\n' '{}' ;; *" 'install' "*) ;; - *'rm -f -- '*) ;; + *'rm -rf -- '*) ;; *' true') exit 23 ;; *) echo "unexpected ssh invocation: $*" >&2; exit 24 ;; esac @@ -205,7 +208,7 @@ esac let common = [ "machine", "init", - "deploy@host", + "deploy@2001:db8::1", "--yes", "--storage", "none", @@ -228,6 +231,13 @@ esac assert!(!output.status.success(), "fake daemon must not accept RPC"); let setup_log = fs::read_to_string(&log).unwrap(); assert!(setup_log.contains("scp "), "{setup_log}"); + assert!( + setup_log.contains("deploy@[2001:db8::1]:/tmp/ployz-bootstrap-"), + "{setup_log}" + ); + assert!(!setup_log.contains(" -P 1 "), "{setup_log}"); + assert!(setup_log.contains(daemon_archive()), "{setup_log}"); + assert!(setup_log.contains("checksums.txt"), "{setup_log}"); assert!( setup_log.contains(&format!( "'install' '--version' '{}' '--storage' 'none' '--group-user' 'deploy'", @@ -235,6 +245,8 @@ esac )), "{setup_log}" ); + assert!(setup_log.contains("'--release-dir' '/tmp/ployz-bootstrap-")); + assert!(setup_log.contains("rm -rf --"), "{setup_log}"); assert!(!setup_log.contains("base64"), "{setup_log}"); fs::write(&log, "").unwrap(); @@ -242,7 +254,7 @@ esac .args([ "machine", "init", - "deploy@host", + "deploy@2001:db8::1", "--no-install", "--yes", "--storage", @@ -266,3 +278,122 @@ esac assert!(!no_install_log.contains("scp "), "{no_install_log}"); assert!(!no_install_log.contains("'install'"), "{no_install_log}"); } + +#[test] +fn remote_preflight_timeout_kills_the_child_and_cleans_the_stage() { + let fixture = Fixture::new(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &fixture.path().join("installer.log")); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("ssh"), + &format!( + r#"#!/bin/sh +printf 'ssh %s\n' "$*" >> "$LOG" +case "$*" in + *' whoami') echo root ;; + *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'mkdir -m 700 -- '*) ;; + *'chmod 700 '*' version') echo $$ > "$PLOYZ_TEST_PID"; exec sleep 30 ;; + *'rm -rf -- '*) ;; + *) exit 24 ;; +esac +"#, + env::consts::ARCH + ), + ); + write_executable(&bin.join("scp"), "#!/bin/sh\nexit 0\n"); + + let output = run( + &[ + "machine", + "init", + "root@host", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-timeout", + ], + fixture.path(), + &release, + &bin, + ); + + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("timed out after 10 seconds"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let log = fs::read_to_string(log).unwrap(); + assert!(log.contains("rm -rf --"), "{log}"); + let pid = fs::read_to_string(fixture.path().join("preflight.pid")) + .unwrap() + .trim() + .to_owned(); + assert!( + !Command::new("kill") + .args(["-0", &pid]) + .status() + .unwrap() + .success(), + "timed-out SSH child {pid} survived" + ); +} + +#[test] +fn remote_cleanup_failure_is_returned_after_successful_installation() { + let fixture = Fixture::new(); + let release = release_fixture(fixture.path(), &fixture.path().join("installer.log")); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("ssh"), + &format!( + r#"#!/bin/sh +case "$*" in + *' whoami') echo root ;; + *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'mkdir -m 700 -- '*) ;; + *'chmod 700 '*' version') printf '%s\n' '{}' ;; + *" 'install' "*) ;; + *'rm -rf -- '*) exit 29 ;; + *) exit 24 ;; +esac +"#, + env::consts::ARCH, + env!("CARGO_PKG_VERSION") + ), + ); + write_executable(&bin.join("scp"), "#!/bin/sh\nexit 0\n"); + + let output = run( + &[ + "machine", + "init", + "root@host", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-cleanup", + ], + fixture.path(), + &release, + &bin, + ); + + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("remote bootstrap cleanup exited with exit status: 29"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 309df66231..3a3a30bcce 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -31,7 +31,7 @@ Beta: `v0.2.0-beta.1` with Cargo version `0.2.0-beta.1`. Nightly, `-rc`, and oth Fast CI on `main` is the merge gate. The ignored cluster suite is informing. A red nightly there does not block a tag or Publish. -Before Publish, run `scripts/qualify-release.sh` against real Linux Machines using the draft musl archives (`PLOYZ_ARTIFACT_DIR`). That script does not pick a cloud vendor. You pass SSH targets. Those hosts must be uninitialized Machines unless you set `PLOYZ_QUALIFY_RESET=1`, which accepts a reset and destroys managed containers. Pass a qualification key with `PLOYZ_QUALIFY_SSH_KEY` or `-i` in `PLOYZ_QUALIFY_SSH_OPTS` so `ssh` and `ployz machine init`/`add` use the same identity. +Before Publish, run `scripts/qualify-release.sh` against real Linux Machines using the draft musl archives (`PLOYZ_ARTIFACT_DIR`). That script does not pick a cloud vendor. You pass SSH targets. Those hosts must be uninitialized Machines unless you set `PLOYZ_QUALIFY_RESET=1`, which accepts a reset and destroys managed containers. Pass a qualification key with `PLOYZ_QUALIFY_SSH_KEY`; the normal `ployz machine init`/`add` setup path verifies and installs the local release artifacts. When the informing cluster suite and that run disagree, the real Machines are the authority. Testkit bugs do not block a release. diff --git a/scripts/qualify-release.sh b/scripts/qualify-release.sh index 9b83d172fb..b307ee1f8d 100755 --- a/scripts/qualify-release.sh +++ b/scripts/qualify-release.sh @@ -6,7 +6,7 @@ # PLOYZ_ARTIFACT_DIR=/path/to/dist \ # scripts/qualify-release.sh # -# Optional: PLOYZ_QUALIFY_DRY_RUN=1, PLOYZ_QUALIFY_SSH_OPTS, PLOYZ_QUALIFY_SSH_KEY, +# Optional: PLOYZ_QUALIFY_DRY_RUN=1, PLOYZ_QUALIFY_SSH_KEY, # PLOYZ_QUALIFY_CONTEXT, PLOYZ_QUALIFY_RESET=1. # Hosts must be uninitialized unless PLOYZ_QUALIFY_RESET=1. Reset destroys # managed containers on that Machine. @@ -18,9 +18,9 @@ COMPOSE_DIR=$ROOT/scripts/qualify-release HOSTS=${PLOYZ_QUALIFY_HOSTS:-} ARTIFACT_DIR=${PLOYZ_ARTIFACT_DIR:-} DRY_RUN=${PLOYZ_QUALIFY_DRY_RUN:-0} -SSH_OPTS=${PLOYZ_QUALIFY_SSH_OPTS:-"-o StrictHostKeyChecking=accept-new"} CONTEXT=${PLOYZ_QUALIFY_CONTEXT:-qualify} RESET=${PLOYZ_QUALIFY_RESET:-0} +SSH_KEY=${PLOYZ_QUALIFY_SSH_KEY:-} CONFIG_DIR= error() { echo "ERROR: $*" >&2; exit 1; } @@ -55,39 +55,6 @@ cli_archive() { esac } -daemon_archive() { - case "$1" in - x86_64) echo ployzd_linux_amd64.tar.gz ;; - aarch64) echo ployzd_linux_arm64.tar.gz ;; - *) error "unsupported Machine architecture $1" ;; - esac -} - -if [ -n "${PLOYZ_QUALIFY_SSH_KEY:-}" ]; then - SSH_OPTS="$SSH_OPTS -i $PLOYZ_QUALIFY_SSH_KEY" -fi - -ssh_identity() { - # shellcheck disable=SC2086 - set -- $SSH_OPTS - while [ $# -gt 0 ]; do - case $1 in - -i | --identity) - printf '%s\n' "${2:-}" - return - ;; - esac - shift - done -} - -SSH_KEY=$(ssh_identity) - -ssh_host() { - # shellcheck disable=SC2086 - ssh $SSH_OPTS "$1" "${@:2}" -} - [ -n "$HOSTS" ] || error "set PLOYZ_QUALIFY_HOSTS to one or more user@host targets" [ -n "$ARTIFACT_DIR" ] || error "set PLOYZ_ARTIFACT_DIR to the draft archive directory" [ -f "$COMPOSE_DIR/compose.yaml" ] || error "missing $COMPOSE_DIR/compose.yaml" @@ -113,54 +80,24 @@ if [ "$DRY_RUN" != 0 ]; then else echo "reset: no (initialized hosts fail without PLOYZ_QUALIFY_RESET=1)" fi - echo "steps: verify and run temporary artifact ployzd, install daemon from --release-dir (always replace), machine init --no-install, machine add, deploy named volume qualify-data, volume ls" + echo "steps: verify the artifact CLI, run normal machine init/add from the local release, deploy named volume qualify-data, volume ls" exit 0 fi work=$(mktemp -d) trap 'rm -rf "$work" "$CONFIG_DIR"' EXIT +verify_archive "$(cli_archive)" tar -xzf "$ARTIFACT_DIR/$(cli_archive)" -C "$work" PLOYZ=$work/ployz [ -x "$PLOYZ" ] || error "CLI archive did not contain ployz" version=$("$PLOYZ" version) || error "could not read ployz version from the artifact" CONFIG_DIR=$(mktemp -d) export PLOYZ_CONFIG=$CONFIG_DIR/config.yaml - -install_host() { - local host=$1 arch archive remote bootstrap observed status - arch=$(ssh_host "$host" uname -m) - archive=$(daemon_archive "$arch") - verify_archive "$archive" - bootstrap=$(mktemp -d "$work/bootstrap.XXXXXX") - tar -xzf "$ARTIFACT_DIR/$archive" -C "$bootstrap" ployzd - [ -x "$bootstrap/ployzd" ] || error "$archive did not contain executable ployzd" - remote=/tmp/ployz-qualify-$$ - ssh_host "$host" mkdir -p "$remote" - # shellcheck disable=SC2086 - if ! scp $SSH_OPTS "$bootstrap/ployzd" "$ARTIFACT_DIR/$archive" "$ARTIFACT_DIR/checksums.txt" "$host:$remote/"; then - ssh_host "$host" rm -rf "$remote" || true - return 1 - fi - if ! observed=$(ssh_host "$host" "$remote/ployzd" version) || [ "$observed" != "$version" ]; then - ssh_host "$host" rm -rf "$remote" || true - error "bootstrap daemon on $host reported ${observed:-no version}, expected $version" - fi - status=0 - ssh_host "$host" sudo "$remote/ployzd" install \ - --version "$version" \ - --release-dir "$remote" || status=$? - ssh_host "$host" rm -rf "$remote" || true - return "$status" -} - -for host in "${HOST_LIST[@]}"; do - echo "install $host" - install_host "$host" -done +export PLOYZ_RELEASE_DIR=$ARTIFACT_DIR first=${HOST_LIST[0]} echo "machine init $first" -init_cmd=("$PLOYZ" machine init --no-install --context "$CONTEXT") +init_cmd=("$PLOYZ" machine init --version "$version" --context "$CONTEXT") if [ "$RESET" != 0 ]; then init_cmd+=(--yes) fi @@ -172,7 +109,7 @@ fi i=1 while [ "$i" -lt "${#HOST_LIST[@]}" ]; do echo "machine add ${HOST_LIST[$i]}" - add_cmd=("$PLOYZ" machine add --yes --no-install --context "$CONTEXT") + add_cmd=("$PLOYZ" machine add --yes --version "$version" --context "$CONTEXT") if [ -n "$SSH_KEY" ]; then add_cmd+=(--ssh-key "$SSH_KEY") fi diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index 5b26508090..bc1d9dcdc8 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -53,19 +53,20 @@ case "$1" in action=$2 case "$action" in init|add) ;; *) exit 1 ;; esac shift 2 - reset=no key= target= context= no_install=no + reset=no key= target= context= no_install=no version= while [ "$#" -gt 0 ]; do case "$1" in --yes) reset=yes ;; --ssh-key) key=$2; shift ;; --context) context=$2; shift ;; + --version) version=$2; shift ;; --no-install) no_install=yes ;; root@*) target=$1 ;; *) echo "unexpected machine argument: $1" >&2; exit 1 ;; esac shift done - printf '%s target=%s reset=%s key=%s context=%s no_install=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" >> "$LOG" + printf '%s target=%s reset=%s key=%s context=%s no_install=%s version=%s release=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" "$version" "${PLOYZ_RELEASE_DIR:-}" >> "$LOG" ;; *) echo "unexpected command: $*" >&2; exit 1 ;; esac @@ -84,42 +85,18 @@ done sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 > checksums.txt ) -export SSH_LOG=$TMP/ssh.log -cat > "$TMP/bin/ssh" <<'SSH' -#!/bin/sh -set -eu -key= -while [ "$#" -gt 0 ]; do - case "$1" in - -i) key=$2; shift 2 ;; - -o) shift 2 ;; - *) break ;; - esac -done -[ "$key" = /tmp/qualify-key ] || { echo 'remote command lost SSH identity' >&2; exit 1; } -printf '%s identity=%s\n' "${0##*/}" "$key" >> "$SSH_LOG" -case "$*" in - *'/ployzd version') echo 1.2.3 ;; - *'uname -m') echo x86_64 ;; -esac -SSH -chmod 0755 "$TMP/bin/ssh" -ln -s ssh "$TMP/bin/scp" for reset in 0 1; do : > "$LOG" PATH="$TMP/bin:$PATH" PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$TMP" \ - PLOYZ_QUALIFY_SSH_OPTS='-i /tmp/qualify-key' PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ + PLOYZ_QUALIFY_SSH_KEY=/tmp/qualify-key PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ "$ROOT/scripts/qualify-release.sh" >/dev/null expected_reset=no [ "$reset" = 0 ] || expected_reset=yes - grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=yes" "$LOG" || fail "init lost its target, reset policy, or SSH identity" - grep -Fxq 'add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=yes' "$LOG" || fail "add lost its target or SSH identity" + grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 release=$TMP" "$LOG" || fail "init lost its target, release, version, reset policy, or SSH identity" + grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 release=$TMP" "$LOG" || fail "add lost its target, release, version, or SSH identity" grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "named-volume fixture was not deployed" grep -Fxq 'volume context=qualify file= yes=no' "$LOG" || fail "volume ls did not query the qualification context" done -for command in ssh scp; do - grep -Fxq "$command identity=/tmp/qualify-key" "$SSH_LOG" || fail "$command was not exercised" -done cat > "$TMP/bin/cargo" <<'CARGO' #!/bin/sh @@ -149,7 +126,7 @@ output=$( printf '%s\n' "$output" | grep -Fq 'qualify dry-run' || fail "dry-run did not print the plan" printf '%s\n' "$output" | grep -Fq 'named volume' || fail "dry-run omitted the named-volume step" printf '%s\n' "$output" | grep -Fq 'reset: no' || fail "dry-run omitted the default no-reset policy" -printf '%s\n' "$output" | grep -Fq 'always replace' || fail "dry-run omitted forced daemon replace" +printf '%s\n' "$output" | grep -Fq 'normal machine init/add' || fail "dry-run omitted normal setup path" "$ROOT/scripts/check-layer3-runner.sh" echo "qualify-release contracts passed" From 6c8e104624f20706ddce7f869c82546df81435dd Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:36:23 +0000 Subject: [PATCH 09/30] fix: preserve upgrade attempt evidence --- crates/ployz-core/build.rs | 1 + crates/ployz-core/tests/contract.rs | 5 +- crates/ployzd/src/installer/host.rs | 78 +++++++- crates/ployzd/src/installer/upgrade.rs | 189 +++++++++++++++++++ crates/ployzd/src/machine_api/local.rs | 15 +- crates/ployzd/src/machine_api/local_tests.rs | 69 ++++++- 6 files changed, 348 insertions(+), 9 deletions(-) diff --git a/crates/ployz-core/build.rs b/crates/ployz-core/build.rs index b79ff89a72..ce9d0352cd 100644 --- a/crates/ployz-core/build.rs +++ b/crates/ployz-core/build.rs @@ -1,6 +1,7 @@ include!("src/rpc_catalog.rs"); fn main() { + println!("cargo:rerun-if-changed=src/rpc_catalog.rs"); macro_rules! build_service { ( package $package:literal diff --git a/crates/ployz-core/tests/contract.rs b/crates/ployz-core/tests/contract.rs index cdc6962fcf..0c527e7e1f 100644 --- a/crates/ployz-core/tests/contract.rs +++ b/crates/ployz-core/tests/contract.rs @@ -1513,7 +1513,10 @@ fn machine_upgrade_request_and_receipt_have_one_typed_wire_contract() { }; let response = RpcResponse::from(attempt.clone()); assert_eq!( - response.decode::().unwrap(), + response + .clone() + .decode::() + .unwrap(), attempt ); assert_eq!( diff --git a/crates/ployzd/src/installer/host.rs b/crates/ployzd/src/installer/host.rs index 2ca869412e..ee03efb3cd 100644 --- a/crates/ployzd/src/installer/host.rs +++ b/crates/ployzd/src/installer/host.rs @@ -9,8 +9,10 @@ use std::{ }; use semver::Version; +use tonic::transport::Endpoint; use crate::filesystem::atomic_write; +use ployz_core::{DescribeContractRequest, MachineRpcClient, op}; use super::release::{fetch, installed_release}; use super::{Error, InstallPaths, PLOYZ_USER, command_exists, run_apt, run_host, systemctl}; @@ -296,12 +298,78 @@ pub(super) async fn verify_running_daemon( )); } match installed_release(&paths.bin_dir.join("ployzd")).await? { - Some(observed) if &observed == target => Ok(()), + Some(observed) if &observed == target => {} Some(observed) => Err(Error::Verification(format!( "activated daemon reported {observed}, expected {target}" - ))), - None => Err(Error::Verification( - "activated daemon no longer reports a valid version".into(), - )), + )))?, + None => { + return Err(Error::Verification( + "activated daemon no longer reports a valid version".into(), + )); + } + } + verify_daemon_contract(&paths.run_dir.join("ployz.sock"), target).await +} + +async fn verify_daemon_contract(socket: &Path, target: &Version) -> Result<(), Error> { + let endpoint = + Endpoint::from_shared(format!("unix:{}", socket.display())).map_err(|error| { + Error::Verification(format!("invalid Machine API socket address: {error}")) + })?; + let channel = tokio::time::timeout(std::time::Duration::from_secs(10), endpoint.connect()) + .await + .map_err(|_| Error::Verification("Machine API readiness timed out after 10s".into()))? + .map_err(|error| Error::Verification(format!("Machine API is not ready: {error}")))?; + let payload = op::DescribeContract::into_request(DescribeContractRequest {}) + .encode() + .map_err(|error| Error::Verification(format!("encode readiness request: {error}")))?; + let response = tokio::time::timeout( + std::time::Duration::from_secs(10), + MachineRpcClient::new(channel).describe_contract(tonic::Request::new(payload)), + ) + .await + .map_err(|_| Error::Verification("Machine API readiness timed out after 10s".into()))? + .map_err(|error| Error::Verification(format!("Machine API is not ready: {error}")))? + .into_inner() + .decode_response() + .and_then(|response| response.decode::()) + .map_err(|error| Error::Verification(format!("decode readiness response: {error}")))?; + require_running_version(&response.daemon_version, target) +} + +fn require_running_version(observed: &str, target: &Version) -> Result<(), Error> { + let observed = Version::parse(observed).map_err(|_| { + Error::Verification(format!( + "running Machine API reported invalid daemon version {observed:?}" + )) + })?; + if &observed == target { + Ok(()) + } else { + Err(Error::Verification(format!( + "running Machine API reported {observed}, expected {target}" + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn running_machine_api_version_must_match_the_target() { + let target = Version::parse("1.2.3-beta.4").unwrap(); + assert!(require_running_version("1.2.3-beta.4", &target).is_ok()); + assert!(matches!( + require_running_version("1.2.3-beta.3", &target), + Err(Error::Verification(message)) + if message == "running Machine API reported 1.2.3-beta.3, expected 1.2.3-beta.4" + )); + assert!(matches!( + require_running_version("not-a-version", &target), + Err(Error::Verification(message)) + if message + == "running Machine API reported invalid daemon version \"not-a-version\"" + )); } } diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs index f00935a52c..19bc681a3f 100644 --- a/crates/ployzd/src/installer/upgrade.rs +++ b/crates/ployzd/src/installer/upgrade.rs @@ -93,6 +93,24 @@ impl StoredReleaseSource { } } +/// Return the durable result of retrying the same request without requiring installation +/// ownership. The receipt is atomically replaced, so an active worker can be observed safely. +pub fn existing_request( + request: &RequestMachineUpgradeRequest, + data_dir: &Path, +) -> Result, Error> { + let Some(stored) = read_optional(data_dir)? else { + return Ok(None); + }; + if stored.attempt.attempt_id() != request.attempt_id { + return Ok(None); + } + if stored.requested != request.release { + return Err(Error::AttemptConflict(request.attempt_id)); + } + Ok(Some(stored.attempt)) +} + /// Persist an accepted attempt and launch its old-daemon worker while installation admission is /// held. Reusing the same attempt ID returns the same attempt. pub async fn request_locked( @@ -421,6 +439,9 @@ fn write(data_dir: &Path, stored: &StoredAttempt) -> Result<(), Error> { mod tests { use super::*; + const CONTRACT_CASE: &str = "PLOYZ_UPGRADE_CONTRACT_CASE"; + const CONTRACT_ROOT: &str = "PLOYZ_UPGRADE_CONTRACT_ROOT"; + #[test] fn stored_attempt_round_trips_without_exposing_local_source_in_response() { let attempt_id = MachineUpgradeAttemptId::random(); @@ -446,4 +467,172 @@ mod tests { stored ); } + + #[test] + fn upgrade_attempt_contract() { + if let Ok(case) = env::var(CONTRACT_CASE) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(run_contract_case(&case)); + let root = PathBuf::from(env::var_os(CONTRACT_ROOT).unwrap()); + fs::write(root.join("child-completed"), case).unwrap(); + return; + } + + for case in ["retry-active", "interrupted", "launch-failed"] { + let root = tempfile::Builder::new() + .prefix(&format!("ployzd-upgrade-{case}-")) + .tempdir() + .unwrap(); + let commands = root.path().join("commands"); + fs::create_dir(&commands).unwrap(); + write_script( + &commands.join("systemd-run"), + if case == "launch-failed" { + "echo worker launch refused >&2; exit 1" + } else { + "printf '%s\\n' \"$*\" >> \"$PLOYZ_UPGRADE_COMMAND_LOG\"" + }, + ); + write_script( + &commands.join("systemctl"), + if case == "interrupted" { + "exit 3" + } else { + "exit 0" + }, + ); + fs::create_dir(root.path().join("release")).unwrap(); + let output = std::process::Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + "installer::upgrade::tests::upgrade_attempt_contract", + "--nocapture", + ]) + .env(CONTRACT_CASE, case) + .env(CONTRACT_ROOT, root.path()) + .env(QUALIFICATION_RELEASE_DIR, root.path().join("release")) + .env( + "PLOYZ_UPGRADE_COMMAND_LOG", + root.path().join("commands.log"), + ) + .env("PATH", &commands) + .output() + .unwrap(); + assert!( + output.status.success(), + "{case}: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::read_to_string(root.path().join("child-completed")).unwrap(), + case + ); + } + } + + async fn run_contract_case(case: &str) { + let root = PathBuf::from(env::var_os(CONTRACT_ROOT).unwrap()); + let data = root.join("data"); + let run = root.join("run"); + let admission = admission::Admission::new(&run, &data); + let attempt_id = MachineUpgradeAttemptId::parse("a".repeat(32)).unwrap(); + let request = RequestMachineUpgradeRequest { + attempt_id, + release: MachineRelease::parse("1.2.3").unwrap(), + }; + let guard = admission.try_install().unwrap(); + let accepted = request_locked(request.clone(), &data, &run, &guard) + .await + .unwrap(); + drop(guard); + + match case { + "retry-active" => { + assert!(matches!(accepted, MachineUpgradeAttempt::Accepted { .. })); + assert_eq!( + existing_request(&request, &data).unwrap(), + Some(accepted.clone()) + ); + assert!(matches!( + admission.try_mutation(), + Err(admission::Error::Busy) + )); + let conflict = RequestMachineUpgradeRequest { + attempt_id, + release: MachineRelease::parse("1.2.4").unwrap(), + }; + assert!(matches!( + existing_request(&conflict, &data), + Err(Error::AttemptConflict(id)) if id == attempt_id + )); + + let guard = admission.try_install().unwrap(); + assert_eq!( + request_locked(request, &data, &run, &guard).await.unwrap(), + accepted + ); + let other = RequestMachineUpgradeRequest { + attempt_id: MachineUpgradeAttemptId::parse("b".repeat(32)).unwrap(), + release: MachineRelease::parse("1.2.3").unwrap(), + }; + assert!(matches!( + request_locked(other, &data, &run, &guard).await, + Err(Error::Busy) + )); + drop(guard); + let log = fs::read_to_string(root.join("commands.log")).unwrap(); + assert_eq!(log.lines().count(), 1, "{log}"); + for required in [ + "--property=Type=exec", + "--property=RuntimeMaxSec=15min", + "--property=NoNewPrivileges=yes", + "--property=ProtectSystem=full", + "upgrade-worker", + "--attempt", + attempt_id.as_str(), + ] { + assert!(log.contains(required), "missing {required}: {log}"); + } + } + "interrupted" => { + let observed = inspect(Some(attempt_id), &data, &run).await.unwrap(); + assert!(matches!( + observed, + MachineUpgradeAttempt::Interrupted { + stage: MachineUpgradeStage::Launching, + .. + } + )); + assert!(!admission.active()); + assert!(admission.try_mutation().is_ok()); + } + "launch-failed" => { + assert!(matches!( + accepted, + MachineUpgradeAttempt::Failed { + stage: MachineUpgradeStage::Launching, + ref error, + .. + } if error == "launch Machine upgrade worker: worker launch refused" + )); + assert!(!admission.active()); + assert_eq!( + inspect(Some(attempt_id), &data, &run).await.unwrap(), + accepted + ); + } + other => panic!("unknown upgrade contract case {other}"), + } + } + + fn write_script(path: &Path, body: &str) { + use std::os::unix::fs::PermissionsExt; + + fs::write(path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } } diff --git a/crates/ployzd/src/machine_api/local.rs b/crates/ployzd/src/machine_api/local.rs index 8cdae79a50..a9bd3bf333 100644 --- a/crates/ployzd/src/machine_api/local.rs +++ b/crates/ployzd/src/machine_api/local.rs @@ -709,14 +709,25 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; + let (data_dir, run_dir) = match self.local.upgrade_paths() { + Ok(paths) => paths, + Err(error) => return local_error(error), + }; + match crate::installer::upgrade::existing_request(&request, &data_dir) { + Ok(Some(attempt)) => return respond(attempt), + Ok(None) => {} + Err(error) => return respond(upgrade_error(error)), + } let admission = match self.local.try_upgrade_admission() { Ok(admission) => admission, Err(error) => return local_error(error), }; + debug_assert_eq!(admission.data_dir, data_dir); + debug_assert_eq!(admission.run_dir, run_dir); match crate::installer::upgrade::request_locked( request, - &admission.data_dir, - &admission.run_dir, + &data_dir, + &run_dir, &admission.install, ) .await diff --git a/crates/ployzd/src/machine_api/local_tests.rs b/crates/ployzd/src/machine_api/local_tests.rs index aa2c49b336..1876c0646f 100644 --- a/crates/ployzd/src/machine_api/local_tests.rs +++ b/crates/ployzd/src/machine_api/local_tests.rs @@ -6,7 +6,8 @@ use crate::machine::{LocalMachineError, LocalMachineStore, StoreError}; use ployz_core::{ ContainerAddress, ContainerId, ContainerKind, ContainerObservation, ContainerRuntimeObservation, GET_CONTAINER_OBSERVATIONS_CAPABILITY, - GetContainerObservationsRequest, HealthObservation, MachineId, MachineRpc, ProjectName, + GetContainerObservationsRequest, HealthObservation, MachineId, MachineRelease, MachineRpc, + MachineUpgradeAttemptId, ProjectName, RequestMachineUpgradeRequest, ResetRequest, ResolvedServiceSpec, RpcErrorCode, RpcResponseBody, RuntimeWatchRequest, ServiceId, ServiceName, op, }; @@ -101,6 +102,72 @@ fn allocator_not_quiet_is_retryable_unavailable() { assert_eq!(error.message, "Allocator is not quiet"); } +#[tokio::test] +async fn upgrade_and_machine_mutations_refuse_each_other_at_the_rpc_boundary() { + let data_dir = std::env::temp_dir().join(format!( + "ployzd-upgrade-rpc-admission-{}", + MachineId::random() + )); + let store = Arc::new(Mutex::new(LocalMachineStore::open(&data_dir).unwrap())); + let service = MachineService::with_cluster(store, watch::channel(false).0, None); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let mutation = tokio::spawn({ + let local = service.local(); + let started = Arc::clone(&started); + let release = Arc::clone(&release); + async move { + local + .run_mutation(async move { + started.notify_one(); + release.notified().await; + Ok(()) + }) + .await + } + }); + started.notified().await; + + let response = service + .request_machine_upgrade(Request::new( + op::RequestMachineUpgrade::into_request(RequestMachineUpgradeRequest { + attempt_id: MachineUpgradeAttemptId::parse("a".repeat(32)).unwrap(), + release: MachineRelease::parse("1.2.3").unwrap(), + }) + .encode() + .unwrap(), + )) + .await + .unwrap() + .into_inner() + .decode_response() + .unwrap(); + assert!(matches!( + response.body, + RpcResponseBody::Error(error) if error.code == RpcErrorCode::Conflict + )); + release.notify_one(); + mutation.await.unwrap().unwrap(); + + std::fs::write(data_dir.join(".upgrade-active"), "attempt").unwrap(); + let response = service + .reset(Request::new( + op::Reset::into_request(ResetRequest {}).encode().unwrap(), + )) + .await + .unwrap() + .into_inner() + .decode_response() + .unwrap(); + assert!(matches!( + response.body, + RpcResponseBody::Error(error) if error.code == RpcErrorCode::Conflict + )); + + drop(service); + std::fs::remove_dir_all(data_dir).unwrap(); +} + #[test] fn not_allocator_does_not_allocate() { let RpcResponseBody::Error(error) = local_error(LocalMachineError::NotAllocator) From 873c4fadd4ee5ebc5f4661b4fe444b573515baac Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:42:33 +0000 Subject: [PATCH 10/30] fix(setup): verify downloads before staging --- crates/ployz/src/provisioning/bootstrap.rs | 128 ++++++++++----------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/crates/ployz/src/provisioning/bootstrap.rs b/crates/ployz/src/provisioning/bootstrap.rs index f5b7cac179..ae7109087b 100644 --- a/crates/ployz/src/provisioning/bootstrap.rs +++ b/crates/ployz/src/provisioning/bootstrap.rs @@ -39,52 +39,53 @@ impl Bootstrap { })?; let archive = directory.path().join(archive_name); let checksums = directory.path().join("checksums.txt"); - let local_source = if let Some(source) = env::var_os("PLOYZ_RELEASE_DIR") { - let source = PathBuf::from(source); - copy_file( - &source.join(archive_name), - &archive, - "copy bootstrap daemon archive", - )?; - copy_file( - &source.join("checksums.txt"), - &checksums, - "copy bootstrap release checksums", - )?; - true - } else { - let base = format!( - "{RELEASE_REPOSITORY}/releases/download/v{}", - env!("CARGO_PKG_VERSION") - ); - let client = reqwest::Client::builder() - .https_only(true) - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(120)) - .user_agent("ployz-bootstrap") - .build() - .map_err(|source| ProvisionError::BootstrapDownload { - stage: "build bootstrap download client", - source, - })?; - download( - &client, - &format!("{base}/{archive_name}"), - &archive, - "download bootstrap daemon archive", - ) - .await?; - download( - &client, - &format!("{base}/checksums.txt"), - &checksums, - "download bootstrap release checksums", - ) - .await?; - false - }; + let (archive_bytes, checksum_bytes, local_source) = + if let Some(source) = env::var_os("PLOYZ_RELEASE_DIR") { + let source = PathBuf::from(source); + let archive_bytes = + read_file(&source.join(archive_name), "copy bootstrap daemon archive")?; + let checksum_bytes = read_file( + &source.join("checksums.txt"), + "copy bootstrap release checksums", + )?; + (archive_bytes, checksum_bytes, true) + } else { + let base = format!( + "{RELEASE_REPOSITORY}/releases/download/v{}", + env!("CARGO_PKG_VERSION") + ); + let client = reqwest::Client::builder() + .https_only(true) + .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(30)) + .user_agent("ployz-bootstrap") + .build() + .map_err(|source| ProvisionError::BootstrapDownload { + stage: "build bootstrap download client", + source, + })?; + let archive_bytes = download( + &client, + &format!("{base}/{archive_name}"), + "download bootstrap daemon archive", + ) + .await?; + let checksum_bytes = download( + &client, + &format!("{base}/checksums.txt"), + "download bootstrap release checksums", + ) + .await?; + (archive_bytes, checksum_bytes, false) + }; - verify_checksum(&archive, &checksums, archive_name)?; + verify_checksum(&archive_bytes, &checksum_bytes, archive_name)?; + write_file(&archive, &archive_bytes, "stage bootstrap daemon archive")?; + write_file( + &checksums, + &checksum_bytes, + "stage bootstrap release checksums", + )?; let daemon = directory.path().join("ployzd"); let mut extract = Command::new("tar"); extract @@ -158,10 +159,9 @@ pub(super) async fn verify_command_version( async fn download( client: &reqwest::Client, url: &str, - destination: &Path, stage: &'static str, -) -> Result<(), ProvisionError> { - let bytes = client +) -> Result, ProvisionError> { + client .get(url) .send() .await @@ -170,26 +170,22 @@ async fn download( .map_err(|source| ProvisionError::BootstrapDownload { stage, source })? .bytes() .await - .map_err(|source| ProvisionError::BootstrapDownload { stage, source })?; - fs::write(destination, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) + .map(|bytes| bytes.to_vec()) + .map_err(|source| ProvisionError::BootstrapDownload { stage, source }) } -fn copy_file(source: &Path, destination: &Path, stage: &'static str) -> Result<(), ProvisionError> { - fs::copy(source, destination) - .map(|_| ()) - .map_err(|source| ProvisionError::BootstrapIo { stage, source }) +fn read_file(path: &Path, stage: &'static str) -> Result, ProvisionError> { + fs::read(path).map_err(|source| ProvisionError::BootstrapIo { stage, source }) } fn verify_checksum( - archive: &Path, - checksums: &Path, + archive: &[u8], + checksums: &[u8], archive_name: &str, ) -> Result<(), ProvisionError> { - let checksums = - fs::read_to_string(checksums).map_err(|source| ProvisionError::BootstrapIo { - stage: "read bootstrap release checksums", - source, - })?; + let checksums = std::str::from_utf8(checksums).map_err(|error| { + ProvisionError::BootstrapVerification(format!("checksums.txt is not UTF-8: {error}")) + })?; let expected = checksums .lines() .find_map(|line| { @@ -206,11 +202,7 @@ fn verify_checksum( "checksums.txt has no SHA-256 hash for {archive_name}" )) })?; - let bytes = fs::read(archive).map_err(|source| ProvisionError::BootstrapIo { - stage: "read bootstrap daemon archive", - source, - })?; - let actual = hex::encode(Sha256::digest(bytes)); + let actual = hex::encode(Sha256::digest(archive)); if actual == expected { Ok(()) } else { @@ -220,6 +212,10 @@ fn verify_checksum( } } +fn write_file(path: &Path, bytes: &[u8], stage: &'static str) -> Result<(), ProvisionError> { + fs::write(path, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + async fn command_status(stage: &'static str, command: Command) -> Result<(), ProvisionError> { let status = tokio::process::Command::from(command) .status() From 21ce3b595c62e26e54ffa3569a158adc45686d1e Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:42:33 +0000 Subject: [PATCH 11/30] fix(setup): verify downloads before staging --- crates/ployz/src/provisioning/bootstrap.rs | 128 ++++++++++----------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/crates/ployz/src/provisioning/bootstrap.rs b/crates/ployz/src/provisioning/bootstrap.rs index f5b7cac179..ae7109087b 100644 --- a/crates/ployz/src/provisioning/bootstrap.rs +++ b/crates/ployz/src/provisioning/bootstrap.rs @@ -39,52 +39,53 @@ impl Bootstrap { })?; let archive = directory.path().join(archive_name); let checksums = directory.path().join("checksums.txt"); - let local_source = if let Some(source) = env::var_os("PLOYZ_RELEASE_DIR") { - let source = PathBuf::from(source); - copy_file( - &source.join(archive_name), - &archive, - "copy bootstrap daemon archive", - )?; - copy_file( - &source.join("checksums.txt"), - &checksums, - "copy bootstrap release checksums", - )?; - true - } else { - let base = format!( - "{RELEASE_REPOSITORY}/releases/download/v{}", - env!("CARGO_PKG_VERSION") - ); - let client = reqwest::Client::builder() - .https_only(true) - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(120)) - .user_agent("ployz-bootstrap") - .build() - .map_err(|source| ProvisionError::BootstrapDownload { - stage: "build bootstrap download client", - source, - })?; - download( - &client, - &format!("{base}/{archive_name}"), - &archive, - "download bootstrap daemon archive", - ) - .await?; - download( - &client, - &format!("{base}/checksums.txt"), - &checksums, - "download bootstrap release checksums", - ) - .await?; - false - }; + let (archive_bytes, checksum_bytes, local_source) = + if let Some(source) = env::var_os("PLOYZ_RELEASE_DIR") { + let source = PathBuf::from(source); + let archive_bytes = + read_file(&source.join(archive_name), "copy bootstrap daemon archive")?; + let checksum_bytes = read_file( + &source.join("checksums.txt"), + "copy bootstrap release checksums", + )?; + (archive_bytes, checksum_bytes, true) + } else { + let base = format!( + "{RELEASE_REPOSITORY}/releases/download/v{}", + env!("CARGO_PKG_VERSION") + ); + let client = reqwest::Client::builder() + .https_only(true) + .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(30)) + .user_agent("ployz-bootstrap") + .build() + .map_err(|source| ProvisionError::BootstrapDownload { + stage: "build bootstrap download client", + source, + })?; + let archive_bytes = download( + &client, + &format!("{base}/{archive_name}"), + "download bootstrap daemon archive", + ) + .await?; + let checksum_bytes = download( + &client, + &format!("{base}/checksums.txt"), + "download bootstrap release checksums", + ) + .await?; + (archive_bytes, checksum_bytes, false) + }; - verify_checksum(&archive, &checksums, archive_name)?; + verify_checksum(&archive_bytes, &checksum_bytes, archive_name)?; + write_file(&archive, &archive_bytes, "stage bootstrap daemon archive")?; + write_file( + &checksums, + &checksum_bytes, + "stage bootstrap release checksums", + )?; let daemon = directory.path().join("ployzd"); let mut extract = Command::new("tar"); extract @@ -158,10 +159,9 @@ pub(super) async fn verify_command_version( async fn download( client: &reqwest::Client, url: &str, - destination: &Path, stage: &'static str, -) -> Result<(), ProvisionError> { - let bytes = client +) -> Result, ProvisionError> { + client .get(url) .send() .await @@ -170,26 +170,22 @@ async fn download( .map_err(|source| ProvisionError::BootstrapDownload { stage, source })? .bytes() .await - .map_err(|source| ProvisionError::BootstrapDownload { stage, source })?; - fs::write(destination, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) + .map(|bytes| bytes.to_vec()) + .map_err(|source| ProvisionError::BootstrapDownload { stage, source }) } -fn copy_file(source: &Path, destination: &Path, stage: &'static str) -> Result<(), ProvisionError> { - fs::copy(source, destination) - .map(|_| ()) - .map_err(|source| ProvisionError::BootstrapIo { stage, source }) +fn read_file(path: &Path, stage: &'static str) -> Result, ProvisionError> { + fs::read(path).map_err(|source| ProvisionError::BootstrapIo { stage, source }) } fn verify_checksum( - archive: &Path, - checksums: &Path, + archive: &[u8], + checksums: &[u8], archive_name: &str, ) -> Result<(), ProvisionError> { - let checksums = - fs::read_to_string(checksums).map_err(|source| ProvisionError::BootstrapIo { - stage: "read bootstrap release checksums", - source, - })?; + let checksums = std::str::from_utf8(checksums).map_err(|error| { + ProvisionError::BootstrapVerification(format!("checksums.txt is not UTF-8: {error}")) + })?; let expected = checksums .lines() .find_map(|line| { @@ -206,11 +202,7 @@ fn verify_checksum( "checksums.txt has no SHA-256 hash for {archive_name}" )) })?; - let bytes = fs::read(archive).map_err(|source| ProvisionError::BootstrapIo { - stage: "read bootstrap daemon archive", - source, - })?; - let actual = hex::encode(Sha256::digest(bytes)); + let actual = hex::encode(Sha256::digest(archive)); if actual == expected { Ok(()) } else { @@ -220,6 +212,10 @@ fn verify_checksum( } } +fn write_file(path: &Path, bytes: &[u8], stage: &'static str) -> Result<(), ProvisionError> { + fs::write(path, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + async fn command_status(stage: &'static str, command: Command) -> Result<(), ProvisionError> { let status = tokio::process::Command::from(command) .status() From 7583526a1febe1ddab7dd969ad8d61393901fe05 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:49:08 +0000 Subject: [PATCH 12/30] test(compose): create ignored FIFOs portably --- crates/ployz/src/compose/build_inputs.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/ployz/src/compose/build_inputs.rs b/crates/ployz/src/compose/build_inputs.rs index cb8cedc081..5094fe5c34 100644 --- a/crates/ployz/src/compose/build_inputs.rs +++ b/crates/ployz/src/compose/build_inputs.rs @@ -344,12 +344,10 @@ mod tests { fs::write(source.join(".dockerignore"), "ignored\n").unwrap(); fs::write(source.join("included"), "original").unwrap(); let _socket = UnixListener::bind(source.join("ignored/socket")).unwrap(); - rustix::fs::mknodat( + rustix::fs::mkfifoat( rustix::fs::CWD, source.join("ignored/fifo"), - rustix::fs::FileType::Fifo, rustix::fs::Mode::RUSR, - 0, ) .unwrap(); fs::write(source.join("ignored/unreadable"), "private").unwrap(); From dacaf265ff99444845ff1dddbedb35fe62e0448a Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:49:08 +0000 Subject: [PATCH 13/30] test(compose): create ignored FIFOs portably --- crates/ployz/src/compose/build_inputs.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/ployz/src/compose/build_inputs.rs b/crates/ployz/src/compose/build_inputs.rs index cb8cedc081..5094fe5c34 100644 --- a/crates/ployz/src/compose/build_inputs.rs +++ b/crates/ployz/src/compose/build_inputs.rs @@ -344,12 +344,10 @@ mod tests { fs::write(source.join(".dockerignore"), "ignored\n").unwrap(); fs::write(source.join("included"), "original").unwrap(); let _socket = UnixListener::bind(source.join("ignored/socket")).unwrap(); - rustix::fs::mknodat( + rustix::fs::mkfifoat( rustix::fs::CWD, source.join("ignored/fifo"), - rustix::fs::FileType::Fifo, rustix::fs::Mode::RUSR, - 0, ) .unwrap(); fs::write(source.join("ignored/unreadable"), "private").unwrap(); From 54ffb632fd686301e048d004029bdc88f050b6cf Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:54:16 +0000 Subject: [PATCH 14/30] test(compose): use POSIX mkfifo fixture --- crates/ployz/src/compose/build_inputs.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/ployz/src/compose/build_inputs.rs b/crates/ployz/src/compose/build_inputs.rs index 5094fe5c34..51f777e8cc 100644 --- a/crates/ployz/src/compose/build_inputs.rs +++ b/crates/ployz/src/compose/build_inputs.rs @@ -344,12 +344,13 @@ mod tests { fs::write(source.join(".dockerignore"), "ignored\n").unwrap(); fs::write(source.join("included"), "original").unwrap(); let _socket = UnixListener::bind(source.join("ignored/socket")).unwrap(); - rustix::fs::mkfifoat( - rustix::fs::CWD, - source.join("ignored/fifo"), - rustix::fs::Mode::RUSR, - ) - .unwrap(); + assert!( + std::process::Command::new("mkfifo") + .arg(source.join("ignored/fifo")) + .status() + .unwrap() + .success() + ); fs::write(source.join("ignored/unreadable"), "private").unwrap(); fs::set_permissions( source.join("ignored/unreadable"), From fd7eb84a895556820efbb917adf3f7b832d444f0 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:55:24 +0000 Subject: [PATCH 15/30] test: qualify bounded machine upgrades --- docs/RELEASE.md | 2 +- evidence/product-paths.tsv | 1 + scripts/qualify-release.sh | 325 ++++++++++++++++++++++++--- scripts/qualify-release/compose.yaml | 7 +- scripts/test-qualify-release.sh | 224 ++++++++++++++---- 5 files changed, 486 insertions(+), 73 deletions(-) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 3a3a30bcce..380f903a17 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -31,7 +31,7 @@ Beta: `v0.2.0-beta.1` with Cargo version `0.2.0-beta.1`. Nightly, `-rc`, and oth Fast CI on `main` is the merge gate. The ignored cluster suite is informing. A red nightly there does not block a tag or Publish. -Before Publish, run `scripts/qualify-release.sh` against real Linux Machines using the draft musl archives (`PLOYZ_ARTIFACT_DIR`). That script does not pick a cloud vendor. You pass SSH targets. Those hosts must be uninitialized Machines unless you set `PLOYZ_QUALIFY_RESET=1`, which accepts a reset and destroys managed containers. Pass a qualification key with `PLOYZ_QUALIFY_SSH_KEY`; the normal `ployz machine init`/`add` setup path verifies and installs the local release artifacts. +Before Publish, run `scripts/qualify-release.sh` against real Linux Machines using two draft musl archive sets: `PLOYZ_ARTIFACT_DIR` is the source release and `PLOYZ_UPGRADE_ARTIFACT_DIR` is the target release. The versions must differ. The script does not pick a cloud vendor. You pass SSH targets. Those hosts must be uninitialized Machines unless you set `PLOYZ_QUALIFY_RESET=1`, which accepts a reset and destroys managed containers. Pass a qualification key with `PLOYZ_QUALIFY_SSH_KEY`; the normal `ployz machine init`/`add` path installs the verified source, then the qualifier proves a target upgrade through client disconnection, persistent ZFS-backed traffic, corrupt preflight rejection, failed activation evidence, and explicit previous-binary repair. When the informing cluster suite and that run disagree, the real Machines are the authority. Testkit bugs do not block a release. diff --git a/evidence/product-paths.tsv b/evidence/product-paths.tsv index 30f3088ece..65173f8970 100644 --- a/evidence/product-paths.tsv +++ b/evidence/product-paths.tsv @@ -1,3 +1,4 @@ path rung1 rung2 rung3 rung4 rung5 machine-installer crates/ployzd/src/installer/mod.rs::tests::{installation_interface_contract,zfs_candidate_download_contract}; crates/ployzd/src/installer/release.rs::tests - - - ployzd install --software-only with a foreign-owned freshly built musl release on the retained Ubuntu systemd Machine machine-setup crates/ployz/src/provisioning.rs::tests; scripts/test-{missing-ssh-client,qualify-release}.sh crates/ployz/tests/cloud_init/daemon_sync.rs crates/ployz/tests/setup_cli.rs - scripts/qualify-release.sh with verified musl CLI and daemon archives on real Machines +machine-upgrade crates/ployzd/src/installer/upgrade.rs::tests::upgrade_attempt_contract; crates/ployzd/src/machine_api/local_tests.rs::upgrade_and_machine_mutations_refuse_each_other_at_the_rpc_boundary crates/ployz-core/tests/{contract,capabilities}.rs crates/ployz/tests/cli_shape.rs::machine_upgrade_requires_explicit_targets_and_has_typed_inspection - scripts/qualify-release.sh with real source/target musl releases, independent worker, disconnected client, persistent ZFS traffic, corrupt preflight, failed activation, and explicit repair diff --git a/scripts/qualify-release.sh b/scripts/qualify-release.sh index b307ee1f8d..84a10c2988 100755 --- a/scripts/qualify-release.sh +++ b/scripts/qualify-release.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# Qualify draft musl archives against real Linux Machines. +# Qualify two draft musl releases against real Linux Machines. # Pass SSH targets. This script does not pick a cloud vendor. # # PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' \ -# PLOYZ_ARTIFACT_DIR=/path/to/dist \ +# PLOYZ_ARTIFACT_DIR=/path/to/source/dist \ +# PLOYZ_UPGRADE_ARTIFACT_DIR=/path/to/target/dist \ # scripts/qualify-release.sh # # Optional: PLOYZ_QUALIFY_DRY_RUN=1, PLOYZ_QUALIFY_SSH_KEY, @@ -17,16 +18,23 @@ ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) COMPOSE_DIR=$ROOT/scripts/qualify-release HOSTS=${PLOYZ_QUALIFY_HOSTS:-} ARTIFACT_DIR=${PLOYZ_ARTIFACT_DIR:-} +UPGRADE_ARTIFACT_DIR=${PLOYZ_UPGRADE_ARTIFACT_DIR:-} DRY_RUN=${PLOYZ_QUALIFY_DRY_RUN:-0} CONTEXT=${PLOYZ_QUALIFY_CONTEXT:-qualify} RESET=${PLOYZ_QUALIFY_RESET:-0} SSH_KEY=${PLOYZ_QUALIFY_SSH_KEY:-} CONFIG_DIR= +TRAFFIC_PID= +TRAFFIC_STOP=/var/lib/ployz/qualification/traffic.stop +REMOTE_RELEASE_ROOT=/var/lib/ployz/qualification/releases +APP_URL=http://127.0.0.1:18082/identity +APP_VALUE=qualify-persistent-data error() { echo "ERROR: $*" >&2; exit 1; } -need_archive() { - [ -f "$ARTIFACT_DIR/$1" ] || error "PLOYZ_ARTIFACT_DIR is missing $1" +need_file() { + local directory=$1 label=$2 file=$3 + [ -f "$directory/$file" ] || error "$label is missing $file" } sha256() { @@ -38,11 +46,11 @@ sha256() { } verify_archive() { - local archive=$1 expected actual - expected=$(awk -v archive="$archive" '$2 == archive || $2 == "*" archive { print $1; exit }' "$ARTIFACT_DIR/checksums.txt") - [ -n "$expected" ] || error "checksums.txt has no hash for $archive" - actual=$(sha256 "$ARTIFACT_DIR/$archive") - [ "$actual" = "$expected" ] || error "$archive checksum was $actual, expected $expected" + local directory=$1 label=$2 archive=$3 expected actual + expected=$(awk -v archive="$archive" '$2 == archive || $2 == "*" archive { print $1; exit }' "$directory/checksums.txt") + [ -n "$expected" ] || error "$label checksums.txt has no hash for $archive" + actual=$(sha256 "$directory/$archive") + [ "$actual" = "$expected" ] || error "$label $archive checksum was $actual, expected $expected" } cli_archive() { @@ -55,24 +63,166 @@ cli_archive() { esac } +daemon_archive() { + case "$1" in + x86_64) echo ployzd_linux_amd64.tar.gz ;; + aarch64) echo ployzd_linux_arm64.tar.gz ;; + *) error "unsupported Machine architecture $1" ;; + esac +} + +SSH_ARGS=() +if [ -n "$SSH_KEY" ]; then + SSH_ARGS=(-i "$SSH_KEY") +fi + +ssh_host() { + # shellcheck disable=SC2029 # Every post-host argument is intentionally interpreted remotely. + ssh "${SSH_ARGS[@]}" "$@" +} + +scp_host() { + scp "${SSH_ARGS[@]}" "$@" +} + +json_field() { + python3 -c 'import json, sys +value = json.load(sys.stdin) +for part in sys.argv[1].split("."): + value = value[part] +print(value)' "$1" +} + +machine_state_signature() { + ssh_host "$first" sudo cat /var/lib/ployz/machine.json | python3 -c 'import hashlib, json, sys +value = json.load(sys.stdin) +def stable(item): + if isinstance(item, dict): + return {key: stable(child) for key, child in item.items() if key not in {"runtime", "selected_endpoints"}} + if isinstance(item, list): + return [stable(child) for child in item] + return item +encoded = json.dumps(stable(value), sort_keys=True, separators=(",", ":")).encode() +print(hashlib.sha256(encoded).hexdigest())' +} + +read_receipt() { + ssh_host "$first" 'sudo cat /var/lib/ployz/upgrade-attempt.json 2>/dev/null || true' +} + +wait_for_new_receipt() { + local previous=$1 receipt attempt_id i + for ((i = 0; i < 150; i++)); do + receipt=$(read_receipt) + attempt_id=$(printf '%s' "$receipt" | json_field attempt.attempt_id 2>/dev/null || true) + if [ -n "$attempt_id" ] && [ "$attempt_id" != "$previous" ]; then + printf '%s\n' "$receipt" + return + fi + sleep 0.2 + done + error "Machine did not record a new upgrade attempt within 30 seconds" +} + +wait_for_outcome() { + local attempt_id=$1 expected=$2 receipt observed_id outcome i + for ((i = 0; i < 900; i++)); do + receipt=$(read_receipt) + observed_id=$(printf '%s' "$receipt" | json_field attempt.attempt_id 2>/dev/null || true) + outcome=$(printf '%s' "$receipt" | json_field attempt.outcome 2>/dev/null || true) + if [ "$observed_id" = "$attempt_id" ] && [ "$outcome" = "$expected" ]; then + printf '%s\n' "$receipt" + return + fi + case "$outcome" in + failed|interrupted|succeeded) + [ "$observed_id" != "$attempt_id" ] || error "upgrade $attempt_id ended as $outcome, expected $expected" + ;; + esac + sleep 1 + done + error "upgrade $attempt_id did not become $expected within 15 minutes" +} + +require_field() { + local json=$1 field=$2 expected=$3 actual + actual=$(printf '%s' "$json" | json_field "$field") + [ "$actual" = "$expected" ] || error "$field was $actual, expected $expected" +} + +wait_for_application() { + local value i + for ((i = 0; i < 120; i++)); do + value=$(ssh_host "$first" "curl -fsS --max-time 2 '$APP_URL'" 2>/dev/null || true) + if [ "$value" = "$APP_VALUE" ]; then + return + fi + sleep 1 + done + error "qualification application did not serve its persistent value" +} + +start_traffic() { + ssh_host "$first" "sudo rm -f '$TRAFFIC_STOP'" + # shellcheck disable=SC2016 # The loop and its variables belong to the remote shell. + ssh_host "$first" 'while [ ! -e /var/lib/ployz/qualification/traffic.stop ]; do value=$(curl -fsS --max-time 2 http://127.0.0.1:18082/identity 2>&1 || true); if [ "$value" = qualify-persistent-data ]; then echo ok; else printf "failure: %s\n" "$value"; fi; sleep 0.2; done' >"$work/traffic.log" 2>&1 & + TRAFFIC_PID=$! +} + +stop_traffic() { + if [ -z "$TRAFFIC_PID" ]; then + return 0 + fi + ssh_host "$first" "sudo touch '$TRAFFIC_STOP'" >/dev/null 2>&1 || true + wait "$TRAFFIC_PID" || true + TRAFFIC_PID= +} + +cleanup() { + stop_traffic + [ -z "${work:-}" ] || rm -rf "$work" + [ -z "$CONFIG_DIR" ] || rm -rf "$CONFIG_DIR" +} + +stage_remote_release() { + local label=$1 directory=$2 archive=$3 upload + upload=/tmp/ployz-qualify-$$-$label + ssh_host "$first" "rm -rf '$upload' && mkdir -p '$upload'" + scp_host "$directory/$archive" "$directory/checksums.txt" "$first:$upload/" + ssh_host "$first" "sudo rm -rf '$REMOTE_RELEASE_ROOT/$label' && sudo install -d -m 0755 '$REMOTE_RELEASE_ROOT/$label' && sudo install -m 0644 '$upload/$archive' '$upload/checksums.txt' '$REMOTE_RELEASE_ROOT/$label/' && rm -rf '$upload'" +} + +select_remote_release() { + ssh_host "$first" "sudo ln -sfn '$REMOTE_RELEASE_ROOT/$1' '$REMOTE_RELEASE_ROOT/current'" +} + +assert_remote_hash() { + local path=$1 expected=$2 + ssh_host "$first" "test \"\$(sudo sha256sum '$path' | awk '{print \$1}')\" = '$expected'" +} + [ -n "$HOSTS" ] || error "set PLOYZ_QUALIFY_HOSTS to one or more user@host targets" -[ -n "$ARTIFACT_DIR" ] || error "set PLOYZ_ARTIFACT_DIR to the draft archive directory" +[ -n "$ARTIFACT_DIR" ] || error "set PLOYZ_ARTIFACT_DIR to the source draft archive directory" +[ -n "$UPGRADE_ARTIFACT_DIR" ] || error "set PLOYZ_UPGRADE_ARTIFACT_DIR to the target draft archive directory" [ -f "$COMPOSE_DIR/compose.yaml" ] || error "missing $COMPOSE_DIR/compose.yaml" +command -v python3 >/dev/null 2>&1 || error "python3 is required to inspect qualification evidence" read -r -a HOST_LIST <<<"$HOSTS" [ "${#HOST_LIST[@]}" -ge 1 ] || error "PLOYZ_QUALIFY_HOSTS is empty" -need_archive ployz_linux_amd64.tar.gz -need_archive ployz_linux_arm64.tar.gz -need_archive ployzd_linux_amd64.tar.gz -need_archive ployzd_linux_arm64.tar.gz -need_archive "$(cli_archive)" -need_archive checksums.txt +for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do + need_file "$ARTIFACT_DIR" PLOYZ_ARTIFACT_DIR "$archive" +done +need_file "$ARTIFACT_DIR" PLOYZ_ARTIFACT_DIR "$(cli_archive)" +need_file "$ARTIFACT_DIR" PLOYZ_ARTIFACT_DIR checksums.txt +need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR "$(cli_archive)" +need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR checksums.txt if [ "$DRY_RUN" != 0 ]; then echo "qualify dry-run" echo "hosts: ${HOST_LIST[*]}" - echo "artifacts: $ARTIFACT_DIR" + echo "source artifacts: $ARTIFACT_DIR" + echo "target artifacts: $UPGRADE_ARTIFACT_DIR" echo "compose: $COMPOSE_DIR/compose.yaml" echo "ssh-key: ${SSH_KEY:-cli-default}" if [ "$RESET" != 0 ]; then @@ -80,24 +230,31 @@ if [ "$DRY_RUN" != 0 ]; then else echo "reset: no (initialized hosts fail without PLOYZ_QUALIFY_RESET=1)" fi - echo "steps: verify the artifact CLI, run normal machine init/add from the local release, deploy named volume qualify-data, volume ls" + echo "steps: normal ZFS machine init/add from the source release; persistent application traffic; target upgrade with client disconnect and reconnect; corrupt preflight; failed activation; explicit previous-binary repair" exit 0 fi work=$(mktemp -d) -trap 'rm -rf "$work" "$CONFIG_DIR"' EXIT -verify_archive "$(cli_archive)" -tar -xzf "$ARTIFACT_DIR/$(cli_archive)" -C "$work" -PLOYZ=$work/ployz -[ -x "$PLOYZ" ] || error "CLI archive did not contain ployz" -version=$("$PLOYZ" version) || error "could not read ployz version from the artifact" +trap cleanup EXIT +mkdir -p "$work/source-cli" "$work/target-cli" +verify_archive "$ARTIFACT_DIR" source "$(cli_archive)" +verify_archive "$UPGRADE_ARTIFACT_DIR" target "$(cli_archive)" +tar -xzf "$ARTIFACT_DIR/$(cli_archive)" -C "$work/source-cli" +tar -xzf "$UPGRADE_ARTIFACT_DIR/$(cli_archive)" -C "$work/target-cli" +PLOYZ=$work/source-cli/ployz +TARGET_PLOYZ=$work/target-cli/ployz +[ -x "$PLOYZ" ] || error "source CLI archive did not contain ployz" +[ -x "$TARGET_PLOYZ" ] || error "target CLI archive did not contain ployz" +source_version=$("$PLOYZ" version) || error "could not read source CLI version" +target_version=$("$TARGET_PLOYZ" version) || error "could not read target CLI version" +[ "$source_version" != "$target_version" ] || error "source and target versions must differ" CONFIG_DIR=$(mktemp -d) export PLOYZ_CONFIG=$CONFIG_DIR/config.yaml export PLOYZ_RELEASE_DIR=$ARTIFACT_DIR first=${HOST_LIST[0]} -echo "machine init $first" -init_cmd=("$PLOYZ" machine init --version "$version" --context "$CONTEXT") +echo "machine init $first at $source_version" +init_cmd=("$PLOYZ" machine init --version "$source_version" --storage zfs --name qualify-1 --context "$CONTEXT") if [ "$RESET" != 0 ]; then init_cmd+=(--yes) fi @@ -109,7 +266,7 @@ fi i=1 while [ "$i" -lt "${#HOST_LIST[@]}" ]; do echo "machine add ${HOST_LIST[$i]}" - add_cmd=("$PLOYZ" machine add --yes --version "$version" --context "$CONTEXT") + add_cmd=("$PLOYZ" machine add --yes --version "$source_version" --storage zfs --name "qualify-$((i + 1))" --context "$CONTEXT") if [ -n "$SSH_KEY" ]; then add_cmd+=(--ssh-key "$SSH_KEY") fi @@ -117,11 +274,119 @@ while [ "$i" -lt "${#HOST_LIST[@]}" ]; do i=$((i + 1)) done -echo "deploy named volume" -"$PLOYZ" deploy --yes --context "$CONTEXT" -f "$COMPOSE_DIR/compose.yaml" +machine_arch=$(ssh_host "$first" uname -m) +machine_archive=$(daemon_archive "$machine_arch") +need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR "$machine_archive" +verify_archive "$UPGRADE_ARTIFACT_DIR" target "$machine_archive" + +mkdir -p "$work/releases/target" "$work/releases/corrupt" "$work/releases/failure" "$work/target-payload" +cp "$UPGRADE_ARTIFACT_DIR/$machine_archive" "$UPGRADE_ARTIFACT_DIR/checksums.txt" "$work/releases/target/" +cp "$UPGRADE_ARTIFACT_DIR/$machine_archive" "$UPGRADE_ARTIFACT_DIR/checksums.txt" "$work/releases/corrupt/" +printf 'corrupt\n' >>"$work/releases/corrupt/$machine_archive" +tar -xzf "$UPGRADE_ARTIFACT_DIR/$machine_archive" -C "$work/target-payload" +[ -x "$work/target-payload/ployzd" ] || error "target daemon archive did not contain ployzd" +[ -f "$work/target-payload/ployz-uninstall" ] || error "target daemon archive did not contain ployz-uninstall" +target_daemon_hash=$(sha256 "$work/target-payload/ployzd") +cat >"$work/target-payload/ployzd" <&2 +exit 42 +EOF +chmod 0755 "$work/target-payload/ployzd" +tar -czf "$work/releases/failure/$machine_archive" -C "$work/target-payload" ployzd ployz-uninstall +failure_daemon_hash=$(sha256 "$work/target-payload/ployzd") +printf '%s %s\n' "$(sha256 "$work/releases/failure/$machine_archive")" "$machine_archive" >"$work/releases/failure/checksums.txt" + +for release in target corrupt failure; do + stage_remote_release "$release" "$work/releases/$release" "$machine_archive" +done +select_remote_release target +ssh_host "$first" "sudo install -d -m 0755 /etc/systemd/system/ployz.service.d && printf '[Service]\nEnvironment=PLOYZ_UPGRADE_RELEASE_DIR=$REMOTE_RELEASE_ROOT/current\n' | sudo tee /etc/systemd/system/ployz.service.d/qualification.conf >/dev/null && sudo systemctl daemon-reload && sudo systemctl restart ployz.service" +echo "deploy persistent ZFS-backed application" +"$PLOYZ" deploy --yes --context "$CONTEXT" -f "$COMPOSE_DIR/compose.yaml" volumes=$("$PLOYZ" volume ls --context "$CONTEXT") printf '%s\n' "$volumes" printf '%s\n' "$volumes" | grep -Eq 'qualify-data' || error "named volume qualify-data did not appear in volume ls" +wait_for_application +state_before=$(machine_state_signature) +start_traffic + +echo "upgrade qualify-1 from $source_version to $target_version, then disconnect the requesting client" +"$PLOYZ" machine upgrade "$target_version" --machine qualify-1 --context "$CONTEXT" >"$work/success-client.log" 2>&1 & +client_pid=$! +success_receipt=$(wait_for_new_receipt "") +success_id=$(printf '%s' "$success_receipt" | json_field attempt.attempt_id) +kill "$client_pid" >/dev/null 2>&1 || true +wait "$client_pid" >/dev/null 2>&1 || true +success_receipt=$(wait_for_outcome "$success_id" succeeded) +require_field "$success_receipt" attempt.version "$target_version" +success_inspection=$("$PLOYZ" machine upgrade inspect qualify-1 --attempt "$success_id" --output json --context "$CONTEXT") +require_field "$success_inspection" outcome succeeded +require_field "$success_inspection" attempt_id "$success_id" +require_field "$success_inspection" version "$target_version" +assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" +wait_for_application +[ "$(machine_state_signature)" = "$state_before" ] || error "Machine identity, pairing, or configuration changed during upgrade" -echo "qualify-release passed on ${HOST_LIST[*]}" +echo "reject a checksum-corrupt target before activation" +select_remote_release corrupt +if "$PLOYZ" machine upgrade "$target_version" --machine qualify-1 --context "$CONTEXT" >"$work/corrupt-client.log" 2>&1; then + error "checksum-corrupt upgrade unexpectedly succeeded" +fi +grep -qi checksum "$work/corrupt-client.log" || error "checksum-corrupt upgrade did not report checksum evidence" +corrupt_receipt=$(wait_for_new_receipt "$success_id") +corrupt_id=$(printf '%s' "$corrupt_receipt" | json_field attempt.attempt_id) +corrupt_receipt=$(wait_for_outcome "$corrupt_id" failed) +require_field "$corrupt_receipt" attempt.stage verifying +assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" +ssh_host "$first" 'test ! -e /var/lib/ployz/.upgrade-active' +wait_for_application + +echo "activate an intentionally failing daemon and preserve local diagnostic evidence" +select_remote_release failure +"$PLOYZ" machine upgrade "$target_version" --machine qualify-1 --context "$CONTEXT" >"$work/failure-client.log" 2>&1 & +client_pid=$! +failure_receipt=$(wait_for_new_receipt "$corrupt_id") +failure_id=$(printf '%s' "$failure_receipt" | json_field attempt.attempt_id) +kill "$client_pid" >/dev/null 2>&1 || true +wait "$client_pid" >/dev/null 2>&1 || true +failure_receipt=$(wait_for_outcome "$failure_id" failed) +failure_stage=$(printf '%s' "$failure_receipt" | json_field attempt.stage) +case "$failure_stage" in + restarting|readiness) ;; + *) error "failed activation was reported at $failure_stage" ;; +esac +ssh_host "$first" "sudo journalctl -u 'ployz-upgrade-$failure_id.service' --no-pager -n 200 || true" +sleep 3 +assert_remote_hash /usr/local/bin/ployzd "$failure_daemon_hash" +assert_remote_hash /usr/local/bin/ployzd.previous "$target_daemon_hash" +ssh_host "$first" 'test ! -e /var/lib/ployz/.upgrade-active && ! sudo systemctl is-active --quiet ployz.service' +wait_for_application + +echo "repair explicitly from the one retained previous binary" +ssh_host "$first" 'sudo systemctl stop ployz.service || true; sudo cp --preserve=mode,ownership /usr/local/bin/ployzd.previous /usr/local/bin/.ployzd-repair; sudo sync /usr/local/bin/.ployzd-repair; sudo mv /usr/local/bin/.ployzd-repair /usr/local/bin/ployzd; sudo sync /usr/local/bin; sudo systemctl start ployz.service' +failure_inspection=$("$PLOYZ" machine upgrade inspect qualify-1 --attempt "$failure_id" --output json --context "$CONTEXT") +require_field "$failure_inspection" outcome failed +require_field "$failure_inspection" attempt_id "$failure_id" +require_field "$failure_inspection" stage "$failure_stage" +assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" +wait_for_application +[ "$(machine_state_signature)" = "$state_before" ] || error "Machine identity, pairing, or configuration changed after explicit repair" + +stop_traffic +if grep -q '^failure:' "$work/traffic.log"; then + cat "$work/traffic.log" >&2 + error "application traffic failed during daemon upgrade or repair" +fi +traffic_successes=$(grep -c '^ok$' "$work/traffic.log" || true) +[ "$traffic_successes" -ge 5 ] || error "continuous traffic collected only $traffic_successes successful probes" +printf 'continuous application traffic: %s successful probes, 0 failures\n' "$traffic_successes" + +volumes=$("$PLOYZ" volume ls --context "$CONTEXT") +printf '%s\n' "$volumes" | grep -Eq 'qualify-data' || error "named volume disappeared after upgrade" +echo "qualify-release passed on ${HOST_LIST[*]}: $source_version -> $target_version" diff --git a/scripts/qualify-release/compose.yaml b/scripts/qualify-release/compose.yaml index 2ad6c43afc..b243238c00 100644 --- a/scripts/qualify-release/compose.yaml +++ b/scripts/qualify-release/compose.yaml @@ -1,8 +1,13 @@ services: data: image: alpine:3.23.3 - command: ["sleep", "infinity"] + command: + - sh + - -c + - test -s /data/identity || printf 'qualify-persistent-data\n' > /data/identity; exec httpd -f -p 8080 -h /data volumes: - qualify-data:/data + x-machines: [qualify-1] + x-ports: [18082:8080/tcp@host] volumes: qualify-data: diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index bc1d9dcdc8..253ca38c16 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -16,6 +16,8 @@ if grep -qi vultr "$ROOT/scripts/qualify-release.sh" "$ROOT/docs/RELEASE.md"; th fi grep -Fq 'qualify-data' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no named volume" +grep -Fq '18082:8080/tcp@host' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no traffic endpoint" +grep -Fq '/data/identity' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture does not serve persistent data" if "$ROOT/scripts/qualify-release.sh" >/dev/null 2>&1; then fail "qualify-release accepted empty hosts" @@ -23,10 +25,17 @@ fi TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT -mkdir -p "$TMP/bin" +SOURCE=$TMP/source +TARGET=$TMP/target +mkdir -p "$TMP/bin" "$SOURCE" "$TARGET" export LOG=$TMP/calls.log -cat > "$TMP/ployz" <<'CLI' +export SSH_LOG=$TMP/ssh.log +export QUALIFY_STATE=$TMP/current-release +printf 'target\n' >"$QUALIFY_STATE" + +cat >"$SOURCE/ployz" <<'CLI' #!/bin/sh +set -eu case "$1" in version) printf '1.2.3\n' ;; deploy|volume) @@ -46,87 +55,220 @@ case "$1" in esac shift done - printf '%s context=%s file=%s yes=%s\n' "$action" "$context" "$file" "$yes" >> "$LOG" + printf '%s context=%s file=%s yes=%s\n' "$action" "$context" "$file" "$yes" >>"$LOG" [ "$action" != volume ] || printf 'qualify-data\n' ;; machine) action=$2 - case "$action" in init|add) ;; *) exit 1 ;; esac shift 2 - reset=no key= target= context= no_install=no version= - while [ "$#" -gt 0 ]; do - case "$1" in - --yes) reset=yes ;; - --ssh-key) key=$2; shift ;; - --context) context=$2; shift ;; - --version) version=$2; shift ;; - --no-install) no_install=yes ;; - root@*) target=$1 ;; - *) echo "unexpected machine argument: $1" >&2; exit 1 ;; - esac - shift - done - printf '%s target=%s reset=%s key=%s context=%s no_install=%s version=%s release=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" "$version" "${PLOYZ_RELEASE_DIR:-}" >> "$LOG" + case "$action" in + init|add) + reset=no key= target= context= no_install=no version= storage= name= + while [ "$#" -gt 0 ]; do + case "$1" in + --yes) reset=yes ;; + --ssh-key) key=$2; shift ;; + --context) context=$2; shift ;; + --version) version=$2; shift ;; + --storage) storage=$2; shift ;; + --name) name=$2; shift ;; + --no-install) no_install=yes ;; + root@*) target=$1 ;; + *) echo "unexpected machine argument: $1" >&2; exit 1 ;; + esac + shift + done + printf '%s target=%s reset=%s key=%s context=%s no_install=%s version=%s storage=%s name=%s release=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" "$version" "$storage" "$name" "${PLOYZ_RELEASE_DIR:-}" >>"$LOG" + ;; + upgrade) + if [ "${1:-}" = inspect ]; then + shift + machine=$1 + shift + attempt= context= output= + while [ "$#" -gt 0 ]; do + case "$1" in + --attempt) attempt=$2; shift ;; + --context) context=$2; shift ;; + --output) output=$2; shift ;; + *) echo "unexpected upgrade inspect argument: $1" >&2; exit 1 ;; + esac + shift + done + current=$(cat "$QUALIFY_STATE") + printf 'upgrade-inspect machine=%s attempt=%s context=%s output=%s current=%s\n' "$machine" "$attempt" "$context" "$output" "$current" >>"$LOG" + case "$current" in + target) printf '{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","version":"1.2.4"}\n' ;; + failure) printf '{"outcome":"failed","attempt_id":"cccccccccccccccccccccccccccccccc","target":"1.2.4","stage":"restarting","error":"restart daemon failed"}\n' ;; + *) exit 1 ;; + esac + exit + fi + release=$1 + shift + machine= context= + while [ "$#" -gt 0 ]; do + case "$1" in + --machine) machine=$2; shift ;; + --context) context=$2; shift ;; + *) echo "unexpected upgrade argument: $1" >&2; exit 1 ;; + esac + shift + done + current=$(cat "$QUALIFY_STATE") + printf 'upgrade release=%s machine=%s context=%s current=%s\n' "$release" "$machine" "$context" "$current" >>"$LOG" + case "$current" in + corrupt) echo 'daemon archive checksum mismatch' >&2; exit 1 ;; + target|failure) sleep 30 ;; + *) exit 1 ;; + esac + ;; + *) echo "unexpected machine action: $action" >&2; exit 1 ;; + esac ;; *) echo "unexpected command: $*" >&2; exit 1 ;; esac CLI -chmod 0755 "$TMP/ployz" -for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployz_macos_amd64.tar.gz ployz_macos_arm64.tar.gz ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do - tar -czf "$TMP/$archive" -C "$TMP" ployz +chmod 0755 "$SOURCE/ployz" + +cat >"$TARGET/ployz" <<'CLI' +#!/bin/sh +[ "${1:-}" = version ] && printf '1.2.4\n' +CLI +chmod 0755 "$TARGET/ployz" + +cat >"$SOURCE/ployzd" <<'DAEMON' +#!/bin/sh +[ "${1:-}" = version ] && printf '1.2.3\n' +DAEMON +cat >"$TARGET/ployzd" <<'DAEMON' +#!/bin/sh +[ "${1:-}" = version ] && printf '1.2.4\n' +DAEMON +printf '#!/bin/sh\nexit 0\n' >"$SOURCE/ployz-uninstall" +cp "$SOURCE/ployz-uninstall" "$TARGET/ployz-uninstall" +chmod 0755 "$SOURCE/ployzd" "$TARGET/ployzd" "$SOURCE/ployz-uninstall" "$TARGET/ployz-uninstall" + +for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployz_macos_amd64.tar.gz ployz_macos_arm64.tar.gz; do + tar -czf "$SOURCE/$archive" -C "$SOURCE" ployz + tar -czf "$TARGET/$archive" -C "$TARGET" ployz done -cp "$TMP/ployz" "$TMP/ployzd" -cp "$TMP/ployz" "$TMP/ployz-uninstall" for archive in ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do - tar -czf "$TMP/$archive" -C "$TMP" ployzd ployz-uninstall + tar -czf "$SOURCE/$archive" -C "$SOURCE" ployzd ployz-uninstall + tar -czf "$TARGET/$archive" -C "$TARGET" ployzd ployz-uninstall done ( - cd "$TMP" - sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 > checksums.txt + cd "$SOURCE" + sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 >checksums.txt +) +( + cd "$TARGET" + sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 >checksums.txt ) +cat >"$TMP/bin/ssh" <<'SSH' +#!/bin/sh +set -eu +key= +while [ "$#" -gt 0 ]; do + case "$1" in + -i) key=$2; shift 2 ;; + *) break ;; + esac +done +[ "$key" = /tmp/qualify-key ] || { echo 'remote command lost SSH identity' >&2; exit 1; } +host=$1 +shift +command=$* +printf 'ssh host=%s command=%s\n' "$host" "$command" >>"$SSH_LOG" +case "$command" in + *'uname -m'*) printf 'x86_64\n' ;; + *'ln -sfn'*'/target'*'/current'*) printf 'target\n' >"$QUALIFY_STATE" ;; + *'ln -sfn'*'/corrupt'*'/current'*) printf 'corrupt\n' >"$QUALIFY_STATE" ;; + *'ln -sfn'*'/failure'*'/current'*) printf 'failure\n' >"$QUALIFY_STATE" ;; + *'upgrade-attempt.json'*) + case "$(cat "$QUALIFY_STATE")" in + target) printf '{"requested":"1.2.4","attempt":{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","version":"1.2.4"}}\n' ;; + corrupt) printf '{"requested":"1.2.4","attempt":{"outcome":"failed","attempt_id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","target":"1.2.4","stage":"verifying","error":"checksum mismatch"}}\n' ;; + failure) printf '{"requested":"1.2.4","attempt":{"outcome":"failed","attempt_id":"cccccccccccccccccccccccccccccccc","target":"1.2.4","stage":"restarting","error":"restart daemon failed"}}\n' ;; + esac + ;; + *'sudo cat /var/lib/ployz/machine.json'*) + printf '{"body":{"phase":"participating","machine":{"id":"11111111111111111111111111111111","runtime":{"daemon_version":"1.2.3"}}},"wireguard_private_key":"secret","cloud_pairing":null,"selected_endpoints":{}}\n' + ;; + *'while [ ! -e /var/lib/ployz/qualification/traffic.stop'*) + printf 'ok\nok\nok\nok\nok\nok\n' + ;; + *'curl -fsS --max-time 2'*) printf 'qualify-persistent-data\n' ;; + *'journalctl -u'*) printf 'intentional qualification activation failure\n' ;; +esac +SSH +chmod 0755 "$TMP/bin/ssh" + +cat >"$TMP/bin/scp" <<'SCP' +#!/bin/sh +set -eu +key= +if [ "${1:-}" = -i ]; then + key=$2 + shift 2 +fi +[ "$key" = /tmp/qualify-key ] || { echo 'remote copy lost SSH identity' >&2; exit 1; } +printf 'scp %s\n' "$*" >>"$SSH_LOG" +SCP +chmod 0755 "$TMP/bin/scp" + for reset in 0 1; do - : > "$LOG" - PATH="$TMP/bin:$PATH" PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$TMP" \ - PLOYZ_QUALIFY_SSH_KEY=/tmp/qualify-key PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ + : >"$LOG" + PATH="$TMP/bin:$PATH" PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$SOURCE" \ + PLOYZ_UPGRADE_ARTIFACT_DIR="$TARGET" PLOYZ_QUALIFY_SSH_KEY=/tmp/qualify-key \ + PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ "$ROOT/scripts/qualify-release.sh" >/dev/null expected_reset=no [ "$reset" = 0 ] || expected_reset=yes - grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 release=$TMP" "$LOG" || fail "init lost its target, release, version, reset policy, or SSH identity" - grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 release=$TMP" "$LOG" || fail "add lost its target, release, version, or SSH identity" - grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "named-volume fixture was not deployed" - grep -Fxq 'volume context=qualify file= yes=no' "$LOG" || fail "volume ls did not query the qualification context" + grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-1 release=$SOURCE" "$LOG" || fail "init lost its target, release, storage, name, version, reset policy, or SSH identity" + grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-2 release=$SOURCE" "$LOG" || fail "add lost its target, release, storage, name, version, or SSH identity" + grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "persistent-volume fixture was not deployed" + grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=target' "$LOG" || fail "target upgrade was not requested" + grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=corrupt' "$LOG" || fail "corrupt preflight was not requested" + grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=failure' "$LOG" || fail "failed activation was not requested" + grep -Fxq 'upgrade-inspect machine=qualify-1 attempt=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa context=qualify output=json current=target' "$LOG" || fail "successful attempt was not inspected after reconnect" + grep -Fxq 'upgrade-inspect machine=qualify-1 attempt=cccccccccccccccccccccccccccccccc context=qualify output=json current=failure' "$LOG" || fail "failed attempt was not inspected after repair" done +grep -Fq 'ssh host=root@192.0.2.10 command=' "$SSH_LOG" || fail "SSH evidence path was not exercised" +grep -Fq 'scp ' "$SSH_LOG" || fail "target release was not staged with SCP" -cat > "$TMP/bin/cargo" <<'CARGO' +cat >"$TMP/bin/cargo" <<'CARGO' #!/bin/sh case " $* " in *' --no-run '*' --ignored '*|*' --ignored '*' --no-run '*) echo 'informing tests must execute' >&2; exit 1 ;; *' --no-run '*) [ ! -s "$LOG" ] || exit 1 - printf 'compile\n' >> "$LOG" + printf 'compile\n' >>"$LOG" exit 0 ;; *' --no-fail-fast '*) case " $* " in - *' --ignored '*|*' --include-ignored '*) printf '%s\n' "$*" >> "$LOG" ;; + *' --ignored '*|*' --include-ignored '*) printf '%s\n' "$*" >>"$LOG" ;; *) echo 'informing tests must be selected' >&2; exit 1 ;; esac ;; *) echo 'test invocation must collect all binary failures' >&2; exit 1 ;; esac CARGO chmod 0755 "$TMP/bin/cargo" -: > "$LOG" +: >"$LOG" PATH="$TMP/bin:$PATH" "$ROOT/scripts/run-layer3-tests.sh" grep -Eq -- '--(include-)?ignored' "$LOG" || fail "layer3 runner did not execute tests" output=$( - PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$TMP" \ - PLOYZ_QUALIFY_DRY_RUN=1 PLOYZ_QUALIFY_RESET=0 "$ROOT/scripts/qualify-release.sh" + PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$SOURCE" \ + PLOYZ_UPGRADE_ARTIFACT_DIR="$TARGET" PLOYZ_QUALIFY_DRY_RUN=1 \ + PLOYZ_QUALIFY_RESET=0 "$ROOT/scripts/qualify-release.sh" ) printf '%s\n' "$output" | grep -Fq 'qualify dry-run' || fail "dry-run did not print the plan" -printf '%s\n' "$output" | grep -Fq 'named volume' || fail "dry-run omitted the named-volume step" +printf '%s\n' "$output" | grep -Fq 'normal ZFS machine init/add' || fail "dry-run omitted normal storage setup" +printf '%s\n' "$output" | grep -Fq 'client disconnect and reconnect' || fail "dry-run omitted connection loss" +printf '%s\n' "$output" | grep -Fq 'failed activation' || fail "dry-run omitted failure evidence" printf '%s\n' "$output" | grep -Fq 'reset: no' || fail "dry-run omitted the default no-reset policy" -printf '%s\n' "$output" | grep -Fq 'normal machine init/add' || fail "dry-run omitted normal setup path" "$ROOT/scripts/check-layer3-runner.sh" echo "qualify-release contracts passed" From 13812477565a26db6751432bd2be186be0e412b4 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 02:54:16 +0000 Subject: [PATCH 16/30] test(compose): use POSIX mkfifo fixture --- crates/ployz/src/compose/build_inputs.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/ployz/src/compose/build_inputs.rs b/crates/ployz/src/compose/build_inputs.rs index 5094fe5c34..51f777e8cc 100644 --- a/crates/ployz/src/compose/build_inputs.rs +++ b/crates/ployz/src/compose/build_inputs.rs @@ -344,12 +344,13 @@ mod tests { fs::write(source.join(".dockerignore"), "ignored\n").unwrap(); fs::write(source.join("included"), "original").unwrap(); let _socket = UnixListener::bind(source.join("ignored/socket")).unwrap(); - rustix::fs::mkfifoat( - rustix::fs::CWD, - source.join("ignored/fifo"), - rustix::fs::Mode::RUSR, - ) - .unwrap(); + assert!( + std::process::Command::new("mkfifo") + .arg(source.join("ignored/fifo")) + .status() + .unwrap() + .success() + ); fs::write(source.join("ignored/unreadable"), "private").unwrap(); fs::set_permissions( source.join("ignored/unreadable"), From 22c3b57da45605f0f37fa19bd45e37ba7cc297f3 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 03:01:51 +0000 Subject: [PATCH 17/30] fix(setup): refresh SSH groups after install --- crates/ployz/src/provisioning.rs | 23 ++++++++++++++++++++++- crates/ployz/tests/setup_cli.rs | 4 ++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/ployz/src/provisioning.rs b/crates/ployz/src/provisioning.rs index 7252556fce..3050d05b0d 100644 --- a/crates/ployz/src/provisioning.rs +++ b/crates/ployz/src/provisioning.rs @@ -338,6 +338,23 @@ impl Remote { Err(ProvisionError::CleanupFailed { status }) } } + + async fn close_control_master(&self) { + let Some(control_path) = self.control_path.as_deref() else { + return; + }; + let mut command = Command::new("ssh"); + command.args(crate::connect::ssh_control_args(Some(control_path))); + if let Some(port) = self.destination.port() { + command.arg("-p").arg(port.to_string()); + } + command.args(["-O", "exit", self.destination.target()]); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let _ = tokio::process::Command::from(command).status().await; + } } fn ssh_key(matches: &ArgMatches) -> PathBuf { @@ -477,7 +494,11 @@ pub async fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<( remote.install(&remote_daemon, &arguments, via_sudo).await } .await; - finish_remote(primary, remote.cleanup(&remote_directory).await) + let cleanup = remote.cleanup(&remote_directory).await; + // Host preparation may add the SSH user to the ployz group. A multiplexed + // session authenticated before installation retains its old group list. + remote.close_control_master().await; + finish_remote(primary, cleanup) } /// Install and start local `ployzd` through a verified temporary daemon. diff --git a/crates/ployz/tests/setup_cli.rs b/crates/ployz/tests/setup_cli.rs index 049581b064..0db5740831 100644 --- a/crates/ployz/tests/setup_cli.rs +++ b/crates/ployz/tests/setup_cli.rs @@ -192,6 +192,7 @@ case "$*" in *'chmod 700 '*' version') printf '%s\n' '{}' ;; *" 'install' "*) ;; *'rm -rf -- '*) ;; + *' -O exit '*) ;; *' true') exit 23 ;; *) echo "unexpected ssh invocation: $*" >&2; exit 24 ;; esac @@ -247,6 +248,9 @@ esac ); assert!(setup_log.contains("'--release-dir' '/tmp/ployz-bootstrap-")); assert!(setup_log.contains("rm -rf --"), "{setup_log}"); + let close = setup_log.find(" -O exit deploy@2001:db8::1").unwrap(); + let reconnect = setup_log.rfind(" true").unwrap(); + assert!(close < reconnect, "{setup_log}"); assert!(!setup_log.contains("base64"), "{setup_log}"); fs::write(&log, "").unwrap(); From bae633111ed29faad7b31c89dd397963d53d1e52 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 03:01:51 +0000 Subject: [PATCH 18/30] fix(setup): refresh SSH groups after install --- crates/ployz/src/provisioning.rs | 23 ++++++++++++++++++++++- crates/ployz/tests/setup_cli.rs | 4 ++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/ployz/src/provisioning.rs b/crates/ployz/src/provisioning.rs index 7252556fce..3050d05b0d 100644 --- a/crates/ployz/src/provisioning.rs +++ b/crates/ployz/src/provisioning.rs @@ -338,6 +338,23 @@ impl Remote { Err(ProvisionError::CleanupFailed { status }) } } + + async fn close_control_master(&self) { + let Some(control_path) = self.control_path.as_deref() else { + return; + }; + let mut command = Command::new("ssh"); + command.args(crate::connect::ssh_control_args(Some(control_path))); + if let Some(port) = self.destination.port() { + command.arg("-p").arg(port.to_string()); + } + command.args(["-O", "exit", self.destination.target()]); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let _ = tokio::process::Command::from(command).status().await; + } } fn ssh_key(matches: &ArgMatches) -> PathBuf { @@ -477,7 +494,11 @@ pub async fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<( remote.install(&remote_daemon, &arguments, via_sudo).await } .await; - finish_remote(primary, remote.cleanup(&remote_directory).await) + let cleanup = remote.cleanup(&remote_directory).await; + // Host preparation may add the SSH user to the ployz group. A multiplexed + // session authenticated before installation retains its old group list. + remote.close_control_master().await; + finish_remote(primary, cleanup) } /// Install and start local `ployzd` through a verified temporary daemon. diff --git a/crates/ployz/tests/setup_cli.rs b/crates/ployz/tests/setup_cli.rs index 049581b064..0db5740831 100644 --- a/crates/ployz/tests/setup_cli.rs +++ b/crates/ployz/tests/setup_cli.rs @@ -192,6 +192,7 @@ case "$*" in *'chmod 700 '*' version') printf '%s\n' '{}' ;; *" 'install' "*) ;; *'rm -rf -- '*) ;; + *' -O exit '*) ;; *' true') exit 23 ;; *) echo "unexpected ssh invocation: $*" >&2; exit 24 ;; esac @@ -247,6 +248,9 @@ esac ); assert!(setup_log.contains("'--release-dir' '/tmp/ployz-bootstrap-")); assert!(setup_log.contains("rm -rf --"), "{setup_log}"); + let close = setup_log.find(" -O exit deploy@2001:db8::1").unwrap(); + let reconnect = setup_log.rfind(" true").unwrap(); + assert!(close < reconnect, "{setup_log}"); assert!(!setup_log.contains("base64"), "{setup_log}"); fs::write(&log, "").unwrap(); From 840728a4185a1962b9162978edbb820fd3745d00 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 03:58:54 +0000 Subject: [PATCH 19/30] Fix provisioned volume metadata registration --- crates/ployzd/src/docker/volume.rs | 84 +++++++++++--- crates/ployzd/src/docker/volume/tests.rs | 135 +++++++++++++++++++++++ 2 files changed, 202 insertions(+), 17 deletions(-) diff --git a/crates/ployzd/src/docker/volume.rs b/crates/ployzd/src/docker/volume.rs index 7506b4dcfd..d35b8c18d7 100644 --- a/crates/ployzd/src/docker/volume.rs +++ b/crates/ployzd/src/docker/volume.rs @@ -65,22 +65,53 @@ impl ContainerRuntime { match self.inspect_volume(machine_id, name).await { Ok(volume) => verify_volume(source, &volume), Err(error) if volume_not_found(&error) => { - let request = source - .to_create_volume_request() - .expect("managed Docker Volume sources have creation requests"); - match self.create_volume(machine_id, request).await { - Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), - Ok(CreateVolumeReport::Unverified { id, error }) => { - Err(Error::VolumeCreatedButUnverified { - id, - error: Box::new(error), - }) - } - Err(error) if volume_conflict(&error) => { - verify_volume(source, &self.inspect_volume(machine_id, name).await?) - } - Err(error) => Err(error), - } + self.create_managed_volume(machine_id, source).await + } + Err(error) => Err(error), + } + } + + async fn create_managed_volume( + &self, + machine_id: &MachineId, + source: &VolumeSource, + ) -> Result<(), Error> { + let name = source + .docker_volume_name() + .expect("managed Docker Volume sources have names"); + let request = source + .to_create_volume_request() + .expect("managed Docker Volume sources have creation requests"); + match self.create_volume(machine_id, request).await { + Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), + Ok(CreateVolumeReport::Unverified { id, error }) => { + Err(Error::VolumeCreatedButUnverified { + id, + error: Box::new(error), + }) + } + Err(error) if volume_conflict(&error) => { + verify_volume(source, &self.inspect_volume(machine_id, name).await?) + } + Err(error) => Err(error), + } + } + + async fn finish_prepared_volume( + &self, + machine_id: &MachineId, + source: &VolumeSource, + ) -> Result<(), Error> { + let name = source + .docker_volume_name() + .expect("provisioned Volume has a name"); + match self.inspect_volume(machine_id, name).await { + Ok(volume) if prepared_volume_needs_registration(source, &volume) => { + self.create_managed_volume(machine_id, source).await + } + Ok(volume) => verify_volume(source, &volume), + Err(error) if volume_not_found(&error) => { + self.create_managed_volume(machine_id, source).await } Err(error) => Err(error), } @@ -126,7 +157,7 @@ impl ContainerRuntime { ) -> Result<(), Error> { for spec in specs { for volume in spec.volume_graph().mounted_provisioned_volumes() { - self.ensure_volume_source(machine_id, &volume.source) + self.finish_prepared_volume(machine_id, &volume.source) .await?; } } @@ -332,6 +363,25 @@ fn verify_volume(source: &VolumeSource, observed: &DockerVolume) -> Result<(), E Ok(()) } +fn prepared_volume_needs_registration(source: &VolumeSource, observed: &DockerVolume) -> bool { + let ployz_core::RawVolumeSource::Provisioned { + name, + maximum_bytes, + .. + } = source.kind() + else { + return false; + }; + observed.id.name == *name + && observed.options.is_empty() + && observed.labels.is_empty() + && matches!( + &observed.storage, + DockerVolumeStorageObservation::Provisioned { bound_bytes, .. } + if bound_bytes.get() == maximum_bytes.get() + ) +} + fn volume_not_found(error: &Error) -> bool { docker_status(error, 404) } diff --git a/crates/ployzd/src/docker/volume/tests.rs b/crates/ployzd/src/docker/volume/tests.rs index 285c06399a..64addd88f6 100644 --- a/crates/ployzd/src/docker/volume/tests.rs +++ b/crates/ployzd/src/docker/volume/tests.rs @@ -297,6 +297,141 @@ async fn missing_provisioned_volume_uses_ployz_driver_bound_and_labels() { ); } +#[tokio::test] +async fn preparation_registers_metadata_for_storage_created_by_plugin() { + let (runtime, fake) = fake_runtime().await; + let machine_id = MachineId::random(); + let source = provisioned_source("prepared", 2_147_483_648); + fake.volumes.lock().unwrap().insert( + "app_prepared".into(), + serde_json::json!({ + "Name":"app_prepared", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_prepared", + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, + "Options":null, + "Labels":null + }), + ); + let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( + vec![source.clone()], + ))]; + + assert!(matches!( + runtime.ensure_volume_source(&machine_id, &source).await, + Err(Error::VolumeShapeMismatch { .. }) + )); + assert!( + fake.requests + .lock() + .unwrap() + .iter() + .all(|(method, _)| method != Method::POST) + ); + + runtime + .ensure_provisioned_volumes(&machine_id, &specs) + .await + .unwrap(); + + let create = fake + .request_bodies + .lock() + .unwrap() + .iter() + .find(|(path, _)| path.ends_with("/volumes/create")) + .map(|(_, request)| request.clone()) + .expect("Docker metadata registration request"); + assert_eq!( + create, + serde_json::json!({ + "Name":"app_prepared", + "Driver":"ployz", + "DriverOpts":{"size":"2147483648b"}, + "Labels":{"backup":"daily","ployz.managed":"","ployz.project.name":"app"} + }) + ); + let observed = runtime + .inspect_volume(&machine_id, source.docker_volume_name().unwrap()) + .await + .unwrap(); + assert!(source.matches_managed_volume(&observed)); +} + +#[tokio::test] +async fn preparation_refuses_mismatched_or_partially_registered_volumes() { + let (runtime, fake) = fake_runtime().await; + let cases = [ + ( + "wrong-driver", + serde_json::json!({ + "Name":"app_wrong-driver", + "Driver":"local", + "Mountpoint":"/var/lib/docker/volumes/app_wrong-driver/_data", + "Options":null, + "Labels":null + }), + ), + ( + "wrong-bound", + serde_json::json!({ + "Name":"app_wrong-bound", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_wrong-bound", + "Status":{"bound_bytes":1073741824_u64,"used_bytes":0}, + "Options":null, + "Labels":null + }), + ), + ( + "partial-options", + serde_json::json!({ + "Name":"app_partial-options", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_partial-options", + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, + "Options":{"size":"2147483648b"}, + "Labels":null + }), + ), + ( + "partial-labels", + serde_json::json!({ + "Name":"app_partial-labels", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_partial-labels", + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, + "Options":null, + "Labels":{"ployz.managed":"","ployz.project.name":"app"} + }), + ), + ]; + + for (name, observed) in cases { + let source = provisioned_source(name, 2_147_483_648); + fake.volumes + .lock() + .unwrap() + .insert(format!("app_{name}"), observed); + let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( + vec![source], + ))]; + assert!(matches!( + runtime + .ensure_provisioned_volumes(&MachineId::random(), &specs) + .await, + Err(Error::VolumeShapeMismatch { .. }) + )); + } + assert!( + fake.requests + .lock() + .unwrap() + .iter() + .all(|(method, _)| method != Method::POST) + ); +} + #[tokio::test] async fn existing_managed_volume_allows_extra_labels() { let (runtime, fake) = fake_runtime().await; From 7f7f9f9312b5320ee377bba561d4bc003e31b824 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:07:23 +0000 Subject: [PATCH 20/30] fix(upgrade): own mutation admission and durable outcomes --- crates/ployz-core/src/rpc.rs | 88 +--- crates/ployz-core/src/rpc/upgrade.rs | 87 ++++ crates/ployz-core/tests/contract.rs | 16 +- crates/ployz/src/cluster.rs | 28 +- crates/ployz/src/handlers/machine/remove.rs | 2 +- crates/ployz/src/handlers/machine/upgrade.rs | 353 ++++++++++---- crates/ployzd/src/installer/admission.rs | 149 ------ crates/ployzd/src/installer/mod.rs | 55 +-- crates/ployzd/src/installer/release.rs | 14 +- crates/ployzd/src/installer/storage.rs | 9 +- crates/ployzd/src/installer/upgrade.rs | 322 +++++++------ crates/ployzd/src/lib.rs | 1 + crates/ployzd/src/machine/local_machine.rs | 112 ++--- .../src/machine/local_machine/container.rs | 189 +++++++- .../src/machine/local_machine/upgrade.rs | 64 +++ crates/ployzd/src/machine/mod.rs | 449 +---------------- .../ployzd/src/machine/register_tests/mod.rs | 2 + crates/ployzd/src/machine/store.rs | 451 ++++++++++++++++++ crates/ployzd/src/machine_api/local.rs | 185 ++----- crates/ployzd/src/machine_api/local_tests.rs | 10 +- crates/ployzd/src/main.rs | 2 +- crates/ployzd/src/mutation.rs | 205 ++++++++ crates/ployzd/src/volume_plugin.rs | 383 +-------------- crates/ployzd/src/volume_plugin/capacity.rs | 46 +- crates/ployzd/src/volume_plugin/storage.rs | 376 +++++++++++++++ .../volume_plugin/tests/first_pool_tests.rs | 5 +- scripts/qualify-release.sh | 21 +- scripts/qualify-release/compose.yaml | 6 +- scripts/test-qualify-release.sh | 10 +- 29 files changed, 2037 insertions(+), 1603 deletions(-) create mode 100644 crates/ployz-core/src/rpc/upgrade.rs delete mode 100644 crates/ployzd/src/installer/admission.rs create mode 100644 crates/ployzd/src/machine/local_machine/upgrade.rs create mode 100644 crates/ployzd/src/machine/store.rs create mode 100644 crates/ployzd/src/mutation.rs create mode 100644 crates/ployzd/src/volume_plugin/storage.rs diff --git a/crates/ployz-core/src/rpc.rs b/crates/ployz-core/src/rpc.rs index 6d69a87f51..2258e82bf0 100644 --- a/crates/ployz-core/src/rpc.rs +++ b/crates/ployz-core/src/rpc.rs @@ -6,6 +6,9 @@ use std::{ }; use ts_rs::TS; +mod upgrade; +pub use upgrade::*; + use ipnet::Ipv4Net; use prost::Message; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::DeserializeOwned}; @@ -457,18 +460,6 @@ pub struct UpdateMachineRequest { pub update: MachineUpdate, } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -pub struct RequestMachineUpgradeRequest { - pub attempt_id: crate::MachineUpgradeAttemptId, - pub release: crate::MachineRelease, -} - -#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub struct InspectMachineUpgradeRequest { - #[serde(default)] - pub attempt_id: Option, -} - #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct RemoveLocalMachineRequest { #[serde(default)] @@ -731,79 +722,6 @@ pub struct MachineUpdated { pub machine: Machine, } -crate::value::open_string_enum!(MachineUpgradeStage, Unknown { - Launching => "launching", - Preparing => "preparing", - Acquiring => "acquiring", - Verifying => "verifying", - Activating => "activating", - Restarting => "restarting", - Readiness => "readiness", -}); - -/// Durable local evidence for one bounded Machine upgrade attempt. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TS)] -#[serde(tag = "outcome", rename_all = "snake_case")] -pub enum MachineUpgradeAttempt { - Accepted { - attempt_id: crate::MachineUpgradeAttemptId, - target: crate::MachineVersion, - }, - Running { - attempt_id: crate::MachineUpgradeAttemptId, - target: crate::MachineVersion, - stage: MachineUpgradeStage, - }, - Succeeded { - attempt_id: crate::MachineUpgradeAttemptId, - /// The requested version observed running and locally ready. - version: crate::MachineVersion, - }, - Failed { - attempt_id: crate::MachineUpgradeAttemptId, - target: crate::MachineVersion, - stage: MachineUpgradeStage, - error: String, - }, - Interrupted { - attempt_id: crate::MachineUpgradeAttemptId, - target: crate::MachineVersion, - stage: MachineUpgradeStage, - }, -} - -impl MachineUpgradeAttempt { - #[must_use] - pub fn attempt_id(&self) -> crate::MachineUpgradeAttemptId { - match self { - Self::Accepted { attempt_id, .. } - | Self::Running { attempt_id, .. } - | Self::Succeeded { attempt_id, .. } - | Self::Failed { attempt_id, .. } - | Self::Interrupted { attempt_id, .. } => *attempt_id, - } - } - - #[must_use] - pub fn target(&self) -> &crate::MachineVersion { - match self { - Self::Accepted { target, .. } - | Self::Running { target, .. } - | Self::Failed { target, .. } - | Self::Interrupted { target, .. } => target, - Self::Succeeded { version, .. } => version, - } - } - - #[must_use] - pub fn is_terminal(&self) -> bool { - matches!( - self, - Self::Succeeded { .. } | Self::Failed { .. } | Self::Interrupted { .. } - ) - } -} - #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize, TS)] pub struct LocalMachineRemoved { #[serde(default)] diff --git a/crates/ployz-core/src/rpc/upgrade.rs b/crates/ployz-core/src/rpc/upgrade.rs new file mode 100644 index 0000000000..ba34adeafc --- /dev/null +++ b/crates/ployz-core/src/rpc/upgrade.rs @@ -0,0 +1,87 @@ +//! Typed requests and durable outcomes for one Machine upgrade. + +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +/// Request one exact or channel-selected Machine release under a stable retry identity. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RequestMachineUpgradeRequest { + /// Identity reused by every retry of the same request. + pub attempt_id: crate::MachineUpgradeAttemptId, + /// Exact version or supported release channel to resolve once. + pub release: crate::MachineRelease, +} + +/// Read the latest attempt, optionally requiring one exact retry identity. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct InspectMachineUpgradeRequest { + /// Required attempt identity, or the latest local attempt when absent. + #[serde(default)] + pub attempt_id: Option, +} + +crate::value::open_string_enum!(MachineUpgradeStage, Unknown { + Launching => "launching", + Preparing => "preparing", + Acquiring => "acquiring", + Verifying => "verifying", + Activating => "activating", + Restarting => "restarting", + Readiness => "readiness", +}); + +/// Durable local evidence for one bounded Machine upgrade attempt. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TS)] +pub struct MachineUpgradeAttempt { + /// Stable identity assigned before the request is dispatched. + pub attempt_id: crate::MachineUpgradeAttemptId, + /// Exact version resolved before any host mutation. + pub target: crate::MachineVersion, + /// State-specific evidence for the attempt. + #[serde(flatten)] + #[ts(flatten)] + pub outcome: MachineUpgradeOutcome, +} + +/// State-specific evidence for a Machine upgrade attempt. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TS)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum MachineUpgradeOutcome { + /// The durable receipt and active marker exist and worker launch was accepted. + Accepted, + /// The independent worker reached this installation stage. + Running { + /// Latest stage durably recorded by the worker. + stage: MachineUpgradeStage, + }, + /// The target daemon was observed running and ready through its local API. + Succeeded { + /// Exact version reported by the ready, activated daemon. + version: crate::MachineVersion, + }, + /// The worker recorded a terminal installation failure. + Failed { + /// First installation stage that did not complete. + stage: MachineUpgradeStage, + /// Operator-facing failure evidence. + error: String, + }, + /// A positively stopped worker left no terminal result. + Interrupted { + /// Last durable stage before the worker stopped. + stage: MachineUpgradeStage, + }, +} + +impl MachineUpgradeAttempt { + /// Whether the attempt has durable terminal evidence. + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!( + self.outcome, + MachineUpgradeOutcome::Succeeded { .. } + | MachineUpgradeOutcome::Failed { .. } + | MachineUpgradeOutcome::Interrupted { .. } + ) + } +} diff --git a/crates/ployz-core/tests/contract.rs b/crates/ployz-core/tests/contract.rs index 0c527e7e1f..790c3bba56 100644 --- a/crates/ployz-core/tests/contract.rs +++ b/crates/ployz-core/tests/contract.rs @@ -17,10 +17,10 @@ use ployz_core::{ InspectMachineUpgradeRequest, InspectWireGuardRequest, LIST_IMAGES_CAPABILITY, ListImagesRequest, MANAGED_LABEL, MachineFailure, MachineGateway, MachineId, MachineImages, MachineName, MachineRelease, MachineSubnet, MachineSuccess, MachineTokenRequest, MachineUpdate, - MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeStage, MachineVersion, - ManagementAddress, NameMatches, OpaquePayload, PROJECT_NAME_LABEL, PROTOCOL_MAJOR, - PULL_IMAGE_FROM_MACHINE_CAPABILITY, PartialResult, PortPublication, ProjectName, - PublicIpDiscovery, PublicIpUpdate, PullImageFromMachineRequest, QualifiedService, + MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeOutcome, MachineUpgradeStage, + MachineVersion, ManagementAddress, NameMatches, OpaquePayload, PROJECT_NAME_LABEL, + PROTOCOL_MAJOR, PULL_IMAGE_FROM_MACHINE_CAPABILITY, PartialResult, PortPublication, + ProjectName, PublicIpDiscovery, PublicIpUpdate, PullImageFromMachineRequest, QualifiedService, RESET_MACHINE_CAPABILITY, RemoveLocalMachineRequest, RemoveMachineRequest, RequestMachineUpgradeRequest, RequestedServiceSpec, ReserveDomainRequest, ResetAccepted, ResetRequest, ResolvedServiceSpec, ResponseKind, RpcError, RpcErrorCode, RpcRequestBody, @@ -1505,11 +1505,13 @@ fn machine_upgrade_request_and_receipt_have_one_typed_wire_contract() { RpcRequestBody::InspectMachineUpgrade(inspect) ); - let attempt = MachineUpgradeAttempt::Failed { + let attempt = MachineUpgradeAttempt { attempt_id, target: MachineVersion::parse("1.2.3-beta.4").unwrap(), - stage: MachineUpgradeStage::Verifying, - error: "checksum mismatch".into(), + outcome: MachineUpgradeOutcome::Failed { + stage: MachineUpgradeStage::Verifying, + error: "checksum mismatch".into(), + }, }; let response = RpcResponse::from(attempt.clone()); assert_eq!( diff --git a/crates/ployz/src/cluster.rs b/crates/ployz/src/cluster.rs index a9dc478c64..44f21d74e0 100644 --- a/crates/ployz/src/cluster.rs +++ b/crates/ployz/src/cluster.rs @@ -133,7 +133,27 @@ impl Client { request: T::Request, target: Option<&MachineTarget>, ) -> Result { - let payload = T::into_request(request).encode()?; + self.call_repeatable_for::(request, target, crate::setup_retry::WAIT) + .await + .map_err(|error| match error { + crate::setup_retry::Error::Permanent(error) => error, + crate::setup_retry::Error::Exhausted(message) => { + ConnectError::Attempt(message.into()) + } + }) + } + + /// Retry a read or stable-identity request for the caller's remaining time budget. + pub(crate) async fn call_repeatable_for( + &mut self, + request: T::Request, + target: Option<&MachineTarget>, + wait: Duration, + ) -> Result> { + let payload = T::into_request(request) + .encode() + .map_err(ConnectError::from) + .map_err(crate::setup_retry::Error::Permanent)?; let mut redial = false; let operation = T::PATH.rsplit('/').next().unwrap_or(T::PATH); let destination = target.map_or_else( @@ -144,7 +164,7 @@ impl Client { crate::setup_retry::run( self, &progress, - crate::setup_retry::WAIT, + wait, ConnectError::is_setup_retryable, async |client| { let reconnect = redial; @@ -160,10 +180,6 @@ impl Client { }, ) .await - .map_err(|error| match error { - crate::setup_retry::Error::Permanent(error) => error, - crate::setup_retry::Error::Exhausted(message) => ConnectError::Attempt(message.into()), - }) } /// Setup mutations must not be replayed after a lost response. diff --git a/crates/ployz/src/handlers/machine/remove.rs b/crates/ployz/src/handlers/machine/remove.rs index a64d86f47f..80dc96f41e 100644 --- a/crates/ployz/src/handlers/machine/remove.rs +++ b/crates/ployz/src/handlers/machine/remove.rs @@ -112,7 +112,7 @@ pub(in crate::handlers) fn remove(root: &ArgMatches) -> Result<(), Error> { }) } -fn select_machine( +pub(super) fn select_machine( machines: &[ployz_core::MachineObservation], selector: &str, ) -> Result { diff --git a/crates/ployz/src/handlers/machine/upgrade.rs b/crates/ployz/src/handlers/machine/upgrade.rs index 929442f8e6..479ba16796 100644 --- a/crates/ployz/src/handlers/machine/upgrade.rs +++ b/crates/ployz/src/handlers/machine/upgrade.rs @@ -5,7 +5,7 @@ use std::{collections::BTreeSet, time::Duration}; use clap::ArgMatches; use ployz_core::{ InspectMachineUpgradeRequest, Machine, MachineRelease, MachineTarget, MachineUpgradeAttempt, - MachineUpgradeAttemptId, NameMatches, RequestMachineUpgradeRequest, op, + MachineUpgradeAttemptId, MachineUpgradeOutcome, RequestMachineUpgradeRequest, op, }; use tokio::time::Instant; @@ -16,6 +16,44 @@ use super::super::{Error, leaf_matches, string_values, with_client}; const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(16 * 60); const POLL_INTERVAL: Duration = Duration::from_secs(1); +trait UpgradeRequests { + async fn request_upgrade( + &mut self, + request: RequestMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result>; + + async fn inspect_upgrade( + &mut self, + request: InspectMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result>; +} + +impl UpgradeRequests for Client { + async fn request_upgrade( + &mut self, + request: RequestMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result> { + self.call_repeatable_for::(request, Some(target), wait) + .await + } + + async fn inspect_upgrade( + &mut self, + request: InspectMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result> { + self.call_repeatable_for::(request, Some(target), wait) + .await + } +} + pub(in crate::handlers) fn upgrade(root: &ArgMatches) -> Result<(), Error> { let matches = leaf_matches(root); let release = matches @@ -26,24 +64,31 @@ pub(in crate::handlers) fn upgrade(root: &ArgMatches) -> Result<(), Error> { with_client(root, |client| { Box::pin(async move { let machines = selected_machines(client, &selectors).await?; - for machine in machines { + for (index, machine) in machines.iter().enumerate() { let attempt_id = MachineUpgradeAttemptId::random(); - let attempt = run_one(client, &machine, release.clone(), attempt_id).await?; - print_attempt(&machine, &attempt); - match attempt { - MachineUpgradeAttempt::Succeeded { .. } => {} - MachineUpgradeAttempt::Failed { error, .. } => { + let attempt = match run_one(client, machine, release.clone(), attempt_id).await { + Ok(attempt) => attempt, + Err(error) => { + print_unattempted(&machines[index + 1..], machine); + return Err(error); + } + }; + print_attempt(machine, &attempt); + match attempt.outcome { + MachineUpgradeOutcome::Succeeded { .. } => {} + MachineUpgradeOutcome::Failed { error, .. } => { + print_unattempted(&machines[index + 1..], machine); return Err(Error::usage(error)); } - MachineUpgradeAttempt::Interrupted { .. } => { + MachineUpgradeOutcome::Interrupted { .. } => { + print_unattempted(&machines[index + 1..], machine); return Err(Error::usage(format!( "Machine {} upgrade was interrupted; {}", machine.name, journal_hint(attempt_id) ))); } - MachineUpgradeAttempt::Accepted { .. } - | MachineUpgradeAttempt::Running { .. } => { + MachineUpgradeOutcome::Accepted | MachineUpgradeOutcome::Running { .. } => { unreachable!("run_one returns only terminal evidence") } } @@ -90,40 +135,20 @@ async fn selected_machines( let mut selected = Vec::with_capacity(selectors.len()); let mut ids = BTreeSet::new(); for selector in selectors { - let target = MachineTarget::parse(selector)?; - let machine = match target.resolve(visible.iter().map(|entry| &entry.machine)) { - NameMatches::None => { - return Err(Error::usage(format!( - "Machine {} was not found", - selector.escape_debug() - ))); - } - NameMatches::One(machine) => machine, - matches @ NameMatches::Ambiguous { .. } => { - return Err(Error::usage(format!( - "Machine name {} is ambiguous: {}", - selector.escape_debug(), - matches - .iter() - .map(|machine| machine.id.as_str()) - .collect::>() - .join(", ") - ))); - } - }; + let machine = super::remove::select_machine(&visible, selector)?; if !ids.insert(machine.id) { return Err(Error::usage(format!( "Machine {} was selected more than once", machine.name ))); } - selected.push(machine.clone()); + selected.push(machine); } Ok(selected) } async fn run_one( - client: &mut Client, + client: &mut impl UpgradeRequests, machine: &Machine, release: MachineRelease, attempt_id: MachineUpgradeAttemptId, @@ -134,9 +159,13 @@ async fn run_one( attempt_id, release, }; - let accepted = request_until(deadline, client, &target, request) - .await - .map_err(|error| uncertain(machine, attempt_id, error))?; + let accepted = match request_until(deadline, client, &target, request).await { + Ok(accepted) => accepted, + Err(crate::setup_retry::Error::Permanent(error)) => return Err(error.into()), + Err(crate::setup_retry::Error::Exhausted(error)) => { + return Err(uncertain(machine, attempt_id, error)); + } + }; print_attempt(machine, &accepted); if accepted.is_terminal() { return Ok(accepted); @@ -151,7 +180,12 @@ async fn run_one( } let observed = inspect_until(deadline, client, &target, attempt_id) .await - .map_err(|error| uncertain(machine, attempt_id, error))?; + .map_err(|error| match error { + crate::setup_retry::Error::Permanent(error) => error.into(), + crate::setup_retry::Error::Exhausted(error) => { + uncertain(machine, attempt_id, error) + } + })?; if observed.is_terminal() { return Ok(observed); } @@ -160,57 +194,34 @@ async fn run_one( async fn request_until( deadline: Instant, - client: &mut Client, + client: &mut impl UpgradeRequests, target: &MachineTarget, request: RequestMachineUpgradeRequest, -) -> Result { - loop { - let outcome = tokio::time::timeout_at( - deadline, - client.call_repeatable::(request.clone(), Some(target)), +) -> Result> { + client + .request_upgrade( + request, + target, + deadline.saturating_duration_since(Instant::now()), ) - .await; - match outcome { - Ok(Ok(value)) => return Ok(value), - Ok(Err(error)) if !error.is_setup_retryable() => return Err(error), - Ok(Err(_)) => {} - Err(_) => { - return Err( - tonic::Status::deadline_exceeded("upgrade observation timed out").into(), - ); - } - } - } + .await } async fn inspect_until( deadline: Instant, - client: &mut Client, + client: &mut impl UpgradeRequests, target: &MachineTarget, attempt_id: MachineUpgradeAttemptId, -) -> Result { - loop { - let outcome = tokio::time::timeout_at( - deadline, - client.call_repeatable::( - InspectMachineUpgradeRequest { - attempt_id: Some(attempt_id), - }, - Some(target), - ), +) -> Result> { + client + .inspect_upgrade( + InspectMachineUpgradeRequest { + attempt_id: Some(attempt_id), + }, + target, + deadline.saturating_duration_since(Instant::now()), ) - .await; - match outcome { - Ok(Ok(value)) => return Ok(value), - Ok(Err(error)) if !error.is_setup_retryable() => return Err(error), - Ok(Err(_)) => {} - Err(_) => { - return Err( - tonic::Status::deadline_exceeded("upgrade observation timed out").into(), - ); - } - } - } + .await } fn print_attempt(machine: &Machine, attempt: &MachineUpgradeAttempt) { @@ -218,41 +229,30 @@ fn print_attempt(machine: &Machine, attempt: &MachineUpgradeAttempt) { } fn print_attempt_target(machine: &str, attempt: &MachineUpgradeAttempt) { - match attempt { - MachineUpgradeAttempt::Accepted { attempt_id, target } => println!( + let MachineUpgradeAttempt { + attempt_id, + target, + outcome, + } = attempt; + match outcome { + MachineUpgradeOutcome::Accepted => println!( "Machine {machine}: upgrade {attempt_id} accepted for {target}; {}", journal_hint(*attempt_id) ), - MachineUpgradeAttempt::Running { - attempt_id, - target, - stage, - } => println!( + MachineUpgradeOutcome::Running { stage } => println!( "Machine {machine}: upgrade {attempt_id} is {} for {target}; {}", stage.as_str(), journal_hint(*attempt_id) ), - MachineUpgradeAttempt::Succeeded { - attempt_id, - version, - } => { + MachineUpgradeOutcome::Succeeded { version } => { println!("Machine {machine}: upgrade {attempt_id} succeeded; running version {version}") } - MachineUpgradeAttempt::Failed { - attempt_id, - target, - stage, - error, - } => println!( + MachineUpgradeOutcome::Failed { stage, error } => println!( "Machine {machine}: upgrade {attempt_id} failed at {} for {target}: {error}; {}", stage.as_str(), journal_hint(*attempt_id) ), - MachineUpgradeAttempt::Interrupted { - attempt_id, - target, - stage, - } => println!( + MachineUpgradeOutcome::Interrupted { stage } => println!( "Machine {machine}: upgrade {attempt_id} was interrupted at {} for {target}; {}", stage.as_str(), journal_hint(*attempt_id) @@ -260,11 +260,33 @@ fn print_attempt_target(machine: &str, attempt: &MachineUpgradeAttempt) { } } +fn print_unattempted(machines: &[Machine], after: &Machine) { + for line in unattempted_lines(machines, after) { + println!("{line}"); + } +} + +fn unattempted_lines(machines: &[Machine], after: &Machine) -> Vec { + machines + .iter() + .map(|machine| { + format!( + "Machine {} ({}): upgrade unattempted after {} ({})", + machine.name, machine.id, after.name, after.id + ) + }) + .collect() +} + fn journal_hint(attempt_id: MachineUpgradeAttemptId) -> String { format!("inspect locally with `journalctl -u ployz-upgrade-{attempt_id}.service`") } -fn uncertain(machine: &Machine, attempt_id: MachineUpgradeAttemptId, error: ConnectError) -> Error { +fn uncertain( + machine: &Machine, + attempt_id: MachineUpgradeAttemptId, + error: impl std::fmt::Display, +) -> Error { Error::usage(format!( "Machine {} ({}) upgrade {attempt_id} outcome is uncertain: {error}; reconnect and run `ployz machine upgrade inspect {} --attempt {attempt_id}`; {}", machine.name, @@ -284,3 +306,132 @@ fn uncertain_timeout(machine: &Machine, attempt_id: MachineUpgradeAttemptId) -> journal_hint(attempt_id) )) } + +#[cfg(test)] +mod tests { + use ployz_core::{ + AdvertisedEndpoint, MachineId, MachineName, MachineRuntime, RpcError, RpcErrorCode, + WireGuardPublicKey, + }; + use serde_json::Value; + + use super::*; + + struct FakeRequests { + request: Option>>, + seen: Vec<(MachineUpgradeAttemptId, String)>, + } + + impl UpgradeRequests for FakeRequests { + async fn request_upgrade( + &mut self, + request: RequestMachineUpgradeRequest, + target: &MachineTarget, + _wait: Duration, + ) -> Result> { + self.seen + .push((request.attempt_id, target.as_str().to_owned())); + self.request.take().expect("one request result") + } + + async fn inspect_upgrade( + &mut self, + _request: InspectMachineUpgradeRequest, + _target: &MachineTarget, + _wait: Duration, + ) -> Result> { + panic!("an unaccepted request is not inspected") + } + } + + #[tokio::test] + async fn definitive_busy_rejection_is_not_reported_as_uncertain() { + let machine = machine('a', 1); + let attempt_id = MachineUpgradeAttemptId::parse("1".repeat(32)).unwrap(); + let mut client = FakeRequests { + request: Some(Err(crate::setup_retry::Error::Permanent( + ConnectError::Remote(RpcError { + code: RpcErrorCode::Conflict, + message: "a Machine upgrade or mutation is active".into(), + details: Value::Null, + }), + ))), + seen: Vec::new(), + }; + + let error = run_one( + &mut client, + &machine, + MachineRelease::parse("beta").unwrap(), + attempt_id, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("upgrade or mutation is active")); + assert!(!error.to_string().contains("outcome is uncertain")); + assert_eq!(client.seen, [(attempt_id, machine.id.as_str().to_owned())]); + } + + #[tokio::test] + async fn lost_request_reply_is_uncertain_and_keeps_the_dispatched_attempt_id() { + let machine = machine('b', 2); + let attempt_id = MachineUpgradeAttemptId::parse("2".repeat(32)).unwrap(); + let mut client = FakeRequests { + request: Some(Err(crate::setup_retry::Error::Exhausted( + "request reply was lost".into(), + ))), + seen: Vec::new(), + }; + + let error = run_one( + &mut client, + &machine, + MachineRelease::parse("1.2.3").unwrap(), + attempt_id, + ) + .await + .unwrap_err() + .to_string(); + + assert!(error.contains("outcome is uncertain"), "{error}"); + assert!(error.contains(attempt_id.as_str()), "{error}"); + assert!(error.contains("machine upgrade inspect"), "{error}"); + assert_eq!(client.seen, [(attempt_id, machine.id.as_str().to_owned())]); + } + + #[test] + fn first_and_middle_failures_report_the_complete_ordered_suffix() { + let machines = [ + machine('a', 1), + machine('b', 2), + machine('c', 3), + machine('d', 4), + ]; + + let first = unattempted_lines(&machines[1..], &machines[0]); + assert_eq!(first.len(), 3); + assert!(first[0].contains("Machine b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)")); + assert!(first[0].contains("after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)")); + assert!(first[1].contains("Machine c (cccccccccccccccccccccccccccccccc)")); + assert!(first[2].contains("Machine d (dddddddddddddddddddddddddddddddd)")); + + let middle = unattempted_lines(&machines[2..], &machines[1]); + assert_eq!(middle.len(), 2); + assert!(middle[0].contains("Machine c (cccccccccccccccccccccccccccccccc)")); + assert!(middle[0].contains("after b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)")); + assert!(middle[1].contains("Machine d (dddddddddddddddddddddddddddddddd)")); + } + + fn machine(id: char, subnet: u8) -> Machine { + Machine { + id: MachineId::parse(id.to_string().repeat(32)).unwrap(), + name: MachineName::parse(id.to_string()).unwrap(), + subnet: format!("10.210.{subnet}.0/24").parse().unwrap(), + public_key: WireGuardPublicKey([subnet; 32]), + public_ip: None, + advertised_endpoints: Vec::::new(), + runtime: MachineRuntime::default(), + } + } +} diff --git a/crates/ployzd/src/installer/admission.rs b/crates/ployzd/src/installer/admission.rs deleted file mode 100644 index f87cff9685..0000000000 --- a/crates/ployzd/src/installer/admission.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! Cross-process exclusion between installation and Machine-local mutations. - -use std::{ - fs::{self, File, OpenOptions, TryLockError}, - io, - os::unix::fs::{OpenOptionsExt, PermissionsExt}, - path::PathBuf, -}; - -use thiserror::Error; - -const LOCK_FILE: &str = ".install.lock"; -const ACTIVE_FILE: &str = ".upgrade-active"; - -#[derive(Clone, Debug)] -pub struct Admission { - lock_dir: PathBuf, - data_dir: PathBuf, -} - -#[derive(Debug, Error)] -pub enum Error { - #[error("a Ployz installation or upgrade is active")] - Busy, - #[error("Machine admission: {0}")] - Io(#[from] io::Error), -} - -pub struct MutationGuard(File); -pub struct InstallGuard(File); - -impl Admission { - #[must_use] - pub fn new(lock_dir: impl Into, data_dir: impl Into) -> Self { - Self { - lock_dir: lock_dir.into(), - data_dir: data_dir.into(), - } - } - - pub fn try_mutation(&self) -> Result { - let file = self.open_lock()?; - file.try_lock_shared().map_err(lock_error)?; - if self.active_path().is_file() { - let _ = file.unlock(); - return Err(Error::Busy); - } - Ok(MutationGuard(file)) - } - - #[must_use] - pub fn lock_dir(&self) -> &std::path::Path { - &self.lock_dir - } - - pub fn try_install(&self) -> Result { - let file = self.open_lock()?; - file.try_lock().map_err(lock_error)?; - Ok(InstallGuard(file)) - } - - pub fn lock_install(&self) -> Result { - let file = self.open_lock()?; - file.lock()?; - Ok(InstallGuard(file)) - } - - #[must_use] - pub fn active(&self) -> bool { - self.active_path().is_file() - } - - pub(super) fn mark_active(&self, attempt_id: &str) -> Result<(), Error> { - fs::create_dir_all(&self.data_dir)?; - crate::filesystem::atomic_write(&self.active_path(), attempt_id.as_bytes(), 0o600)?; - Ok(()) - } - - pub(super) fn clear_active(&self, attempt_id: &str) -> Result<(), Error> { - let path = self.active_path(); - match fs::read_to_string(&path) { - Ok(active) if active == attempt_id => fs::remove_file(path)?, - Ok(_) => return Ok(()), - Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(error.into()), - } - File::open(&self.data_dir)?.sync_all()?; - Ok(()) - } - - fn open_lock(&self) -> Result { - fs::create_dir_all(&self.lock_dir)?; - fs::set_permissions(&self.lock_dir, fs::Permissions::from_mode(0o750))?; - OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .mode(0o600) - .open(self.lock_dir.join(LOCK_FILE)) - .map_err(Into::into) - } - - fn active_path(&self) -> PathBuf { - self.data_dir.join(ACTIVE_FILE) - } -} - -fn lock_error(error: TryLockError) -> Error { - match error { - TryLockError::WouldBlock => Error::Busy, - TryLockError::Error(error) => Error::Io(error), - } -} - -impl Drop for MutationGuard { - fn drop(&mut self) { - let _ = self.0.unlock(); - } -} - -impl Drop for InstallGuard { - fn drop(&mut self) { - let _ = self.0.unlock(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn installation_and_active_handoff_exclude_mutations_across_instances() { - let root = crate::test_dir::TestDir::new("installer-admission"); - let first = Admission::new(root.0.join("run"), root.0.join("data")); - let second = Admission::new(root.0.join("run"), root.0.join("data")); - - let install = first.try_install().unwrap(); - assert!(matches!(second.try_mutation(), Err(Error::Busy))); - first.mark_active("attempt").unwrap(); - drop(install); - assert!(matches!(second.try_mutation(), Err(Error::Busy))); - - let install = second.try_install().unwrap(); - second.clear_active("attempt").unwrap(); - drop(install); - assert!(second.try_mutation().is_ok()); - } -} diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index cdba5845be..c7758b1d92 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -1,6 +1,5 @@ //! Bounded local installation of a Ployz Machine release. -pub mod admission; mod host; mod release; mod storage; @@ -13,9 +12,11 @@ use std::{ process::{Command, Output}, }; -use ployz_core::StorageChoice; +use ployz_core::{MachineUpgradeStage, StorageChoice}; use thiserror::Error; +use crate::mutation; + use self::{ host::{ create_user_and_directories, install_docker, install_prerequisites, install_systemd, @@ -30,20 +31,6 @@ pub use self::release::{ReleaseRequest, ReleaseSource}; const PLOYZ_USER: &str = "ployz"; const DEFAULT_BIN_DIR: &str = "/usr/local/bin"; const DEFAULT_SYSTEMD_DIR: &str = "/etc/systemd/system"; -const DEFAULT_DATA_DIR: &str = "/var/lib/ployz"; -const DEFAULT_RUN_DIR: &str = "/run/ployz"; - -/// Observable stage of the shared installation implementation. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum InstallStage { - Preparing, - Acquiring, - Verifying, - Activating, - Restarting, - Readiness, -} - /// Explicit host work associated with one Machine installation attempt. #[derive(Clone, Debug, Eq, PartialEq)] pub enum Preparation { @@ -155,22 +142,12 @@ impl InstallPaths { } } -/// Install the selected release through the one Machine-local installation seam. -/// -/// # Errors -/// -/// Returns the observed first failed installation stage. Acquisition and preflight failures -/// happen before activation, so the installed daemon remains untouched. -pub async fn install(request: InstallRequest) -> Result { - install_in(request, DEFAULT_DATA_DIR, DEFAULT_RUN_DIR).await -} - /// Install using the daemon's configured data and runtime directories. /// /// # Errors /// /// Returns the observed first failed installation stage. -pub async fn install_in( +pub async fn install( request: InstallRequest, data_dir: impl Into, run_dir: impl Into, @@ -180,8 +157,8 @@ pub async fn install_in( async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result { require_root()?; - let admission = admission::Admission::new(&paths.run_dir, &paths.data_dir); - let lock = admission.try_install().map_err(map_admission_error)?; + let admission = mutation::MutationGate::new(&paths.run_dir, &paths.data_dir); + let lock = admission.try_installation().map_err(map_admission_error)?; upgrade::reconcile_for_install(&admission, &paths.data_dir) .await .map_err(map_upgrade_reconciliation)?; @@ -191,13 +168,13 @@ async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result Result<(), Error>, + _lock: mutation::InstallationGuard, + mut progress: impl FnMut(MachineUpgradeStage) -> Result<(), Error>, ) -> Result { verify_system(request.install_only)?; let target = resolve_release(&request.release, &request.source).await?; - progress(InstallStage::Preparing)?; + progress(MachineUpgradeStage::Preparing)?; if !request.install_only { match &request.preparation { Preparation::SoftwareOnly => verify_software_prerequisites(&paths)?, @@ -227,14 +204,14 @@ async fn install_locked( Readiness::InstallationOnly } else { if restart_required { - progress(InstallStage::Restarting)?; + progress(MachineUpgradeStage::Restarting)?; systemctl("restart daemon", ["restart", "ployz.service"])?; systemctl( "restart volume plugin", ["try-restart", "ployz-volume-plugin.service"], )?; } - progress(InstallStage::Readiness)?; + progress(MachineUpgradeStage::Readiness)?; verify_running_daemon(&paths, &target).await?; Readiness::Running }; @@ -244,10 +221,10 @@ async fn install_locked( }) } -fn map_admission_error(error: admission::Error) -> Error { +fn map_admission_error(error: mutation::Error) -> Error { match error { - admission::Error::Busy => Error::Busy, - admission::Error::Io(source) => Error::Io { + mutation::Error::Busy => Error::Busy, + mutation::Error::Io(source) => Error::Io { stage: "claim Machine installation admission", source, }, @@ -518,8 +495,8 @@ mod tests { install_only: true, }; let result = if case == "busy" { - let admission = admission::Admission::new(&paths.run_dir, &paths.data_dir); - let _held = admission.try_install().unwrap(); + let admission = mutation::MutationGate::new(&paths.run_dir, &paths.data_dir); + let _held = admission.try_installation().unwrap(); install_at(request, paths.clone()).await } else { install_at(request, paths.clone()).await diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs index 2a357d9039..2e36f1d2fc 100644 --- a/crates/ployzd/src/installer/release.rs +++ b/crates/ployzd/src/installer/release.rs @@ -18,7 +18,8 @@ use sha2::{Digest, Sha256}; use tempfile::TempDir; use tokio::{process::Command, time::timeout}; -use super::{Error, InstallPaths, InstallStage, daemon_archive, run_command}; +use super::{Error, InstallPaths, daemon_archive, run_command}; +use ployz_core::MachineUpgradeStage; const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; const CHANNEL_URL: &str = "https://ployz.sh"; @@ -54,7 +55,8 @@ impl FromStr for ReleaseRequest { } /// A release source that is fixed by the local installer invocation. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum ReleaseSource { /// Ployz's fixed, trusted published release and channel endpoints. Published, @@ -157,7 +159,7 @@ pub(super) async fn install_binaries( source: &ReleaseSource, paths: &InstallPaths, target: &Version, - progress: &mut impl FnMut(InstallStage) -> Result<(), Error>, + progress: &mut impl FnMut(MachineUpgradeStage) -> Result<(), Error>, ) -> Result { let installed = installed_release(&paths.bin_dir.join("ployzd")).await?; let replace = replacement_required(source, installed.as_ref(), target); @@ -169,7 +171,7 @@ pub(super) async fn install_binaries( return Ok(false); } - progress(InstallStage::Acquiring)?; + progress(MachineUpgradeStage::Acquiring)?; let archive = daemon_archive()?; let stage = staging_directory(&paths.bin_dir)?; let archive_path = stage.path().join(archive); @@ -177,7 +179,7 @@ pub(super) async fn install_binaries( let archive_bytes = source .release_file(target, archive, "download daemon archive") .await?; - progress(InstallStage::Verifying)?; + progress(MachineUpgradeStage::Verifying)?; verify_checksum(&archive_bytes, archive, &checksum)?; write_private(&archive_path, &archive_bytes, "stage daemon archive")?; extract_archive(&archive_path, stage.path())?; @@ -186,7 +188,7 @@ pub(super) async fn install_binaries( verify_executable(&daemon, target).await?; verify_uninstall(&uninstall)?; sync_staged_files(&daemon, &uninstall, stage.path())?; - progress(InstallStage::Activating)?; + progress(MachineUpgradeStage::Activating)?; activate(&daemon, &uninstall, paths)?; Ok(true) } diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs index 5333fed415..32283ba53a 100644 --- a/crates/ployzd/src/installer/storage.rs +++ b/crates/ployzd/src/installer/storage.rs @@ -309,10 +309,11 @@ fn set_and_verify_zfs_arc_max(cap: u64) -> Result<(), Error> { stage: "apply ZFS ARC limit", source, })?; - writeln!(file, "{cap}").map_err(|source| Error::Io { - stage: "apply ZFS ARC limit", - source, - })?; + file.write_all(cap.to_string().as_bytes()) + .map_err(|source| Error::Io { + stage: "apply ZFS ARC limit", + source, + })?; let observed = fs::read_to_string(path).map_err(|source| Error::Io { stage: "verify ZFS ARC limit", source, diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs index 19bc681a3f..eac00c1505 100644 --- a/crates/ployzd/src/installer/upgrade.rs +++ b/crates/ployzd/src/installer/upgrade.rs @@ -7,8 +7,8 @@ use std::{ }; use ployz_core::{ - MachineRelease, MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeStage, - MachineVersion, RequestMachineUpgradeRequest, + MachineRelease, MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeOutcome, + MachineUpgradeStage, MachineVersion, RequestMachineUpgradeRequest, }; use semver::Version; use serde::{Deserialize, Serialize}; @@ -16,9 +16,9 @@ use thiserror::Error; use tokio::{process::Command, time::timeout}; use super::{ - Error as InstallError, InstallPaths, InstallRequest, InstallStage, Preparation, ReleaseRequest, - ReleaseSource, admission, + Error as InstallError, InstallPaths, InstallRequest, Preparation, ReleaseRequest, ReleaseSource, }; +use crate::mutation; const RECEIPT_FILE: &str = "upgrade-attempt.json"; const QUALIFICATION_RELEASE_DIR: &str = "PLOYZ_UPGRADE_RELEASE_DIR"; @@ -27,82 +27,83 @@ const LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); #[derive(Debug, Error)] pub enum Error { + /// No receipt exists for the requested attempt. #[error("no Machine upgrade attempt was found")] NotFound, + /// A retry identity was reused with another release selector. #[error("Machine upgrade attempt {0} was reused with a different release")] AttemptConflict(MachineUpgradeAttemptId), + /// Another installation or local mutation currently owns the Machine. #[error("a Machine upgrade or mutation is active")] Busy, + /// The durable receipt could not be read. #[error("read Machine upgrade receipt: {0}")] Read(#[source] io::Error), + /// The durable receipt was not valid JSON for the current contract. #[error("decode Machine upgrade receipt: {0}")] Decode(#[source] serde_json::Error), + /// The durable receipt could not be atomically written. #[error("write Machine upgrade receipt: {0}")] Write(#[source] io::Error), + /// The current receipt could not be encoded. #[error("encode Machine upgrade receipt: {0}")] Encode(#[source] serde_json::Error), + /// The process-local qualification source was not a trusted absolute directory. #[error("invalid qualification release directory: {0}")] QualificationSource(String), + /// The requested release could not be parsed or resolved. #[error("resolve Machine release: {0}")] Resolve(#[source] InstallError), + /// The system manager rejected or did not confirm worker launch. #[error("launch Machine upgrade worker: {0}")] Launch(String), + /// The worker process or unit could not be queried. #[error("inspect Machine upgrade worker: {0}")] InspectWorker(#[source] io::Error), + /// `systemctl` returned evidence that did not prove either a running or stopped worker. + #[error("inspect Machine upgrade worker: {0}")] + WorkerEvidence(String), + /// The worker identity does not match the current nonterminal receipt. #[error("Machine upgrade worker does not own active attempt {0}")] NotActive(MachineUpgradeAttemptId), + /// The shared installer recorded a terminal failure. #[error("Machine upgrade failed: {0}")] Installation(#[source] InstallError), + /// Machine mutation ownership could not be claimed or inspected. #[error(transparent)] - Admission(#[from] admission::Error), + Admission(#[from] mutation::Error), } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] struct StoredAttempt { requested: MachineRelease, - source: StoredReleaseSource, + source: ReleaseSource, attempt: MachineUpgradeAttempt, } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum StoredReleaseSource { - Published, - Local { directory: PathBuf }, -} - -impl StoredReleaseSource { - fn current() -> Result { - let Some(directory) = env::var_os(QUALIFICATION_RELEASE_DIR) else { - return Ok(Self::Published); - }; - let directory = PathBuf::from(directory); - if !directory.is_absolute() { - return Err(Error::QualificationSource(format!( - "{QUALIFICATION_RELEASE_DIR} must be an absolute path" - ))); - } - Ok(Self::Local { directory }) - } - - fn installer(&self) -> ReleaseSource { - match self { - Self::Published => ReleaseSource::Published, - Self::Local { directory } => ReleaseSource::Local(directory.clone()), - } +fn current_source() -> Result { + let Some(directory) = env::var_os(QUALIFICATION_RELEASE_DIR) else { + return Ok(ReleaseSource::Published); + }; + let directory = PathBuf::from(directory); + if !directory.is_absolute() { + return Err(Error::QualificationSource(format!( + "{QUALIFICATION_RELEASE_DIR} must be an absolute path" + ))); } + Ok(ReleaseSource::Local(directory)) } /// Return the durable result of retrying the same request without requiring installation /// ownership. The receipt is atomically replaced, so an active worker can be observed safely. -pub fn existing_request( +pub(crate) fn existing_request( request: &RequestMachineUpgradeRequest, data_dir: &Path, ) -> Result, Error> { let Some(stored) = read_optional(data_dir)? else { return Ok(None); }; - if stored.attempt.attempt_id() != request.attempt_id { + if stored.attempt.attempt_id != request.attempt_id { return Ok(None); } if stored.requested != request.release { @@ -113,15 +114,15 @@ pub fn existing_request( /// Persist an accepted attempt and launch its old-daemon worker while installation admission is /// held. Reusing the same attempt ID returns the same attempt. -pub async fn request_locked( +pub(crate) async fn request_locked( request: RequestMachineUpgradeRequest, data_dir: &Path, run_dir: &Path, - _guard: &admission::InstallGuard, + _guard: &mutation::InstallationGuard, ) -> Result { - let admission = admission::Admission::new(run_dir, data_dir); + let admission = mutation::MutationGate::new(run_dir, data_dir); if let Some(mut stored) = read_optional(data_dir)? { - if stored.attempt.attempt_id() == request.attempt_id { + if stored.attempt.attempt_id == request.attempt_id { if stored.requested != request.release { return Err(Error::AttemptConflict(request.attempt_id)); } @@ -134,13 +135,16 @@ pub async fn request_locked( } } - let source = StoredReleaseSource::current()?; + if admission.active()? { + return Err(Error::Busy); + } + let source = current_source()?; let release = request .release .as_str() .parse::() .map_err(Error::Resolve)?; - let target = super::release::resolve_release(&release, &source.installer()) + let target = super::release::resolve_release(&release, &source) .await .map_err(Error::Resolve)?; let target = MachineVersion::parse(target.to_string()) @@ -148,18 +152,17 @@ pub async fn request_locked( let mut stored = StoredAttempt { requested: request.release, source, - attempt: MachineUpgradeAttempt::Accepted { + attempt: MachineUpgradeAttempt { attempt_id: request.attempt_id, target, + outcome: MachineUpgradeOutcome::Accepted, }, }; write(data_dir, &stored)?; admission.mark_active(request.attempt_id.as_str())?; if let Err(error) = launch_worker(request.attempt_id, data_dir, run_dir).await { - stored.attempt = MachineUpgradeAttempt::Failed { - attempt_id: request.attempt_id, - target: stored.attempt.target().clone(), + stored.attempt.outcome = MachineUpgradeOutcome::Failed { stage: MachineUpgradeStage::Launching, error: error.to_string(), }; @@ -170,27 +173,31 @@ pub async fn request_locked( } /// Read an attempt, turning a stopped worker without terminal evidence into interruption. -pub async fn inspect( +pub(crate) async fn inspect( attempt_id: Option, data_dir: &Path, run_dir: &Path, ) -> Result { - let admission = admission::Admission::new(run_dir, data_dir); + let admission = mutation::MutationGate::new(run_dir, data_dir); let mut stored = read(data_dir)?; - if attempt_id.is_some_and(|attempt_id| attempt_id != stored.attempt.attempt_id()) { + if attempt_id.is_some_and(|attempt_id| attempt_id != stored.attempt.attempt_id) { return Err(Error::NotFound); } if stored.attempt.is_terminal() { - if let Ok(_guard) = admission.try_install() { - admission.clear_active(stored.attempt.attempt_id().as_str())?; + match admission.try_installation() { + Ok(_guard) => admission.clear_active(stored.attempt.attempt_id.as_str())?, + Err(mutation::Error::Busy) => {} + Err(error) => return Err(error.into()), } return Ok(stored.attempt); } - if worker_active(stored.attempt.attempt_id()).await? { + if worker_state(stored.attempt.attempt_id).await? == WorkerState::Active { return Ok(stored.attempt); } - let Ok(_guard) = admission.try_install() else { - return Ok(stored.attempt); + let _guard = match admission.try_installation() { + Ok(guard) => guard, + Err(mutation::Error::Busy) => return Ok(stored.attempt), + Err(error) => return Err(error.into()), }; stored = read(data_dir)?; reconcile_locked(&admission, data_dir, &mut stored).await?; @@ -198,24 +205,27 @@ pub async fn inspect( } /// Run the accepted attempt from the transient systemd service. +/// +/// # Errors +/// +/// Returns an ownership, receipt, installation, or durable result error when the worker cannot +/// complete and record the accepted attempt. pub async fn run_worker( attempt_id: MachineUpgradeAttemptId, data_dir: &Path, run_dir: &Path, ) -> Result<(), Error> { - let admission = admission::Admission::new(run_dir, data_dir); - let guard = admission.lock_install()?; + let admission = mutation::MutationGate::new(run_dir, data_dir); + let guard = admission.lock_installation()?; let mut stored = read(data_dir)?; - if stored.attempt.attempt_id() != attempt_id || stored.attempt.is_terminal() { + if stored.attempt.attempt_id != attempt_id || stored.attempt.is_terminal() { return Err(Error::NotActive(attempt_id)); } - let target = stored.attempt.target().clone(); + let target = stored.attempt.target.clone(); let target_version = Version::parse(target.as_str()) .expect("a MachineVersion contains a supported semantic version"); let mut stage = MachineUpgradeStage::Preparing; - stored.attempt = MachineUpgradeAttempt::Running { - attempt_id, - target: target.clone(), + stored.attempt.outcome = MachineUpgradeOutcome::Running { stage: stage.clone(), }; write(data_dir, &stored)?; @@ -223,17 +233,15 @@ pub async fn run_worker( let result = super::install_locked( InstallRequest { release: ReleaseRequest::Exact(target_version), - source: stored.source.installer(), + source: stored.source.clone(), preparation: Preparation::SoftwareOnly, install_only: false, }, InstallPaths::system(data_dir, run_dir), guard, |install_stage| { - stage = upgrade_stage(install_stage); - stored.attempt = MachineUpgradeAttempt::Running { - attempt_id, - target: target.clone(), + stage = install_stage; + stored.attempt.outcome = MachineUpgradeOutcome::Running { stage: stage.clone(), }; write(data_dir, &stored).map_err(|error| InstallError::Io { @@ -246,18 +254,15 @@ pub async fn run_worker( match result { Ok(_) => { - stored.attempt = MachineUpgradeAttempt::Succeeded { - attempt_id, - version: target, + stored.attempt.outcome = MachineUpgradeOutcome::Succeeded { + version: target.clone(), }; write(data_dir, &stored)?; admission.clear_active(attempt_id.as_str())?; Ok(()) } Err(error) => { - stored.attempt = MachineUpgradeAttempt::Failed { - attempt_id, - target, + stored.attempt.outcome = MachineUpgradeOutcome::Failed { stage, error: error.to_string(), }; @@ -269,29 +274,23 @@ pub async fn run_worker( } /// Reconcile a retained receipt before serving requests after daemon restart. -pub async fn reconcile(data_dir: &Path, run_dir: &Path) -> Result<(), Error> { - let admission = admission::Admission::new(run_dir, data_dir); - let Some(mut stored) = read_optional(data_dir)? else { - return Ok(()); - }; - if stored.attempt.is_terminal() { - if let Ok(_guard) = admission.try_install() { - admission.clear_active(stored.attempt.attempt_id().as_str())?; - } - return Ok(()); +pub(crate) async fn reconcile(data_dir: &Path, run_dir: &Path) -> Result<(), Error> { + match inspect(None, data_dir, run_dir).await { + Ok(_) | Err(Error::NotFound) => Ok(()), + Err(error) => Err(error), } - let Ok(_guard) = admission.try_install() else { - return Ok(()); - }; - reconcile_locked(&admission, data_dir, &mut stored).await } pub(super) async fn reconcile_for_install( - admission: &admission::Admission, + admission: &mutation::MutationGate, data_dir: &Path, ) -> Result<(), Error> { let Some(mut stored) = read_optional(data_dir)? else { - return Ok(()); + return if admission.active()? { + Err(Error::Busy) + } else { + Ok(()) + }; }; reconcile_locked(admission, data_dir, &mut stored).await?; if stored.attempt.is_terminal() { @@ -302,31 +301,27 @@ pub(super) async fn reconcile_for_install( } async fn reconcile_locked( - admission: &admission::Admission, + admission: &mutation::MutationGate, data_dir: &Path, stored: &mut StoredAttempt, ) -> Result<(), Error> { if stored.attempt.is_terminal() { - admission.clear_active(stored.attempt.attempt_id().as_str())?; + admission.clear_active(stored.attempt.attempt_id.as_str())?; return Ok(()); } - if worker_active(stored.attempt.attempt_id()).await? { + if worker_state(stored.attempt.attempt_id).await? == WorkerState::Active { return Ok(()); } - let stage = match &stored.attempt { - MachineUpgradeAttempt::Accepted { .. } => MachineUpgradeStage::Launching, - MachineUpgradeAttempt::Running { stage, .. } => stage.clone(), - MachineUpgradeAttempt::Succeeded { .. } - | MachineUpgradeAttempt::Failed { .. } - | MachineUpgradeAttempt::Interrupted { .. } => return Ok(()), - }; - stored.attempt = MachineUpgradeAttempt::Interrupted { - attempt_id: stored.attempt.attempt_id(), - target: stored.attempt.target().clone(), - stage, + let stage = match &stored.attempt.outcome { + MachineUpgradeOutcome::Accepted => MachineUpgradeStage::Launching, + MachineUpgradeOutcome::Running { stage } => stage.clone(), + MachineUpgradeOutcome::Succeeded { .. } + | MachineUpgradeOutcome::Failed { .. } + | MachineUpgradeOutcome::Interrupted { .. } => return Ok(()), }; + stored.attempt.outcome = MachineUpgradeOutcome::Interrupted { stage }; write(data_dir, stored)?; - admission.clear_active(stored.attempt.attempt_id().as_str())?; + admission.clear_active(stored.attempt.attempt_id.as_str())?; Ok(()) } @@ -389,30 +384,69 @@ async fn launch_worker( } } -async fn worker_active(attempt_id: MachineUpgradeAttemptId) -> Result { +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WorkerState { + Active, + Stopped, +} + +async fn worker_state(attempt_id: MachineUpgradeAttemptId) -> Result { let output = Command::new("systemctl") - .args(["is-active", "--quiet", &unit_name(attempt_id)]) + .args([ + "show", + "--property=LoadState", + "--property=ActiveState", + &unit_name(attempt_id), + ]) .output() .await .map_err(Error::InspectWorker)?; - Ok(output.status.success()) + if !output.status.success() { + let detail = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + return Err(Error::WorkerEvidence(if detail.is_empty() { + format!("systemctl show exited with {}", output.status) + } else { + format!("systemctl show exited with {}: {detail}", output.status) + })); + } + let properties = String::from_utf8(output.stdout) + .map_err(|_| Error::WorkerEvidence("systemctl show returned non-UTF-8 output".into()))?; + let mut load = None; + let mut active = None; + for line in properties.lines() { + if let Some(value) = line.strip_prefix("LoadState=") { + if load.replace(value).is_some() { + return Err(Error::WorkerEvidence( + "systemctl show repeated LoadState".into(), + )); + } + } else if let Some(value) = line.strip_prefix("ActiveState=") + && active.replace(value).is_some() + { + return Err(Error::WorkerEvidence( + "systemctl show repeated ActiveState".into(), + )); + } + } + if load == Some("not-found") { + return Ok(WorkerState::Stopped); + } + match active { + Some( + "active" | "activating" | "reloading" | "deactivating" | "maintenance" | "refreshing", + ) => Ok(WorkerState::Active), + Some("inactive" | "failed") => Ok(WorkerState::Stopped), + _ => Err(Error::WorkerEvidence(format!( + "systemctl show returned no recognized state: {}", + properties.trim().escape_debug() + ))), + } } fn unit_name(attempt_id: MachineUpgradeAttemptId) -> String { format!("ployz-upgrade-{attempt_id}.service") } -fn upgrade_stage(stage: InstallStage) -> MachineUpgradeStage { - match stage { - InstallStage::Preparing => MachineUpgradeStage::Preparing, - InstallStage::Acquiring => MachineUpgradeStage::Acquiring, - InstallStage::Verifying => MachineUpgradeStage::Verifying, - InstallStage::Activating => MachineUpgradeStage::Activating, - InstallStage::Restarting => MachineUpgradeStage::Restarting, - InstallStage::Readiness => MachineUpgradeStage::Readiness, - } -} - fn read(data_dir: &Path) -> Result { read_optional(data_dir)?.ok_or(Error::NotFound) } @@ -447,12 +481,11 @@ mod tests { let attempt_id = MachineUpgradeAttemptId::random(); let stored = StoredAttempt { requested: MachineRelease::parse("beta").unwrap(), - source: StoredReleaseSource::Local { - directory: "/root/qualification".into(), - }, - attempt: MachineUpgradeAttempt::Accepted { + source: ReleaseSource::Local("/root/qualification".into()), + attempt: MachineUpgradeAttempt { attempt_id, target: MachineVersion::parse("1.2.3-beta.4").unwrap(), + outcome: MachineUpgradeOutcome::Accepted, }, }; let encoded = serde_json::to_string(&stored).unwrap(); @@ -481,7 +514,12 @@ mod tests { return; } - for case in ["retry-active", "interrupted", "launch-failed"] { + for case in [ + "retry-active", + "interrupted", + "inspection-unknown", + "launch-failed", + ] { let root = tempfile::Builder::new() .prefix(&format!("ployzd-upgrade-{case}-")) .tempdir() @@ -498,10 +536,10 @@ mod tests { ); write_script( &commands.join("systemctl"), - if case == "interrupted" { - "exit 3" - } else { - "exit 0" + match case { + "interrupted" => "printf 'LoadState=loaded\\nActiveState=inactive\\n'", + "inspection-unknown" => "echo manager unavailable >&2; exit 1", + _ => "printf 'LoadState=loaded\\nActiveState=active\\n'", }, ); fs::create_dir(root.path().join("release")).unwrap(); @@ -538,13 +576,13 @@ mod tests { let root = PathBuf::from(env::var_os(CONTRACT_ROOT).unwrap()); let data = root.join("data"); let run = root.join("run"); - let admission = admission::Admission::new(&run, &data); + let admission = mutation::MutationGate::new(&run, &data); let attempt_id = MachineUpgradeAttemptId::parse("a".repeat(32)).unwrap(); let request = RequestMachineUpgradeRequest { attempt_id, release: MachineRelease::parse("1.2.3").unwrap(), }; - let guard = admission.try_install().unwrap(); + let guard = admission.try_installation().unwrap(); let accepted = request_locked(request.clone(), &data, &run, &guard) .await .unwrap(); @@ -552,14 +590,14 @@ mod tests { match case { "retry-active" => { - assert!(matches!(accepted, MachineUpgradeAttempt::Accepted { .. })); + assert!(matches!(accepted.outcome, MachineUpgradeOutcome::Accepted)); assert_eq!( existing_request(&request, &data).unwrap(), Some(accepted.clone()) ); assert!(matches!( admission.try_mutation(), - Err(admission::Error::Busy) + Err(mutation::Error::Busy) )); let conflict = RequestMachineUpgradeRequest { attempt_id, @@ -570,7 +608,7 @@ mod tests { Err(Error::AttemptConflict(id)) if id == attempt_id )); - let guard = admission.try_install().unwrap(); + let guard = admission.try_installation().unwrap(); assert_eq!( request_locked(request, &data, &run, &guard).await.unwrap(), accepted @@ -601,25 +639,35 @@ mod tests { "interrupted" => { let observed = inspect(Some(attempt_id), &data, &run).await.unwrap(); assert!(matches!( - observed, - MachineUpgradeAttempt::Interrupted { + observed.outcome, + MachineUpgradeOutcome::Interrupted { stage: MachineUpgradeStage::Launching, - .. } )); - assert!(!admission.active()); + assert!(!admission.active().unwrap()); assert!(admission.try_mutation().is_ok()); } + "inspection-unknown" => { + assert!(matches!( + inspect(Some(attempt_id), &data, &run).await, + Err(Error::WorkerEvidence(ref message)) + if message.contains("manager unavailable") + )); + assert!(admission.active().unwrap()); + assert!(matches!( + admission.try_mutation(), + Err(mutation::Error::Busy) + )); + } "launch-failed" => { assert!(matches!( - accepted, - MachineUpgradeAttempt::Failed { + accepted.outcome, + MachineUpgradeOutcome::Failed { stage: MachineUpgradeStage::Launching, ref error, - .. } if error == "launch Machine upgrade worker: worker launch refused" )); - assert!(!admission.active()); + assert!(!admission.active().unwrap()); assert_eq!( inspect(Some(attempt_id), &data, &run).await.unwrap(), accepted diff --git a/crates/ployzd/src/lib.rs b/crates/ployzd/src/lib.rs index 5b99dc4955..b899b460a5 100644 --- a/crates/ployzd/src/lib.rs +++ b/crates/ployzd/src/lib.rs @@ -28,6 +28,7 @@ pub mod machine; pub mod machine_api; #[doc(hidden)] pub mod machine_pool; +pub mod mutation; pub mod network; pub mod relay; pub(crate) mod runtime_watch; diff --git a/crates/ployzd/src/machine/local_machine.rs b/crates/ployzd/src/machine/local_machine.rs index 76f6f83e13..6bad65db8a 100644 --- a/crates/ployzd/src/machine/local_machine.rs +++ b/crates/ployzd/src/machine/local_machine.rs @@ -40,6 +40,7 @@ pub struct LocalMachine { } mod container; +mod upgrade; #[derive(Clone)] struct ClusterContext { @@ -121,19 +122,14 @@ pub enum Error { #[error("this Machine is isolation-locked")] IsolationLocked, #[error(transparent)] - Admission(#[from] crate::installer::admission::Error), -} - -pub(crate) struct UpgradeAdmission { - pub(crate) data_dir: std::path::PathBuf, - pub(crate) run_dir: std::path::PathBuf, - pub(crate) install: crate::installer::admission::InstallGuard, - _local: OwnedMutexGuard<()>, + Admission(#[from] crate::mutation::Error), + #[error(transparent)] + Upgrade(#[from] crate::installer::upgrade::Error), } struct MutationAdmission { _local: OwnedMutexGuard<()>, - _installation: crate::installer::admission::MutationGuard, + _installation: crate::mutation::MutationGuard, } impl LocalMachine { @@ -197,10 +193,7 @@ impl LocalMachine { async fn admit_mutation(&self) -> Result { let (local, installation) = { let store = self.lock_store()?; - ( - store.admission_lock.clone(), - store.installation_admission.clone(), - ) + (store.admission_lock.clone(), store.mutation_gate.clone()) }; let local = local.lock_owned().await; let installation = installation.try_mutation()?; @@ -210,7 +203,7 @@ impl LocalMachine { }) } - pub(crate) async fn run_mutation(&self, work: F) -> Result + async fn finish_mutation(&self, work: F) -> Result where T: Send + 'static, F: std::future::Future> + Send + 'static, @@ -223,34 +216,18 @@ impl LocalMachine { .await? } - pub(crate) fn try_upgrade_admission(&self) -> Result { - let (data_dir, run_dir, local, installation) = { - let store = self.lock_store()?; - ( - store.data_dir.clone(), - store.installation_admission.lock_dir().to_owned(), - store.admission_lock.clone(), - store.installation_admission.clone(), - ) - }; - let local = local - .try_lock_owned() - .map_err(|_| crate::installer::admission::Error::Busy)?; - let install = installation.try_install()?; - Ok(UpgradeAdmission { - data_dir, - run_dir, - install, - _local: local, + #[cfg(test)] + pub(crate) async fn hold_mutation_for_test( + &self, + started: Arc, + release: Arc, + ) -> Result<(), Error> { + self.finish_mutation(async move { + started.notify_one(); + release.notified().await; + Ok(()) }) - } - - pub(crate) fn upgrade_paths(&self) -> Result<(std::path::PathBuf, std::path::PathBuf), Error> { - let store = self.lock_store()?; - Ok(( - store.data_dir.clone(), - store.installation_admission.lock_dir().to_owned(), - )) + .await } pub(crate) fn replicated(&self) -> Result<&ReplicatedStore, Error> { @@ -420,7 +397,7 @@ impl LocalMachine { /// /// Returns [`Error::LockPoisoned`] when the local record lock is poisoned /// and [`Error::Store`] when initialize is not legal in the current phase. - pub fn initialize(&self, request: InitializeRequest) -> Result { + fn initialize_admitted(&self, request: InitializeRequest) -> Result { let machine = self.lock_store()?.initialize( request.name, FoundingCluster { @@ -440,12 +417,15 @@ impl LocalMachine { Ok(Initialized { machine }) } - pub(crate) async fn initialize_mutation( - &self, - request: InitializeRequest, - ) -> Result { + /// Initialize this Machine under local mutation admission. + /// + /// # Errors + /// + /// Returns when mutation admission is busy, the local task fails, or initialization is not + /// legal in the current persisted phase. + pub async fn initialize(&self, request: InitializeRequest) -> Result { let local = self.clone(); - self.run_mutation(async move { local.initialize(request) }) + self.finish_mutation(async move { local.initialize_admitted(request) }) .await } @@ -470,7 +450,7 @@ impl LocalMachine { /// fails, and [`Error::Cluster`] when replicated I/O fails. pub async fn register(&self, request: RegisterRequest) -> Result { let local = self.clone(); - self.run_mutation(async move { local.register_admitted(request).await }) + self.finish_mutation(async move { local.register_admitted(request).await }) .await } @@ -589,7 +569,7 @@ impl LocalMachine { /// /// Returns [`Error::LockPoisoned`] when the local record lock is poisoned /// and [`Error::Store`] when join is not legal in the current phase. - pub fn join(&self, request: JoinRequest) -> Result { + fn join_admitted(&self, request: JoinRequest) -> Result { let mut store = self.lock_store()?; store.join( request.registration.assigned_machine, @@ -612,9 +592,16 @@ impl LocalMachine { Ok(JoinAccepted {}) } - pub(crate) async fn join_mutation(&self, request: JoinRequest) -> Result { + /// Persist a join assignment under local mutation admission and request restart. + /// + /// # Errors + /// + /// Returns when mutation admission is busy, the local task fails, or join is not legal in + /// the current persisted phase. + pub async fn join(&self, request: JoinRequest) -> Result { let local = self.clone(); - self.run_mutation(async move { local.join(request) }).await + self.finish_mutation(async move { local.join_admitted(request) }) + .await } /// Persist Cloud Pairing so this Machine can hold Relay Register, or clear @@ -625,7 +612,7 @@ impl LocalMachine { /// Returns [`Error::NotParticipating`] when this Machine is not /// participating, [`Error::LockPoisoned`] when the local record lock is /// poisoned, and [`Error::Store`] when the record cannot be written. - pub fn set_cloud_pairing(&self, pairing: Option) -> Result<(), Error> { + fn set_cloud_pairing_admitted(&self, pairing: Option) -> Result<(), Error> { let mut store = self.lock_store()?; if store.record().phase() != LocalMachinePhase::Participating { return Err(Error::NotParticipating); @@ -634,12 +621,15 @@ impl LocalMachine { Ok(()) } - pub(crate) async fn set_cloud_pairing_mutation( - &self, - pairing: Option, - ) -> Result<(), Error> { + /// Persist or clear Cloud Pairing under local mutation admission. + /// + /// # Errors + /// + /// Returns when mutation admission is busy, the local task fails, this Machine is not + /// participating, or the local record cannot be persisted. + pub async fn set_cloud_pairing(&self, pairing: Option) -> Result<(), Error> { let local = self.clone(); - self.run_mutation(async move { local.set_cloud_pairing(pairing) }) + self.finish_mutation(async move { local.set_cloud_pairing_admitted(pairing) }) .await } @@ -685,7 +675,7 @@ impl LocalMachine { /// when listing visible Machines fails. pub async fn update(&self, request: UpdateMachineRequest) -> Result { let local = self.clone(); - self.run_mutation(async move { local.update_admitted(request).await }) + self.finish_mutation(async move { local.update_admitted(request).await }) .await } @@ -719,7 +709,7 @@ impl LocalMachine { request: RemoveMachineRequest, ) -> Result { let local = self.clone(); - self.run_mutation(async move { local.remove_peer_admitted(request).await }) + self.finish_mutation(async move { local.remove_peer_admitted(request).await }) .await } @@ -754,7 +744,7 @@ impl LocalMachine { request: RemoveLocalMachineRequest, ) -> Result { let local = self.clone(); - self.run_mutation(async move { local.remove_local_admitted(request).await }) + self.finish_mutation(async move { local.remove_local_admitted(request).await }) .await } @@ -805,7 +795,7 @@ impl LocalMachine { /// [`Error::OperationTask`] if the admitted operation task fails. pub async fn reset(&self) -> Result { let local = self.clone(); - self.run_mutation(async move { local.reset_admitted().await }) + self.finish_mutation(async move { local.reset_admitted().await }) .await } diff --git a/crates/ployzd/src/machine/local_machine/container.rs b/crates/ployzd/src/machine/local_machine/container.rs index fbf960f6fd..671ef572e8 100644 --- a/crates/ployzd/src/machine/local_machine/container.rs +++ b/crates/ployzd/src/machine/local_machine/container.rs @@ -1,15 +1,17 @@ //! Machine-local container admission and creation. -use std::path::Path; +use std::{path::Path, sync::Arc}; use ployz_core::{ - ContainerCreated, ContainerKind, LocalMachinePhase, MachineStorageObservation, ProjectName, - ResolvedServiceSpec, + ContainerChanged, ContainerCreated, ContainerId, ContainerKind, CreateVolumeReport, + CreateVolumeRequest, DockerVolumeName, ImageIngestOpened, ImageIngestReason, ImagePulled, + LocalMachinePhase, MachineStorageObservation, ProjectName, PullImageFromMachineRequest, + ResolvedServiceSpec, VolumeRemoved, }; use super::super::ingress::admit_ingress_service; use super::{Error, LocalMachine}; -use crate::docker::{ContainerRequest, GlobalSlotConvergence, GlobalSlotRequest}; +use crate::docker::{ContainerRequest, GlobalSlotConvergence, GlobalSlotRequest, ImageIngest}; use crate::machine::{STORAGE_OBSERVATION_TIMEOUT, local_storage}; impl LocalMachine { @@ -25,7 +27,7 @@ impl LocalMachine { ) -> Result { use ployz_core::{RawVolumeSource, RpcError}; let local = self.clone(); - self.run_mutation(async move { + self.finish_mutation(async move { let containers = local.containers.as_ref().ok_or(Error::DockerUnavailable)?; let record = local.record()?; if !matches!( @@ -93,12 +95,181 @@ impl LocalMachine { ) -> Result { // Once admitted, caller cancellation must not let reset overtake a Docker request. let (local, project, spec) = (self.clone(), project.clone(), spec.clone()); - self.run_mutation( + self.finish_mutation( async move { local.create_container_admitted(kind, &project, &spec).await }, ) .await } + /// Start one local container while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot start the + /// container. + pub(crate) async fn start_container( + &self, + container_id: ContainerId, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .start(&container_id) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await + } + + /// Stop one local container while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot stop the + /// container. + pub(crate) async fn stop_container( + &self, + container_id: ContainerId, + signal: Option, + grace_period_seconds: Option, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .stop(&container_id, signal.as_deref(), grace_period_seconds) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await + } + + /// Remove one local container while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot remove the + /// container. + pub(crate) async fn remove_container( + &self, + container_id: ContainerId, + remove_volumes: bool, + force: bool, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .remove(&container_id, remove_volumes, force) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await + } + + /// Create one local Docker Volume while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when local state or Docker is unavailable, admission is busy, or Docker cannot + /// create the Volume. + pub(crate) async fn create_volume( + &self, + request: CreateVolumeRequest, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + let machine_id = local.record()?.id(); + Ok(local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .create_volume(&machine_id, request) + .await?) + }) + .await + } + + /// Remove one local Docker Volume while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot remove the + /// Volume. + pub(crate) async fn remove_volume( + &self, + name: DockerVolumeName, + force: bool, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .remove_volume(&name, force) + .await?; + Ok(VolumeRemoved {}) + }) + .await + } + + /// Start or reconcile image ingest while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when this Machine is not participating, admission is busy, or the ingest helper + /// cannot start. + pub(crate) async fn ensure_image_ingest( + &self, + ingest: Arc, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + let record = local.record()?; + let address = record + .machine() + .filter(|_| record.phase() == LocalMachinePhase::Participating) + .map(|machine| machine.management_address()) + .ok_or_else(|| { + Error::StoragePreparation( + ImageIngestReason::NotParticipating + .rpc_error("Machine is not participating"), + ) + })?; + ingest + .open(address) + .await + .map_err(Error::StoragePreparation) + }) + .await + } + + /// Pull one image from peer ingest while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or the image transfer fails. + pub(crate) async fn pull_image_from_machine( + &self, + request: PullImageFromMachineRequest, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local.containers.as_ref().ok_or(Error::DockerUnavailable)?; + crate::docker::pull_from_ingest(&request.image, request.source).await?; + Ok(ImagePulled {}) + }) + .await + } + async fn create_container_admitted( &self, kind: ContainerKind, @@ -140,8 +311,10 @@ impl LocalMachine { spec: &ResolvedServiceSpec, ) -> Result { let (local, project, spec) = (self.clone(), project.clone(), spec.clone()); - self.run_mutation(async move { local.converge_global_slot_admitted(&project, &spec).await }) - .await + self.finish_mutation( + async move { local.converge_global_slot_admitted(&project, &spec).await }, + ) + .await } async fn converge_global_slot_admitted( diff --git a/crates/ployzd/src/machine/local_machine/upgrade.rs b/crates/ployzd/src/machine/local_machine/upgrade.rs new file mode 100644 index 0000000000..68ace0d5d8 --- /dev/null +++ b/crates/ployzd/src/machine/local_machine/upgrade.rs @@ -0,0 +1,64 @@ +//! Local ownership of Machine upgrade replay, admission, and durable inspection. + +use ployz_core::{ + InspectMachineUpgradeRequest, MachineUpgradeAttempt, RequestMachineUpgradeRequest, +}; + +use super::{Error, LocalMachine}; + +impl LocalMachine { + /// Resolve, durably accept, and launch one retry-safe Machine upgrade. + /// + /// A replay is read before claiming exclusive mutation ownership so a lost reply can be + /// recovered while the independent worker owns the installation lock. + /// + /// # Errors + /// + /// Returns [`Error::Upgrade`] for a conflicting retry identity, active mutation, release + /// failure, receipt failure, or worker launch uncertainty. Returns [`Error::LockPoisoned`] + /// when the Local Machine store cannot be read. + pub(crate) async fn request_upgrade( + &self, + request: RequestMachineUpgradeRequest, + ) -> Result { + let (data_dir, run_dir, local, gate) = { + let store = self.lock_store()?; + ( + store.data_dir.clone(), + store.run_dir.clone(), + store.admission_lock.clone(), + store.mutation_gate.clone(), + ) + }; + if let Some(attempt) = crate::installer::upgrade::existing_request(&request, &data_dir)? { + return Ok(attempt); + } + let _local = local + .try_lock_owned() + .map_err(|_| crate::mutation::Error::Busy)?; + let installation = gate.try_installation()?; + crate::installer::upgrade::request_locked(request, &data_dir, &run_dir, &installation) + .await + .map_err(Into::into) + } + + /// Read one durable Machine upgrade attempt and reconcile positively stopped work. + /// + /// # Errors + /// + /// Returns [`Error::Upgrade`] when no matching receipt exists or worker and ownership + /// evidence cannot be read. Returns [`Error::LockPoisoned`] when the Local Machine store + /// cannot be read. + pub(crate) async fn inspect_upgrade( + &self, + request: InspectMachineUpgradeRequest, + ) -> Result { + let (data_dir, run_dir) = { + let store = self.lock_store()?; + (store.data_dir.clone(), store.run_dir.clone()) + }; + crate::installer::upgrade::inspect(request.attempt_id, &data_dir, &run_dir) + .await + .map_err(Into::into) + } +} diff --git a/crates/ployzd/src/machine/mod.rs b/crates/ployzd/src/machine/mod.rs index b65e7ff7f2..98ad64ea30 100644 --- a/crates/ployzd/src/machine/mod.rs +++ b/crates/ployzd/src/machine/mod.rs @@ -2,13 +2,11 @@ use std::{ collections::BTreeMap, - fs::{self, File, OpenOptions}, - io::{self, Read, Seek, SeekFrom, Write}, - net::IpAddr, - os::unix::fs::{OpenOptionsExt, PermissionsExt}, - path::{Component, Path, PathBuf}, + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + os::unix::fs::OpenOptionsExt, + path::Path, process::{Command, Stdio}, - sync::Arc, thread, time::{Duration, Instant}, }; @@ -16,20 +14,23 @@ use std::{ use ipnet::Ipv4Net; use ployz_core::{ CloudPairing, LocalMachinePhase, Machine, MachineId, MachineRuntime, MachineStorageObservation, - MachineUpdate, MachineUpdateError, SelectedEndpoint, apply_machine_update, + SelectedEndpoint, }; use serde::{Deserialize, Serialize}; -use thiserror::Error; use crate::machine_pool; use crate::network::WireGuardPrivateKey; -use crate::network::allocate_machine_subnet; mod record_wire; use record_wire::LocalMachineRecordWire; mod ingress; mod local_machine; +mod store; + +#[cfg(test)] +use store::PENDING_RESET_FILE_NAME; +pub use store::{LocalMachineStore, StoreError}; pub(crate) use local_machine::RuntimeWatchTelemetry; pub use local_machine::{Error as LocalMachineError, LocalMachine}; @@ -38,10 +39,6 @@ pub use local_machine::{Error as LocalMachineError, LocalMachine}; mod register_tests; pub const DEFAULT_DATA_DIR: &str = "/var/lib/ployz"; -const STATE_FILE_NAME: &str = "machine.json"; -const TEMPORARY_FILE_NAME: &str = ".machine.json.tmp"; -const PENDING_RESET_FILE_NAME: &str = ".machine.reset.pending"; -const LOCK_FILE_NAME: &str = ".ployzd.lock"; const DOCKER_VERSION_TIMEOUT: Duration = Duration::from_secs(2); const STORAGE_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(2); @@ -355,325 +352,6 @@ impl LocalMachineRecord { } } -pub struct LocalMachineStore { - data_dir: PathBuf, - record: LocalMachineRecord, - _lock: File, - // Lock order: admission, then publication/ingress/Docker, then short store locks. - // Reset waits for admitted operations, including Docker streams with no total deadline. - // ponytail: serialize local creates; use shared admission reads if throughput requires it. - admission_lock: Arc>, - installation_admission: crate::installer::admission::Admission, -} - -pub(crate) struct PreparedReset { - data_dir: PathBuf, - original: LocalMachineRecord, - resetting: LocalMachineRecord, -} - -impl PreparedReset { - pub(crate) fn commit(self, store: &mut LocalMachineStore) -> Result<(), StoreError> { - let mut current = store.record.clone(); - current - .selected_endpoints - .clone_from(&self.original.selected_endpoints); - if store.data_dir != self.data_dir || current != self.original { - return Err(StoreError::ResetPreparationLost(store.data_dir.clone())); - } - fs::rename( - self.data_dir.join(PENDING_RESET_FILE_NAME), - self.data_dir.join(STATE_FILE_NAME), - )?; - store.record = self.resetting.clone(); - if let Err(error) = File::open(&self.data_dir).and_then(|directory| directory.sync_all()) { - eprintln!("failed to sync committed local Machine reset: {error}"); - } - Ok(()) - } -} - -impl Drop for PreparedReset { - fn drop(&mut self) { - let _ = fs::remove_file(self.data_dir.join(PENDING_RESET_FILE_NAME)); - } -} - -impl Drop for LocalMachineStore { - fn drop(&mut self) { - // A concurrently forked child can briefly inherit the flock before exec closes the fd. - let _ = fs2::FileExt::unlock(&self._lock); - } -} - -impl LocalMachineStore { - pub fn open(data_dir: impl AsRef) -> Result { - let data_dir = data_dir.as_ref(); - Self::open_with_admission(data_dir, data_dir.join(".run")) - } - - pub(crate) fn open_with_admission( - data_dir: impl AsRef, - run_dir: impl AsRef, - ) -> Result { - let data_dir = data_dir.as_ref().to_owned(); - let run_dir = run_dir.as_ref().to_owned(); - validate_data_dir(&data_dir)?; - claim_data_dir(&data_dir)?; - fs::create_dir_all(&data_dir)?; - fs::set_permissions(&data_dir, fs::Permissions::from_mode(0o711))?; - let lock = OpenOptions::new() - .create(true) - .truncate(false) - .write(true) - .mode(0o600) - .open(data_dir.join(LOCK_FILE_NAME))?; - fs2::FileExt::try_lock_exclusive(&lock).map_err(|error| { - if error.kind() == io::ErrorKind::WouldBlock { - StoreError::AlreadyRunning(data_dir.clone()) - } else { - StoreError::Io(error) - } - })?; - let path = data_dir.join(STATE_FILE_NAME); - let record = match fs::read(&path) { - Ok(data) => serde_json::from_slice(&data)?, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - match fs::remove_file(data_dir.join(TEMPORARY_FILE_NAME)) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - let record = LocalMachineRecord { - body: LocalMachineBody::Uninitialized { - id: MachineId::random(), - }, - wireguard_private_key: WireGuardPrivateKey::generate(), - wireguard_mtu: None, - cloud_pairing: None, - selected_endpoints: BTreeMap::new(), - }; - save(&data_dir, &record)?; - record - } - Err(error) => return Err(error.into()), - }; - - let mut store = Self { - data_dir: data_dir.clone(), - record, - _lock: lock, - admission_lock: Arc::new(tokio::sync::Mutex::new(())), - installation_admission: crate::installer::admission::Admission::new( - &run_dir, &data_dir, - ), - }; - if store.record.phase() == LocalMachinePhase::Resetting { - let data_dir = store.data_dir.clone(); - store.complete_reset()?; - drop(store); - return Self::open_with_admission(data_dir, run_dir); - } - store.refresh_runtime()?; - Ok(store) - } - - #[must_use] - pub fn record(&self) -> &LocalMachineRecord { - &self.record - } - - fn refresh_runtime(&mut self) -> Result<(), StoreError> { - let runtime = local_runtime(); - let stale = match &self.record.body { - LocalMachineBody::Joining { machine, .. } - | LocalMachineBody::Participating { machine, .. } => machine.runtime != runtime, - LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { - return Ok(()); - } - }; - if !stale { - return Ok(()); - } - let mut refreshed = self.record.clone(); - match &mut refreshed.body { - LocalMachineBody::Joining { machine, .. } - | LocalMachineBody::Participating { machine, .. } => machine.runtime = runtime, - LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { - return Ok(()); - } - } - save(&self.data_dir, &refreshed)?; - self.record = refreshed; - Ok(()) - } - - pub fn begin_reset(&mut self) -> Result<(), StoreError> { - let prepared = self.prepare_reset()?; - prepared.commit(self) - } - - pub(crate) fn prepare_reset(&self) -> Result { - if self.record.phase() == LocalMachinePhase::Resetting { - return Err(StoreError::AlreadyResetting); - } - let resetting = self.record.clone().into_resetting(); - write_record(&self.data_dir.join(PENDING_RESET_FILE_NAME), &resetting)?; - File::open(&self.data_dir)?.sync_all()?; - Ok(PreparedReset { - data_dir: self.data_dir.clone(), - original: self.record.clone(), - resetting, - }) - } - - pub fn initialize( - &mut self, - name: ployz_core::MachineName, - founding_cluster: FoundingCluster, - public_ip: Option, - advertised_endpoints: Vec, - wireguard_mtu: Option, - cloud_pairing: Option, - ) -> Result { - self.require_uninitialized()?; - if advertised_endpoints.is_empty() { - return Err(StoreError::MissingEndpoints); - } - let public_key = self.record.wireguard_private_key.public_key(); - let machine = Machine { - id: self.record.id(), - name, - subnet: allocate_machine_subnet(founding_cluster.network, []) - .map_err(|error| StoreError::InvalidNetwork(error.to_string()))?, - public_key, - public_ip, - advertised_endpoints, - runtime: local_runtime(), - }; - let mut initialized = self.record.clone(); - initialized.body = LocalMachineBody::Participating { - machine: machine.clone(), - origin: ParticipationOrigin::Founder { - cluster: founding_cluster, - }, - }; - initialized.wireguard_mtu = wireguard_mtu; - initialized.cloud_pairing = cloud_pairing; - save(&self.data_dir, &initialized)?; - self.record = initialized; - Ok(machine) - } - - pub fn join( - &mut self, - mut assigned_machine: Machine, - visible_peers: Vec, - target_versions: BTreeMap, - wireguard_mtu: Option, - cloud_pairing: Option, - ) -> Result<(), StoreError> { - self.require_uninitialized()?; - if visible_peers.is_empty() { - return Err(StoreError::MissingPeers); - } - if self.record.wireguard_private_key.public_key() != assigned_machine.public_key { - return Err(StoreError::KeyMismatch); - } - if assigned_machine.advertised_endpoints.is_empty() { - return Err(StoreError::MissingEndpoints); - } - assigned_machine.runtime = local_runtime(); - let mut joining = self.record.clone(); - joining.body = LocalMachineBody::Joining { - machine: assigned_machine, - bootstrap: visible_peers, - min_store_version: target_versions, - }; - joining.wireguard_mtu = wireguard_mtu; - joining.cloud_pairing = cloud_pairing; - save(&self.data_dir, &joining)?; - self.record = joining; - Ok(()) - } - - pub fn update( - &mut self, - update: MachineUpdate, - visible: &[Machine], - ) -> Result { - let mut record = self.record.clone(); - let LocalMachineBody::Participating { machine, .. } = &mut record.body else { - return Err(StoreError::NotParticipating); - }; - let updated = apply_machine_update(machine, visible, update)?; - *machine = updated.clone(); - save(&self.data_dir, &record)?; - self.record = record; - Ok(updated) - } - - fn require_uninitialized(&self) -> Result<(), StoreError> { - if matches!(self.record.body, LocalMachineBody::Uninitialized { .. }) { - Ok(()) - } else { - Err(StoreError::AlreadyInitialized) - } - } - - pub fn complete_catch_up(&mut self) -> Result<(), StoreError> { - let mut participating = self.record.clone(); - let LocalMachineBody::Joining { - machine, bootstrap, .. - } = participating.body - else { - return Err(StoreError::NotJoining); - }; - participating.body = LocalMachineBody::Participating { - machine, - origin: ParticipationOrigin::Join { bootstrap }, - }; - save(&self.data_dir, &participating)?; - self.record = participating; - Ok(()) - } - - pub fn persist_selected_endpoint( - &mut self, - machine_id: MachineId, - endpoint: SelectedEndpoint, - ) -> Result<(), StoreError> { - let mut updated = self.record.clone(); - updated.selected_endpoints.insert(machine_id, endpoint); - save(&self.data_dir, &updated)?; - self.record = updated; - Ok(()) - } - - pub fn persist_cloud_pairing( - &mut self, - pairing: Option, - ) -> Result<(), StoreError> { - let mut updated = self.record.clone(); - updated.cloud_pairing = pairing; - save(&self.data_dir, &updated)?; - self.record = updated; - Ok(()) - } - - pub fn complete_reset(&self) -> Result<(), StoreError> { - if self.record.phase() != LocalMachinePhase::Resetting { - return Err(StoreError::NotResetting); - } - let persisted: LocalMachineRecord = - serde_json::from_slice(&fs::read(self.data_dir.join(STATE_FILE_NAME))?)?; - if persisted != self.record { - return Err(StoreError::OwnershipLost(self.data_dir.clone())); - } - fs::remove_dir_all(&self.data_dir).map_err(StoreError::Io) - } -} - #[must_use] pub fn local_runtime() -> MachineRuntime { MachineRuntime { @@ -780,113 +458,10 @@ fn read_trimmed(path: &str) -> String { .unwrap_or_default() } -fn validate_data_dir(path: &Path) -> Result<(), StoreError> { - if path.file_name().is_none() - || path - .components() - .any(|component| component == Component::ParentDir) - { - return Err(StoreError::UnsafeDataDirectory(path.to_owned())); - } - Ok(()) -} - -fn claim_data_dir(data_dir: &Path) -> Result<(), StoreError> { - match fs::read_dir(data_dir) { - Ok(entries) => { - let names = entries - .map(|entry| Ok(entry?.file_name())) - .collect::>>()?; - let owns_directory = names.iter().any(|name| name == STATE_FILE_NAME) - || names - .iter() - .all(|name| name == TEMPORARY_FILE_NAME || name == LOCK_FILE_NAME); - if owns_directory { - Ok(()) - } else { - Err(StoreError::UnownedDataDirectory(data_dir.to_owned())) - } - } - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } -} - -fn save(data_dir: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { - fs::create_dir_all(data_dir)?; - fs::set_permissions(data_dir, fs::Permissions::from_mode(0o711))?; - - let path = data_dir.join(STATE_FILE_NAME); - let temporary = data_dir.join(TEMPORARY_FILE_NAME); - write_record(&temporary, record)?; - fs::rename(temporary, path)?; - File::open(data_dir)?.sync_all()?; - Ok(()) -} - -fn write_record(path: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { - let mut file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .mode(0o600) - .open(path)?; - file.set_permissions(fs::Permissions::from_mode(0o600))?; - serde_json::to_writer_pretty(&mut file, record)?; - file.write_all(b"\n")?; - file.sync_all()?; - Ok(()) -} - -fn display_data_directory(path: &Path) -> String { - let mut text = String::new(); - for chunk in path.as_os_str().as_encoded_bytes().utf8_chunks() { - text.extend(chunk.valid().escape_debug()); - text.extend(chunk.invalid().escape_ascii().map(char::from)); - } - text -} - -#[derive(Debug, Error)] -pub enum StoreError { - #[error("local Machine record I/O failed: {0}")] - Io(#[from] io::Error), - #[error("local Machine record JSON is invalid: {0}")] - Json(#[from] serde_json::Error), - #[error("machine is not participating")] - NotParticipating, - #[error("machine is already resetting")] - AlreadyResetting, - #[error("machine is not resetting")] - NotResetting, - #[error("machine is not joining")] - NotJoining, - #[error("machine is already initialized")] - AlreadyInitialized, - #[error("at least one advertised endpoint is required")] - MissingEndpoints, - #[error("at least one visible peer is required to join")] - MissingPeers, - #[error("assigned public key does not match this Machine")] - KeyMismatch, - #[error("invalid Cluster network: {0}")] - InvalidNetwork(String), - #[error(transparent)] - MachineUpdate(#[from] MachineUpdateError), - #[error("another daemon already owns data directory {}", display_data_directory(.0))] - AlreadyRunning(PathBuf), - #[error("refusing to clear broad data directory {}", display_data_directory(.0))] - UnsafeDataDirectory(PathBuf), - #[error("refusing to claim nonempty data directory {}", display_data_directory(.0))] - UnownedDataDirectory(PathBuf), - #[error("local Machine record changed before clearing data directory {}", display_data_directory(.0))] - OwnershipLost(PathBuf), - #[error("local Machine record changed before prepared reset was committed in {}", display_data_directory(.0))] - ResetPreparationLost(PathBuf), -} - #[cfg(test)] mod tests { + use std::os::unix::fs::PermissionsExt; + use super::*; #[tokio::test] diff --git a/crates/ployzd/src/machine/register_tests/mod.rs b/crates/ployzd/src/machine/register_tests/mod.rs index c500150c05..f88ca34736 100644 --- a/crates/ployzd/src/machine/register_tests/mod.rs +++ b/crates/ployzd/src/machine/register_tests/mod.rs @@ -143,6 +143,7 @@ async fn register_rpc_exact_replay_returns_the_original_joinable_assignment() { wireguard_mtu: None, cloud_pairing: None, }) + .await .unwrap(); assert_eq!(target.record().unwrap().phase(), LocalMachinePhase::Joining); @@ -184,6 +185,7 @@ async fn register_does_not_reconstruct_membership_while_joining() { wireguard_mtu: None, cloud_pairing: None, }) + .await .unwrap(); assert_eq!(joiner.record().unwrap().phase(), LocalMachinePhase::Joining); diff --git a/crates/ployzd/src/machine/store.rs b/crates/ployzd/src/machine/store.rs new file mode 100644 index 0000000000..bc673cefe2 --- /dev/null +++ b/crates/ployzd/src/machine/store.rs @@ -0,0 +1,451 @@ +//! Durable Local Machine record ownership and persistence. + +use std::{ + collections::BTreeMap, + fs::{self, File, OpenOptions}, + io::{self, Write}, + net::IpAddr, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Component, Path, PathBuf}, + sync::Arc, +}; + +use ployz_core::{ + CloudPairing, LocalMachinePhase, Machine, MachineId, MachineUpdate, MachineUpdateError, + SelectedEndpoint, apply_machine_update, +}; +use thiserror::Error; + +use super::{ + FoundingCluster, LocalMachineBody, LocalMachineRecord, ParticipationOrigin, local_runtime, +}; +use crate::network::{WireGuardPrivateKey, allocate_machine_subnet}; + +const STATE_FILE_NAME: &str = "machine.json"; +const TEMPORARY_FILE_NAME: &str = ".machine.json.tmp"; +pub(super) const PENDING_RESET_FILE_NAME: &str = ".machine.reset.pending"; +const LOCK_FILE_NAME: &str = ".ployzd.lock"; + +pub struct LocalMachineStore { + pub(super) data_dir: PathBuf, + pub(super) run_dir: PathBuf, + record: LocalMachineRecord, + _lock: File, + // Lock order: admission, then publication/ingress/Docker, then short store locks. + // Reset waits for admitted operations, including Docker streams with no total deadline. + // ponytail: serialize local creates; use shared admission reads if throughput requires it. + pub(super) admission_lock: Arc>, + pub(super) mutation_gate: crate::mutation::MutationGate, +} + +pub(crate) struct PreparedReset { + data_dir: PathBuf, + original: LocalMachineRecord, + resetting: LocalMachineRecord, +} + +impl PreparedReset { + pub(crate) fn commit(self, store: &mut LocalMachineStore) -> Result<(), StoreError> { + let mut current = store.record.clone(); + current + .selected_endpoints + .clone_from(&self.original.selected_endpoints); + if store.data_dir != self.data_dir || current != self.original { + return Err(StoreError::ResetPreparationLost(store.data_dir.clone())); + } + fs::rename( + self.data_dir.join(PENDING_RESET_FILE_NAME), + self.data_dir.join(STATE_FILE_NAME), + )?; + store.record = self.resetting.clone(); + if let Err(error) = File::open(&self.data_dir).and_then(|directory| directory.sync_all()) { + eprintln!("failed to sync committed local Machine reset: {error}"); + } + Ok(()) + } +} + +impl Drop for PreparedReset { + fn drop(&mut self) { + let _ = fs::remove_file(self.data_dir.join(PENDING_RESET_FILE_NAME)); + } +} + +impl Drop for LocalMachineStore { + fn drop(&mut self) { + // A concurrently forked child can briefly inherit the flock before exec closes the fd. + let _ = fs2::FileExt::unlock(&self._lock); + } +} + +impl LocalMachineStore { + pub fn open(data_dir: impl AsRef) -> Result { + let data_dir = data_dir.as_ref(); + Self::open_with_admission(data_dir, data_dir.join(".run")) + } + + pub(crate) fn open_with_admission( + data_dir: impl AsRef, + run_dir: impl AsRef, + ) -> Result { + let data_dir = data_dir.as_ref().to_owned(); + let run_dir = run_dir.as_ref().to_owned(); + validate_data_dir(&data_dir)?; + claim_data_dir(&data_dir)?; + fs::create_dir_all(&data_dir)?; + fs::set_permissions(&data_dir, fs::Permissions::from_mode(0o711))?; + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .mode(0o600) + .open(data_dir.join(LOCK_FILE_NAME))?; + fs2::FileExt::try_lock_exclusive(&lock).map_err(|error| { + if error.kind() == io::ErrorKind::WouldBlock { + StoreError::AlreadyRunning(data_dir.clone()) + } else { + StoreError::Io(error) + } + })?; + let path = data_dir.join(STATE_FILE_NAME); + let record = match fs::read(&path) { + Ok(data) => serde_json::from_slice(&data)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + match fs::remove_file(data_dir.join(TEMPORARY_FILE_NAME)) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let record = LocalMachineRecord { + body: LocalMachineBody::Uninitialized { + id: MachineId::random(), + }, + wireguard_private_key: WireGuardPrivateKey::generate(), + wireguard_mtu: None, + cloud_pairing: None, + selected_endpoints: BTreeMap::new(), + }; + save(&data_dir, &record)?; + record + } + Err(error) => return Err(error.into()), + }; + + let mut store = Self { + data_dir: data_dir.clone(), + record, + _lock: lock, + admission_lock: Arc::new(tokio::sync::Mutex::new(())), + run_dir: run_dir.clone(), + mutation_gate: crate::mutation::MutationGate::new(&run_dir, &data_dir), + }; + if store.record.phase() == LocalMachinePhase::Resetting { + let data_dir = store.data_dir.clone(); + store.complete_reset()?; + drop(store); + return Self::open_with_admission(data_dir, run_dir); + } + store.refresh_runtime()?; + Ok(store) + } + + #[must_use] + pub fn record(&self) -> &LocalMachineRecord { + &self.record + } + + fn refresh_runtime(&mut self) -> Result<(), StoreError> { + let runtime = local_runtime(); + let stale = match &self.record.body { + LocalMachineBody::Joining { machine, .. } + | LocalMachineBody::Participating { machine, .. } => machine.runtime != runtime, + LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { + return Ok(()); + } + }; + if !stale { + return Ok(()); + } + let mut refreshed = self.record.clone(); + match &mut refreshed.body { + LocalMachineBody::Joining { machine, .. } + | LocalMachineBody::Participating { machine, .. } => machine.runtime = runtime, + LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { + return Ok(()); + } + } + save(&self.data_dir, &refreshed)?; + self.record = refreshed; + Ok(()) + } + + pub fn begin_reset(&mut self) -> Result<(), StoreError> { + let prepared = self.prepare_reset()?; + prepared.commit(self) + } + + pub(crate) fn prepare_reset(&self) -> Result { + if self.record.phase() == LocalMachinePhase::Resetting { + return Err(StoreError::AlreadyResetting); + } + let resetting = self.record.clone().into_resetting(); + write_record(&self.data_dir.join(PENDING_RESET_FILE_NAME), &resetting)?; + File::open(&self.data_dir)?.sync_all()?; + Ok(PreparedReset { + data_dir: self.data_dir.clone(), + original: self.record.clone(), + resetting, + }) + } + + pub fn initialize( + &mut self, + name: ployz_core::MachineName, + founding_cluster: FoundingCluster, + public_ip: Option, + advertised_endpoints: Vec, + wireguard_mtu: Option, + cloud_pairing: Option, + ) -> Result { + self.require_uninitialized()?; + if advertised_endpoints.is_empty() { + return Err(StoreError::MissingEndpoints); + } + let public_key = self.record.wireguard_private_key.public_key(); + let machine = Machine { + id: self.record.id(), + name, + subnet: allocate_machine_subnet(founding_cluster.network, []) + .map_err(|error| StoreError::InvalidNetwork(error.to_string()))?, + public_key, + public_ip, + advertised_endpoints, + runtime: local_runtime(), + }; + let mut initialized = self.record.clone(); + initialized.body = LocalMachineBody::Participating { + machine: machine.clone(), + origin: ParticipationOrigin::Founder { + cluster: founding_cluster, + }, + }; + initialized.wireguard_mtu = wireguard_mtu; + initialized.cloud_pairing = cloud_pairing; + save(&self.data_dir, &initialized)?; + self.record = initialized; + Ok(machine) + } + + pub fn join( + &mut self, + mut assigned_machine: Machine, + visible_peers: Vec, + target_versions: BTreeMap, + wireguard_mtu: Option, + cloud_pairing: Option, + ) -> Result<(), StoreError> { + self.require_uninitialized()?; + if visible_peers.is_empty() { + return Err(StoreError::MissingPeers); + } + if self.record.wireguard_private_key.public_key() != assigned_machine.public_key { + return Err(StoreError::KeyMismatch); + } + if assigned_machine.advertised_endpoints.is_empty() { + return Err(StoreError::MissingEndpoints); + } + assigned_machine.runtime = local_runtime(); + let mut joining = self.record.clone(); + joining.body = LocalMachineBody::Joining { + machine: assigned_machine, + bootstrap: visible_peers, + min_store_version: target_versions, + }; + joining.wireguard_mtu = wireguard_mtu; + joining.cloud_pairing = cloud_pairing; + save(&self.data_dir, &joining)?; + self.record = joining; + Ok(()) + } + + pub fn update( + &mut self, + update: MachineUpdate, + visible: &[Machine], + ) -> Result { + let mut record = self.record.clone(); + let LocalMachineBody::Participating { machine, .. } = &mut record.body else { + return Err(StoreError::NotParticipating); + }; + let updated = apply_machine_update(machine, visible, update)?; + *machine = updated.clone(); + save(&self.data_dir, &record)?; + self.record = record; + Ok(updated) + } + + fn require_uninitialized(&self) -> Result<(), StoreError> { + if matches!(self.record.body, LocalMachineBody::Uninitialized { .. }) { + Ok(()) + } else { + Err(StoreError::AlreadyInitialized) + } + } + + pub fn complete_catch_up(&mut self) -> Result<(), StoreError> { + let mut participating = self.record.clone(); + let LocalMachineBody::Joining { + machine, bootstrap, .. + } = participating.body + else { + return Err(StoreError::NotJoining); + }; + participating.body = LocalMachineBody::Participating { + machine, + origin: ParticipationOrigin::Join { bootstrap }, + }; + save(&self.data_dir, &participating)?; + self.record = participating; + Ok(()) + } + + pub fn persist_selected_endpoint( + &mut self, + machine_id: MachineId, + endpoint: SelectedEndpoint, + ) -> Result<(), StoreError> { + let mut updated = self.record.clone(); + updated.selected_endpoints.insert(machine_id, endpoint); + save(&self.data_dir, &updated)?; + self.record = updated; + Ok(()) + } + + pub fn persist_cloud_pairing( + &mut self, + pairing: Option, + ) -> Result<(), StoreError> { + let mut updated = self.record.clone(); + updated.cloud_pairing = pairing; + save(&self.data_dir, &updated)?; + self.record = updated; + Ok(()) + } + + pub fn complete_reset(&self) -> Result<(), StoreError> { + if self.record.phase() != LocalMachinePhase::Resetting { + return Err(StoreError::NotResetting); + } + let persisted: LocalMachineRecord = + serde_json::from_slice(&fs::read(self.data_dir.join(STATE_FILE_NAME))?)?; + if persisted != self.record { + return Err(StoreError::OwnershipLost(self.data_dir.clone())); + } + fs::remove_dir_all(&self.data_dir).map_err(StoreError::Io) + } +} + +fn validate_data_dir(path: &Path) -> Result<(), StoreError> { + if path.file_name().is_none() + || path + .components() + .any(|component| component == Component::ParentDir) + { + return Err(StoreError::UnsafeDataDirectory(path.to_owned())); + } + Ok(()) +} + +fn claim_data_dir(data_dir: &Path) -> Result<(), StoreError> { + match fs::read_dir(data_dir) { + Ok(entries) => { + let names = entries + .map(|entry| Ok(entry?.file_name())) + .collect::>>()?; + let owns_directory = names.iter().any(|name| name == STATE_FILE_NAME) + || names + .iter() + .all(|name| name == TEMPORARY_FILE_NAME || name == LOCK_FILE_NAME); + if owns_directory { + Ok(()) + } else { + Err(StoreError::UnownedDataDirectory(data_dir.to_owned())) + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn save(data_dir: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { + fs::create_dir_all(data_dir)?; + fs::set_permissions(data_dir, fs::Permissions::from_mode(0o711))?; + + let path = data_dir.join(STATE_FILE_NAME); + let temporary = data_dir.join(TEMPORARY_FILE_NAME); + write_record(&temporary, record)?; + fs::rename(temporary, path)?; + File::open(data_dir)?.sync_all()?; + Ok(()) +} + +fn write_record(path: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path)?; + file.set_permissions(fs::Permissions::from_mode(0o600))?; + serde_json::to_writer_pretty(&mut file, record)?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) +} + +fn display_data_directory(path: &Path) -> String { + let mut text = String::new(); + for chunk in path.as_os_str().as_encoded_bytes().utf8_chunks() { + text.extend(chunk.valid().escape_debug()); + text.extend(chunk.invalid().escape_ascii().map(char::from)); + } + text +} + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("local Machine record I/O failed: {0}")] + Io(#[from] io::Error), + #[error("local Machine record JSON is invalid: {0}")] + Json(#[from] serde_json::Error), + #[error("machine is not participating")] + NotParticipating, + #[error("machine is already resetting")] + AlreadyResetting, + #[error("machine is not resetting")] + NotResetting, + #[error("machine is not joining")] + NotJoining, + #[error("machine is already initialized")] + AlreadyInitialized, + #[error("at least one advertised endpoint is required")] + MissingEndpoints, + #[error("at least one visible peer is required to join")] + MissingPeers, + #[error("assigned public key does not match this Machine")] + KeyMismatch, + #[error("invalid Cluster network: {0}")] + InvalidNetwork(String), + #[error(transparent)] + MachineUpdate(#[from] MachineUpdateError), + #[error("another daemon already owns data directory {}", display_data_directory(.0))] + AlreadyRunning(PathBuf), + #[error("refusing to clear broad data directory {}", display_data_directory(.0))] + UnsafeDataDirectory(PathBuf), + #[error("refusing to claim nonempty data directory {}", display_data_directory(.0))] + UnownedDataDirectory(PathBuf), + #[error("local Machine record changed before clearing data directory {}", display_data_directory(.0))] + OwnershipLost(PathBuf), + #[error("local Machine record changed before prepared reset was committed in {}", display_data_directory(.0))] + ResetPreparationLost(PathBuf), +} diff --git a/crates/ployzd/src/machine_api/local.rs b/crates/ployzd/src/machine_api/local.rs index a9bd3bf333..7ea5db61de 100644 --- a/crates/ployzd/src/machine_api/local.rs +++ b/crates/ployzd/src/machine_api/local.rs @@ -8,11 +8,10 @@ use std::{ }; use ployz_core::{ - CapabilityAdvertisement, CloudPairing, CloudPairingSet, ContainerChanged, ContainerList, - ContainerObservationMap, ContractDescription, Domain, DomainRecords, ImageIngestReason, - ImagePulled, IngressProxyConfig, LocalMachinePhase, LogMetadata, LogOrigin, MachineId, - MachineLogService, MachineRpc, MachineRpcClient, OpaquePayload, PROTOCOL_MAJOR, Rpc, RpcError, - RpcErrorCode, RpcRequestBody, RpcResponse, VolumeRemoved, op, + CapabilityAdvertisement, CloudPairing, CloudPairingSet, ContainerList, ContainerObservationMap, + ContractDescription, Domain, DomainRecords, IngressProxyConfig, LocalMachinePhase, LogMetadata, + LogOrigin, MachineId, MachineLogService, MachineRpc, MachineRpcClient, OpaquePayload, + PROTOCOL_MAJOR, Rpc, RpcError, RpcErrorCode, RpcRequestBody, RpcResponse, op, }; use serde_json::Value; use tokio::{sync::watch, time::Instant}; @@ -304,7 +303,7 @@ impl MachineRpc for MachineService { ) -> Result, Status> { finish( self.local - .initialize_mutation(expect::(request)?) + .initialize(expect::(request)?) .await, ) } @@ -335,7 +334,7 @@ impl MachineRpc for MachineService { &self, request: Request, ) -> Result, Status> { - finish(self.local.join_mutation(expect::(request)?).await) + finish(self.local.join(expect::(request)?).await) } async fn set_cloud_pairing( @@ -345,7 +344,7 @@ impl MachineRpc for MachineService { let request = expect::(request)?; if let Err(error) = self .local - .set_cloud_pairing_mutation(request.cloud_pairing.clone()) + .set_cloud_pairing(request.cloud_pairing.clone()) .await { return local_error(error); @@ -455,19 +454,7 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers.clone(), - Err(error) => return respond(error), - }; - let container_id = request.container_id; - finish( - self.local - .run_mutation(async move { - containers.start(&container_id).await?; - Ok(ContainerChanged { container_id }) - }) - .await, - ) + finish(self.local.start_container(request.container_id).await) } async fn stop_container( @@ -475,23 +462,13 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers.clone(), - Err(error) => return respond(error), - }; - let container_id = request.container_id; finish( self.local - .run_mutation(async move { - containers - .stop( - &container_id, - request.signal.as_deref(), - request.grace_period_seconds, - ) - .await?; - Ok(ContainerChanged { container_id }) - }) + .stop_container( + request.container_id, + request.signal, + request.grace_period_seconds, + ) .await, ) } @@ -501,19 +478,9 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers.clone(), - Err(error) => return respond(error), - }; - let container_id = request.container_id; finish( self.local - .run_mutation(async move { - containers - .remove(&container_id, request.remove_volumes, request.force) - .await?; - Ok(ContainerChanged { container_id }) - }) + .remove_container(request.container_id, request.remove_volumes, request.force) .await, ) } @@ -523,18 +490,7 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let machine_id = self.local_record()?.id(); - let containers = match self.containers() { - Ok(containers) => containers.clone(), - Err(error) => return respond(error), - }; - finish( - self.local - .run_mutation( - async move { Ok(containers.create_volume(&machine_id, request).await?) }, - ) - .await, - ) + finish(self.local.create_volume(request).await) } async fn inspect_storage( @@ -593,20 +549,7 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers.clone(), - Err(error) => return respond(error), - }; - finish( - self.local - .run_mutation(async move { - containers - .remove_volume(&request.name, request.force) - .await?; - Ok(VolumeRemoved {}) - }) - .await, - ) + finish(self.local.remove_volume(request.name, request.force).await) } async fn exec( @@ -708,48 +651,22 @@ impl MachineRpc for MachineService { &self, request: Request, ) -> Result, Status> { - let request = expect::(request)?; - let (data_dir, run_dir) = match self.local.upgrade_paths() { - Ok(paths) => paths, - Err(error) => return local_error(error), - }; - match crate::installer::upgrade::existing_request(&request, &data_dir) { - Ok(Some(attempt)) => return respond(attempt), - Ok(None) => {} - Err(error) => return respond(upgrade_error(error)), - } - let admission = match self.local.try_upgrade_admission() { - Ok(admission) => admission, - Err(error) => return local_error(error), - }; - debug_assert_eq!(admission.data_dir, data_dir); - debug_assert_eq!(admission.run_dir, run_dir); - match crate::installer::upgrade::request_locked( - request, - &data_dir, - &run_dir, - &admission.install, + finish( + self.local + .request_upgrade(expect::(request)?) + .await, ) - .await - { - Ok(attempt) => respond(attempt), - Err(error) => respond(upgrade_error(error)), - } } async fn inspect_machine_upgrade( &self, request: Request, ) -> Result, Status> { - let request = expect::(request)?; - let (data_dir, run_dir) = match self.local.upgrade_paths() { - Ok(paths) => paths, - Err(error) => return local_error(error), - }; - match crate::installer::upgrade::inspect(request.attempt_id, &data_dir, &run_dir).await { - Ok(attempt) => respond(attempt), - Err(error) => respond(upgrade_error(error)), - } + finish( + self.local + .inspect_upgrade(expect::(request)?) + .await, + ) } async fn remove_local_machine( @@ -803,25 +720,9 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { expect::(request)?; - let record = self.local_record()?; - let Some(machine) = record - .machine() - .filter(|_| record.phase() == LocalMachinePhase::Participating) - else { - return respond( - ImageIngestReason::NotParticipating.rpc_error("Machine is not participating"), - ); - }; - let address = machine.management_address(); - let ingest = Arc::clone(&self.ingest); finish( self.local - .run_mutation(async move { - ingest - .open(address) - .await - .map_err(LocalMachineError::StoragePreparation) - }) + .ensure_image_ingest(Arc::clone(&self.ingest)) .await, ) } @@ -838,17 +739,7 @@ impl MachineRpc for MachineService { details: Value::Null, }); } - if self.containers().is_err() { - return respond(unavailable("Docker is not available")); - } - finish( - self.local - .run_mutation(async move { - crate::docker::pull_from_ingest(&request.image, request.source).await?; - Ok(ImagePulled {}) - }) - .await, - ) + finish(self.local.pull_image_from_machine(request).await) } async fn get_ingress_proxy_config( @@ -1039,24 +930,23 @@ fn local_error(error: LocalMachineError) -> Result, Stat LocalMachineError::AllocatorNotQuiet | LocalMachineError::NotAllocator | LocalMachineError::IsolationLocked => respond(unavailable(&error.to_string())), - LocalMachineError::Admission(crate::installer::admission::Error::Busy) => { - respond(RpcError { - code: RpcErrorCode::Conflict, - message: "a Ployz installation or upgrade is active".into(), - details: Value::Null, - }) - } - LocalMachineError::Admission(crate::installer::admission::Error::Io(error)) => { + LocalMachineError::Admission(crate::mutation::Error::Busy) => respond(RpcError { + code: RpcErrorCode::Conflict, + message: "a Ployz installation or upgrade is active".into(), + details: Value::Null, + }), + LocalMachineError::Admission(crate::mutation::Error::Io(error)) => { Err(Status::internal(error.to_string())) } + LocalMachineError::Upgrade(error) => respond(upgrade_error(error)), } } fn upgrade_error(error: crate::installer::upgrade::Error) -> RpcError { - use crate::installer::{admission, upgrade::Error}; + use crate::{installer::upgrade::Error, mutation}; let code = match &error { Error::NotFound => RpcErrorCode::NotFound, - Error::AttemptConflict(_) | Error::Busy | Error::Admission(admission::Error::Busy) => { + Error::AttemptConflict(_) | Error::Busy | Error::Admission(mutation::Error::Busy) => { RpcErrorCode::Conflict } Error::Resolve(_) => RpcErrorCode::InvalidArgument, @@ -1067,9 +957,10 @@ fn upgrade_error(error: crate::installer::upgrade::Error) -> RpcError { | Error::QualificationSource(_) | Error::Launch(_) | Error::InspectWorker(_) + | Error::WorkerEvidence(_) | Error::NotActive(_) | Error::Installation(_) - | Error::Admission(admission::Error::Io(_)) => RpcErrorCode::Internal, + | Error::Admission(mutation::Error::Io(_)) => RpcErrorCode::Internal, }; RpcError { code, diff --git a/crates/ployzd/src/machine_api/local_tests.rs b/crates/ployzd/src/machine_api/local_tests.rs index 1876c0646f..cf17d89a3e 100644 --- a/crates/ployzd/src/machine_api/local_tests.rs +++ b/crates/ployzd/src/machine_api/local_tests.rs @@ -116,15 +116,7 @@ async fn upgrade_and_machine_mutations_refuse_each_other_at_the_rpc_boundary() { let local = service.local(); let started = Arc::clone(&started); let release = Arc::clone(&release); - async move { - local - .run_mutation(async move { - started.notify_one(); - release.notified().await; - Ok(()) - }) - .await - } + async move { local.hold_mutation_for_test(started, release).await } }); started.notified().await; diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index 9c36a6121b..86bf2cbc51 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -154,7 +154,7 @@ async fn run(args: Args) -> Result<(), Error> { release_dir, ) .map_err(Error::from)?; - let outcome = ployzd::installer::install_in(request, &args.data_dir, &run_dir) + let outcome = ployzd::installer::install(request, &args.data_dir, &run_dir) .await .map_err(io::Error::other)?; match outcome.readiness { diff --git a/crates/ployzd/src/mutation.rs b/crates/ployzd/src/mutation.rs new file mode 100644 index 0000000000..0f858b5d93 --- /dev/null +++ b/crates/ployzd/src/mutation.rs @@ -0,0 +1,205 @@ +//! Cross-process exclusion between Machine installation and local mutation. + +use std::{ + fs::{self, File, OpenOptions, TryLockError}, + io, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::PathBuf, +}; + +use thiserror::Error; + +const LOCK_FILE: &str = ".install.lock"; +const ACTIVE_FILE: &str = ".upgrade-active"; + +/// Shared Machine mutation policy backed by an OS lock and durable upgrade evidence. +#[derive(Clone, Debug)] +pub struct MutationGate { + lock_dir: PathBuf, + data_dir: PathBuf, +} + +/// A refusal or local I/O failure while claiming Machine mutation ownership. +#[derive(Debug, Error)] +pub enum Error { + /// Another mutation, installation, or durable upgrade attempt owns the Machine. + #[error("a Ployz installation or upgrade is active")] + Busy, + /// Local ownership evidence could not be inspected or changed. + #[error("Machine mutation admission: {0}")] + Io(#[from] io::Error), +} + +/// Shared ownership that excludes Machine installation while a mutation finishes. +pub struct MutationGuard(File); + +/// Exclusive ownership that excludes every other Machine installation or mutation. +pub struct InstallationGuard(File); + +impl MutationGate { + /// Bind one gate to its runtime lock and durable data directories. + #[must_use] + pub fn new(lock_dir: impl Into, data_dir: impl Into) -> Self { + Self { + lock_dir: lock_dir.into(), + data_dir: data_dir.into(), + } + } + + /// Claim shared mutation ownership when no installation evidence exists. + /// + /// # Errors + /// + /// Returns [`Error::Busy`] when an installation lock or any durable active-marker entry + /// exists. Returns [`Error::Io`] when either ownership source cannot be inspected. + pub fn try_mutation(&self) -> Result { + let file = self.open_lock()?; + file.try_lock_shared().map_err(lock_error)?; + match fs::symlink_metadata(self.active_path()) { + Ok(_) => { + let _ = file.unlock(); + Err(Error::Busy) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(MutationGuard(file)), + Err(error) => { + let _ = file.unlock(); + Err(Error::Io(error)) + } + } + } + + /// Try to claim exclusive installation ownership without waiting. + /// + /// # Errors + /// + /// Returns [`Error::Busy`] when another owner holds the lock and [`Error::Io`] when the + /// lock directory or file cannot be prepared. + pub fn try_installation(&self) -> Result { + let file = self.open_lock()?; + file.try_lock().map_err(lock_error)?; + Ok(InstallationGuard(file)) + } + + /// Wait for exclusive installation ownership. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the lock directory or file cannot be prepared or locked. + pub fn lock_installation(&self) -> Result { + let file = self.open_lock()?; + file.lock()?; + Ok(InstallationGuard(file)) + } + + /// Whether any filesystem entry exists at the durable active-marker path. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when absence cannot be established from filesystem metadata. + pub fn active(&self) -> Result { + match fs::symlink_metadata(self.active_path()) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(Error::Io(error)), + } + } + + pub(crate) fn mark_active(&self, attempt_id: &str) -> Result<(), Error> { + fs::create_dir_all(&self.data_dir)?; + crate::filesystem::atomic_write(&self.active_path(), attempt_id.as_bytes(), 0o600)?; + Ok(()) + } + + pub(crate) fn clear_active(&self, attempt_id: &str) -> Result<(), Error> { + let path = self.active_path(); + match fs::read_to_string(&path) { + Ok(active) if active == attempt_id => fs::remove_file(path)?, + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + } + File::open(&self.data_dir)?.sync_all()?; + Ok(()) + } + + fn open_lock(&self) -> Result { + fs::create_dir_all(&self.lock_dir)?; + fs::set_permissions(&self.lock_dir, fs::Permissions::from_mode(0o750))?; + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .mode(0o600) + .open(self.lock_dir.join(LOCK_FILE)) + .map_err(Into::into) + } + + fn active_path(&self) -> PathBuf { + self.data_dir.join(ACTIVE_FILE) + } +} + +fn lock_error(error: TryLockError) -> Error { + match error { + TryLockError::WouldBlock => Error::Busy, + TryLockError::Error(error) => Error::Io(error), + } +} + +impl Drop for MutationGuard { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +impl Drop for InstallationGuard { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installation_and_active_handoff_exclude_mutations_across_instances() { + let root = crate::test_dir::TestDir::new("mutation-gate"); + let first = MutationGate::new(root.0.join("run"), root.0.join("data")); + let second = MutationGate::new(root.0.join("run"), root.0.join("data")); + + let install = first.try_installation().unwrap(); + assert!(matches!(second.try_mutation(), Err(Error::Busy))); + first.mark_active("attempt").unwrap(); + drop(install); + assert!(matches!(second.try_mutation(), Err(Error::Busy))); + + let install = second.try_installation().unwrap(); + second.clear_active("attempt").unwrap(); + drop(install); + assert!(second.try_mutation().is_ok()); + } + + #[test] + fn every_existing_marker_type_fails_closed() { + let root = crate::test_dir::TestDir::new("mutation-marker-type"); + let gate = MutationGate::new(root.0.join("run"), root.0.join("data")); + fs::create_dir_all(gate.active_path()).unwrap(); + + assert!(matches!(gate.try_mutation(), Err(Error::Busy))); + assert!(gate.active().unwrap()); + } + + #[test] + fn marker_metadata_errors_are_not_absence() { + let root = crate::test_dir::TestDir::new("mutation-marker-error"); + fs::create_dir_all(&root.0).unwrap(); + let data = root.0.join("data-is-a-file"); + fs::write(&data, "not a directory").unwrap(); + let gate = MutationGate::new(root.0.join("run"), data); + + assert!(matches!(gate.try_mutation(), Err(Error::Io(_)))); + assert!(matches!(gate.active(), Err(Error::Io(_)))); + } +} diff --git a/crates/ployzd/src/volume_plugin.rs b/crates/ployzd/src/volume_plugin.rs index f99069ee4f..c941cd24bc 100644 --- a/crates/ployzd/src/volume_plugin.rs +++ b/crates/ployzd/src/volume_plugin.rs @@ -1,8 +1,7 @@ //! Docker v1 Volume plugin backed by bounded ZFS datasets. use std::{ - collections::BTreeMap, fmt, io, os::unix::net::UnixListener as StdUnixListener, path::PathBuf, - str::FromStr, sync::Arc, + collections::BTreeMap, fmt, io, os::unix::net::UnixListener as StdUnixListener, str::FromStr, }; use axum::{ @@ -13,22 +12,19 @@ use axum::{ response::Response, routing::post, }; -use ployzd::machine_pool::MachinePool; use serde::{Deserialize, Serialize}; -use tokio::{ - net::UnixListener, - process::Command, - sync::{Mutex, OwnedMutexGuard}, -}; +use tokio::net::UnixListener; mod capacity; mod pool; mod removal; +mod storage; -use pool::PoolStorage; +use storage::{ + CapacityAdmission, DATASET_ROOT, Dataset, MOUNT_ROOT, VolumeStorage, checked_command, + parse_size, +}; -const DATASET_ROOT: &str = "ployz"; -const MOUNT_ROOT: &str = "/var/lib/ployz-volumes"; type Result = std::result::Result; #[derive(Debug, thiserror::Error)] @@ -81,371 +77,6 @@ impl DockerVolumeName { } } -#[derive(Clone)] -struct VolumeStorage { - pool: PoolStorage, - zfs: PathBuf, - mutation: Arc>, - installation: ployzd::installer::admission::Admission, -} - -enum CapacityAdmission { - Required, - Ensured, -} - -impl VolumeStorage { - fn new(data_dir: impl Into, run_dir: impl Into) -> Self { - Self { - pool: PoolStorage::new("zpool"), - zfs: "zfs".into(), - mutation: Arc::new(Mutex::new(())), - installation: ployzd::installer::admission::Admission::new(run_dir, data_dir), - } - } - - #[cfg(test)] - fn with_programs(zpool: impl Into, zfs: impl Into) -> Self { - let zpool = zpool.into(); - let backing = zpool.with_file_name("machine-pool"); - let fixture = zpool - .parent() - .expect("test command has a fixture directory") - .to_owned(); - Self { - pool: PoolStorage::new(zpool).with_backing(backing), - zfs: zfs.into(), - mutation: Arc::new(Mutex::new(())), - installation: ployzd::installer::admission::Admission::new( - fixture.join("admission-run"), - fixture.join("admission-data"), - ), - } - } - - async fn admit_mutation( - &self, - ) -> Result<( - OwnedMutexGuard<()>, - ployzd::installer::admission::MutationGuard, - )> { - let local = Arc::clone(&self.mutation).lock_owned().await; - let installation = self - .installation - .try_mutation() - .map_err(|error| VolumeError::from(error.to_string()))?; - Ok((local, installation)) - } - - async fn create_volume( - &self, - pool: &MachinePool, - name: &DockerVolumeName, - requested: u64, - origin: CapacityAdmission, - ) -> Result<()> { - let datasets = self.datasets(pool).await?; - let root = format!("{}/{DATASET_ROOT}", pool.name()); - let volume = format!("{root}/{name}"); - - if let Some(existing) = Self::dataset(&datasets, pool, name)? { - existing.require_mountpoint(&name.mountpoint())?; - existing.require_writable()?; - return if existing.refquota == requested { - Ok(()) - } else { - Err(format!( - "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", - existing.refquota - ) - .into()) - }; - } - - if matches!(origin, CapacityAdmission::Required) { - let commitment = datasets - .iter() - .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) - .map(|dataset| dataset.refquota) - .try_fold(requested, u64::checked_add) - .ok_or_else(|| { - VolumeError::from("Provisioned Volume commitments overflowed u64") - })?; - let managed_used = datasets - .iter() - .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) - .map(|dataset| dataset.active_used_bytes) - .try_fold(0u64, u64::checked_add) - .ok_or("Dataset occupancy overflows u64")?; - self.pool - .ensure_capacity( - pool, - commitment, - pool.used_bytes().saturating_sub(managed_used), - ) - .await?; - } - - if !datasets.iter().any(|dataset| dataset.name == root) { - self.zfs(&[ - "create", - "-o", - "canmount=off", - "-o", - &format!("mountpoint={MOUNT_ROOT}"), - &root, - ]) - .await?; - } - self.zfs(&["create", "-o", &format!("refquota={requested}"), &volume]) - .await?; - Ok(()) - } - - async fn mountpoint(&self, name: &DockerVolumeName) -> Result { - let _guard = self.admit_mutation().await?; - let pool = self.one_pool().await?; - let datasets = self.datasets(&pool).await?; - let dataset = Self::dataset(&datasets, &pool, name)? - .ok_or_else(|| format!("Provisioned Volume {name} does not exist"))?; - dataset.require_provisioned(name)?; - dataset.require_writable()?; - if dataset.mounted { - return Ok(dataset.mountpoint.clone()); - } - self.zfs(&["mount", &dataset.name]).await?; - let datasets = self.datasets(&pool).await?; - let dataset = Self::dataset(&datasets, &pool, name)? - .ok_or_else(|| format!("Provisioned Volume {name} disappeared while mounting"))?; - dataset.require_provisioned(name)?; - dataset.require_writable()?; - if !dataset.mounted { - return Err(format!("Provisioned Volume {name} did not mount").into()); - } - Ok(dataset.mountpoint.clone()) - } - - async fn one_pool(&self) -> Result { - self.pool - .one_usable() - .await? - .ok_or_else(|| "no usable existing Machine Pool".into()) - } - - async fn datasets(&self, pool: &MachinePool) -> Result> { - let output = self - .zfs(&[ - "list", - "-Hp", - "-o", - "name,refquota,used,usedbydataset,mountpoint,mounted,readonly", - "-r", - pool.name(), - ]) - .await?; - output - .lines() - .map(|line| Dataset::parse(line, pool.name())) - .collect() - } - - fn dataset<'datasets>( - datasets: &'datasets [Dataset], - pool: &MachinePool, - name: &DockerVolumeName, - ) -> Result> { - let root_name = format!("{}/{DATASET_ROOT}", pool.name()); - let root = datasets.iter().find(|dataset| dataset.name == root_name); - if let Some(root) = root { - root.require_mountpoint(MOUNT_ROOT)?; - root.require_writable()?; - } - - let requested = format!("{}/{DATASET_ROOT}/{name}", pool.name()); - let descendant_prefix = format!("{requested}/"); - if let Some(dataset) = datasets - .iter() - .find(|dataset| dataset.name.starts_with(&descendant_prefix)) - { - return Err(format!( - "ZFS dataset {} is a descendant of Provisioned Volume {name}; remove it before retrying", - dataset.name - ) - .into()); - } - let dataset = datasets.iter().find(|dataset| dataset.name == requested); - if dataset.is_some() && root.is_none() { - return Err(format!("ZFS did not report managed root dataset {root_name}").into()); - } - Ok(dataset) - } - - async fn zfs(&self, args: &[&str]) -> Result { - checked_command(&self.zfs, args).await - } -} - -struct Dataset { - name: String, - refquota: u64, - used_bytes: u64, - active_used_bytes: u64, - mountpoint: String, - mounted: bool, - readonly: bool, -} - -impl Dataset { - fn parse(line: &str, pool: &str) -> Result { - let mut fields = line.split('\t'); - let invalid = || format!("invalid ZFS dataset output for Pool {pool}: {line}"); - let ( - Some(name), - Some(refquota), - Some(used_bytes), - Some(active_used_bytes), - Some(mountpoint), - Some(mounted), - Some(readonly), - None, - ) = ( - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - ) - else { - return Err(invalid().into()); - }; - let used_bytes = used_bytes.parse::().map_err(|_| invalid())?; - let active_used_bytes = active_used_bytes.parse::().map_err(|_| invalid())?; - if active_used_bytes > used_bytes { - return Err(invalid().into()); - } - Ok(Self { - name: name.to_owned(), - refquota: parse_zfs_bytes(refquota)?, - used_bytes, - active_used_bytes, - mountpoint: mountpoint.to_owned(), - mounted: mounted == "yes", - readonly: match readonly { - "off" => false, - "on" => true, - _ => return Err(invalid().into()), - }, - }) - } - - fn require_mountpoint(&self, expected: &str) -> Result<()> { - if self.mountpoint == expected { - return Ok(()); - } - Err(format!( - "ZFS dataset {} has incompatible mountpoint {}; set it to {expected} before retrying", - self.name, self.mountpoint - ) - .into()) - } - - fn require_writable(&self) -> Result<()> { - if !self.readonly { - return Ok(()); - } - Err(format!( - "ZFS dataset {} is read-only; make it writable before retrying", - self.name - ) - .into()) - } - - fn require_provisioned(&self, name: &DockerVolumeName) -> Result<()> { - if self.refquota == 0 { - return Err(format!( - "ZFS dataset {} has no Provisioned Volume bound; refusing to use it", - self.name - ) - .into()); - } - self.require_mountpoint(&name.mountpoint()) - } -} - -fn parse_zfs_bytes(value: &str) -> Result { - match value { - "none" | "-" => Ok(0), - _ => value - .parse() - .map_err(|_| VolumeError::from(format!("invalid byte count from ZFS: {value}"))), - } -} - -fn parse_size(options: &BTreeMap) -> Result { - if options.len() != 1 || !options.contains_key("size") { - return Err("Volume option size is required and is the only supported option".into()); - } - let value = options - .get("size") - .expect("the only accepted option is size"); - let (amount, suffix) = value.split_at(value.len().saturating_sub(1)); - let multiplier = match suffix { - "b" => 1, - "k" => 1024_u64, - "m" => 1024_u64.pow(2), - "g" => 1024_u64.pow(3), - "t" => 1024_u64.pow(4), - _ => { - return Err(format!( - "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" - ) - .into()); - } - }; - let amount = amount.parse::().map_err(|_| { - format!( - "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" - ) - })?; - if amount == 0 { - return Err("Volume size must be greater than zero".into()); - } - amount - .checked_mul(multiplier) - .ok_or_else(|| format!("Volume size {value:?} overflows bytes").into()) -} - -async fn checked_command(program: &PathBuf, args: &[&str]) -> Result { - let mut attempt = 0; - let output = loop { - match Command::new(program).args(args).output().await { - // ETXTBSY means exec never started; only that launch error is safe to retry. - Err(error) if error.kind() == io::ErrorKind::ExecutableFileBusy && attempt < 4 => { - tokio::time::sleep(std::time::Duration::from_millis(10 << attempt)).await; - attempt += 1; - } - output => { - break output - .map_err(|error| format!("could not run {}: {error}", program.display()))?; - } - } - }; - if !output.status.success() { - return Err(format!( - "{} {} failed: {}", - program.display(), - args.join(" "), - String::from_utf8_lossy(&output.stderr).trim() - ) - .into()); - } - String::from_utf8(output.stdout) - .map_err(|_| format!("{} returned non-UTF-8 output", program.display()).into()) -} - #[derive(Deserialize)] struct CreateRequest { #[serde(rename = "Name")] diff --git a/crates/ployzd/src/volume_plugin/capacity.rs b/crates/ployzd/src/volume_plugin/capacity.rs index 9119fdf973..5bf3bc391c 100644 --- a/crates/ployzd/src/volume_plugin/capacity.rs +++ b/crates/ployzd/src/volume_plugin/capacity.rs @@ -65,11 +65,37 @@ impl VolumeStorage { }) } + async fn inspect_capacity(&self) -> Result { + let admission = self.admit_mutation().await.map_err(unknown)?; + let storage = self.clone(); + tokio::spawn(async move { + let _admission = admission; + let _pool_guard = storage.pool.lock_mutation().await.map_err(unknown)?; + storage.capacity().await.map_err(unknown) + }) + .await + .unwrap_or_else(|error| Err(unknown(error))) + } + async fn prepare( &self, requested: &Volumes, ) -> Result, ployz_core::RpcError> { - let _guard = self.admit_mutation().await.map_err(storage_error)?; + let admission = self.admit_mutation().await.map_err(storage_error)?; + let storage = self.clone(); + let requested = requested.clone(); + tokio::spawn(async move { + let _admission = admission; + storage.prepare_admitted(&requested).await + }) + .await + .unwrap_or_else(|error| Err(unknown(error))) + } + + async fn prepare_admitted( + &self, + requested: &Volumes, + ) -> Result, ployz_core::RpcError> { let _pool_guard = self.pool.lock_mutation().await.map_err(storage_error)?; let capacity = self.capacity().await.map_err(unknown)?; let budget = capacity @@ -130,26 +156,12 @@ impl VolumeStorage { pub(super) async fn inspect( State(storage): State, ) -> Json> { - // Finish import recovery under the locks even if the observer disconnects. - Json( - tokio::spawn(async move { - let _guard = storage.admit_mutation().await.map_err(unknown)?; - let _pool_guard = storage.pool.lock_mutation().await.map_err(unknown)?; - storage.capacity().await.map_err(unknown) - }) - .await - .unwrap_or_else(|error| Err(unknown(error))), - ) + Json(storage.inspect_capacity().await) } pub(super) async fn prepare( State(storage): State, Json(requested): Json, ) -> Json, ployz_core::RpcError>> { - // Finish admitted allocation even if the requesting connection disappears. - Json( - tokio::spawn(async move { storage.prepare(&requested).await }) - .await - .unwrap_or_else(|error| Err(unknown(error))), - ) + Json(storage.prepare(&requested).await) } diff --git a/crates/ployzd/src/volume_plugin/storage.rs b/crates/ployzd/src/volume_plugin/storage.rs new file mode 100644 index 0000000000..044c0bc6d8 --- /dev/null +++ b/crates/ployzd/src/volume_plugin/storage.rs @@ -0,0 +1,376 @@ +//! Durable ZFS Volume storage ownership and mutation admission. + +use std::{collections::BTreeMap, io, path::PathBuf, sync::Arc}; + +use ployzd::machine_pool::MachinePool; +use tokio::{ + process::Command, + sync::{Mutex, OwnedMutexGuard}, +}; + +use super::{DockerVolumeName, Result, VolumeError, pool::PoolStorage}; + +pub(super) const DATASET_ROOT: &str = "ployz"; +pub(super) const MOUNT_ROOT: &str = "/var/lib/ployz-volumes"; + +#[derive(Clone)] +pub(super) struct VolumeStorage { + pub(super) pool: PoolStorage, + pub(super) zfs: PathBuf, + pub(super) mutation: Arc>, + pub(super) installation: ployzd::mutation::MutationGate, +} + +pub(super) enum CapacityAdmission { + Required, + Ensured, +} + +impl VolumeStorage { + pub(super) fn new(data_dir: impl Into, run_dir: impl Into) -> Self { + Self { + pool: PoolStorage::new("zpool"), + zfs: "zfs".into(), + mutation: Arc::new(Mutex::new(())), + installation: ployzd::mutation::MutationGate::new(run_dir, data_dir), + } + } + + #[cfg(test)] + pub(super) fn with_programs(zpool: impl Into, zfs: impl Into) -> Self { + let zpool = zpool.into(); + let backing = zpool.with_file_name("machine-pool"); + let fixture = zpool + .parent() + .expect("test command has a fixture directory") + .to_owned(); + Self { + pool: PoolStorage::new(zpool).with_backing(backing), + zfs: zfs.into(), + mutation: Arc::new(Mutex::new(())), + installation: ployzd::mutation::MutationGate::new( + fixture.join("admission-run"), + fixture.join("admission-data"), + ), + } + } + + pub(super) async fn admit_mutation( + &self, + ) -> Result<(OwnedMutexGuard<()>, ployzd::mutation::MutationGuard)> { + let local = Arc::clone(&self.mutation).lock_owned().await; + let installation = self + .installation + .try_mutation() + .map_err(|error| VolumeError::from(error.to_string()))?; + Ok((local, installation)) + } + + pub(super) async fn create_volume( + &self, + pool: &MachinePool, + name: &DockerVolumeName, + requested: u64, + origin: CapacityAdmission, + ) -> Result<()> { + let datasets = self.datasets(pool).await?; + let root = format!("{}/{DATASET_ROOT}", pool.name()); + let volume = format!("{root}/{name}"); + + if let Some(existing) = Self::dataset(&datasets, pool, name)? { + existing.require_mountpoint(&name.mountpoint())?; + existing.require_writable()?; + return if existing.refquota == requested { + Ok(()) + } else { + Err(format!( + "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", + existing.refquota + ) + .into()) + }; + } + + if matches!(origin, CapacityAdmission::Required) { + let commitment = datasets + .iter() + .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) + .map(|dataset| dataset.refquota) + .try_fold(requested, u64::checked_add) + .ok_or_else(|| { + VolumeError::from("Provisioned Volume commitments overflowed u64") + })?; + let managed_used = datasets + .iter() + .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) + .map(|dataset| dataset.active_used_bytes) + .try_fold(0u64, u64::checked_add) + .ok_or("Dataset occupancy overflows u64")?; + self.pool + .ensure_capacity( + pool, + commitment, + pool.used_bytes().saturating_sub(managed_used), + ) + .await?; + } + + if !datasets.iter().any(|dataset| dataset.name == root) { + self.zfs(&[ + "create", + "-o", + "canmount=off", + "-o", + &format!("mountpoint={MOUNT_ROOT}"), + &root, + ]) + .await?; + } + self.zfs(&["create", "-o", &format!("refquota={requested}"), &volume]) + .await?; + Ok(()) + } + + pub(super) async fn mountpoint(&self, name: &DockerVolumeName) -> Result { + let _guard = self.admit_mutation().await?; + let pool = self.one_pool().await?; + let datasets = self.datasets(&pool).await?; + let dataset = Self::dataset(&datasets, &pool, name)? + .ok_or_else(|| format!("Provisioned Volume {name} does not exist"))?; + dataset.require_provisioned(name)?; + dataset.require_writable()?; + if dataset.mounted { + return Ok(dataset.mountpoint.clone()); + } + self.zfs(&["mount", &dataset.name]).await?; + let datasets = self.datasets(&pool).await?; + let dataset = Self::dataset(&datasets, &pool, name)? + .ok_or_else(|| format!("Provisioned Volume {name} disappeared while mounting"))?; + dataset.require_provisioned(name)?; + dataset.require_writable()?; + if !dataset.mounted { + return Err(format!("Provisioned Volume {name} did not mount").into()); + } + Ok(dataset.mountpoint.clone()) + } + + pub(super) async fn one_pool(&self) -> Result { + self.pool + .one_usable() + .await? + .ok_or_else(|| "no usable existing Machine Pool".into()) + } + + pub(super) async fn datasets(&self, pool: &MachinePool) -> Result> { + let output = self + .zfs(&[ + "list", + "-Hp", + "-o", + "name,refquota,used,usedbydataset,mountpoint,mounted,readonly", + "-r", + pool.name(), + ]) + .await?; + output + .lines() + .map(|line| Dataset::parse(line, pool.name())) + .collect() + } + + pub(super) fn dataset<'datasets>( + datasets: &'datasets [Dataset], + pool: &MachinePool, + name: &DockerVolumeName, + ) -> Result> { + let root_name = format!("{}/{DATASET_ROOT}", pool.name()); + let root = datasets.iter().find(|dataset| dataset.name == root_name); + if let Some(root) = root { + root.require_mountpoint(MOUNT_ROOT)?; + root.require_writable()?; + } + + let requested = format!("{}/{DATASET_ROOT}/{name}", pool.name()); + let descendant_prefix = format!("{requested}/"); + if let Some(dataset) = datasets + .iter() + .find(|dataset| dataset.name.starts_with(&descendant_prefix)) + { + return Err(format!( + "ZFS dataset {} is a descendant of Provisioned Volume {name}; remove it before retrying", + dataset.name + ) + .into()); + } + let dataset = datasets.iter().find(|dataset| dataset.name == requested); + if dataset.is_some() && root.is_none() { + return Err(format!("ZFS did not report managed root dataset {root_name}").into()); + } + Ok(dataset) + } + + pub(super) async fn zfs(&self, args: &[&str]) -> Result { + checked_command(&self.zfs, args).await + } +} + +pub(super) struct Dataset { + pub(super) name: String, + pub(super) refquota: u64, + pub(super) used_bytes: u64, + pub(super) active_used_bytes: u64, + pub(super) mountpoint: String, + pub(super) mounted: bool, + pub(super) readonly: bool, +} + +impl Dataset { + pub(super) fn parse(line: &str, pool: &str) -> Result { + let mut fields = line.split('\t'); + let invalid = || format!("invalid ZFS dataset output for Pool {pool}: {line}"); + let ( + Some(name), + Some(refquota), + Some(used_bytes), + Some(active_used_bytes), + Some(mountpoint), + Some(mounted), + Some(readonly), + None, + ) = ( + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + ) + else { + return Err(invalid().into()); + }; + let used_bytes = used_bytes.parse::().map_err(|_| invalid())?; + let active_used_bytes = active_used_bytes.parse::().map_err(|_| invalid())?; + if active_used_bytes > used_bytes { + return Err(invalid().into()); + } + Ok(Self { + name: name.to_owned(), + refquota: parse_zfs_bytes(refquota)?, + used_bytes, + active_used_bytes, + mountpoint: mountpoint.to_owned(), + mounted: mounted == "yes", + readonly: match readonly { + "off" => false, + "on" => true, + _ => return Err(invalid().into()), + }, + }) + } + + pub(super) fn require_mountpoint(&self, expected: &str) -> Result<()> { + if self.mountpoint == expected { + return Ok(()); + } + Err(format!( + "ZFS dataset {} has incompatible mountpoint {}; set it to {expected} before retrying", + self.name, self.mountpoint + ) + .into()) + } + + pub(super) fn require_writable(&self) -> Result<()> { + if !self.readonly { + return Ok(()); + } + Err(format!( + "ZFS dataset {} is read-only; make it writable before retrying", + self.name + ) + .into()) + } + + pub(super) fn require_provisioned(&self, name: &DockerVolumeName) -> Result<()> { + if self.refquota == 0 { + return Err(format!( + "ZFS dataset {} has no Provisioned Volume bound; refusing to use it", + self.name + ) + .into()); + } + self.require_mountpoint(&name.mountpoint()) + } +} + +fn parse_zfs_bytes(value: &str) -> Result { + match value { + "none" | "-" => Ok(0), + _ => value + .parse() + .map_err(|_| VolumeError::from(format!("invalid byte count from ZFS: {value}"))), + } +} + +pub(super) fn parse_size(options: &BTreeMap) -> Result { + if options.len() != 1 || !options.contains_key("size") { + return Err("Volume option size is required and is the only supported option".into()); + } + let value = options + .get("size") + .expect("the only accepted option is size"); + let (amount, suffix) = value.split_at(value.len().saturating_sub(1)); + let multiplier = match suffix { + "b" => 1, + "k" => 1024_u64, + "m" => 1024_u64.pow(2), + "g" => 1024_u64.pow(3), + "t" => 1024_u64.pow(4), + _ => { + return Err(format!( + "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" + ) + .into()); + } + }; + let amount = amount.parse::().map_err(|_| { + format!( + "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" + ) + })?; + if amount == 0 { + return Err("Volume size must be greater than zero".into()); + } + amount + .checked_mul(multiplier) + .ok_or_else(|| format!("Volume size {value:?} overflows bytes").into()) +} + +pub(super) async fn checked_command(program: &PathBuf, args: &[&str]) -> Result { + let mut attempt = 0; + let output = loop { + match Command::new(program).args(args).output().await { + // ETXTBSY means exec never started; only that launch error is safe to retry. + Err(error) if error.kind() == io::ErrorKind::ExecutableFileBusy && attempt < 4 => { + tokio::time::sleep(std::time::Duration::from_millis(10 << attempt)).await; + attempt += 1; + } + output => { + break output + .map_err(|error| format!("could not run {}: {error}", program.display()))?; + } + } + }; + if !output.status.success() { + return Err(format!( + "{} {} failed: {}", + program.display(), + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + String::from_utf8(output.stdout) + .map_err(|_| format!("{} returned non-UTF-8 output", program.display()).into()) +} diff --git a/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs b/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs index 336bddb8df..23178d2f0b 100644 --- a/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs +++ b/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs @@ -2,9 +2,12 @@ use std::{ os::unix::fs::{MetadataExt, PermissionsExt}, + sync::Arc, time::Duration, }; +use tokio::sync::Mutex; + use super::super::pool::{POOL_BACKING_FILE, PoolStorage}; use super::*; @@ -958,7 +961,7 @@ esac ), zfs: program("zfs"), mutation: Arc::new(Mutex::new(())), - installation: ployzd::installer::admission::Admission::new( + installation: ployzd::mutation::MutationGate::new( directory.join("admission-run"), directory.join("admission-data"), ), diff --git a/scripts/qualify-release.sh b/scripts/qualify-release.sh index 84a10c2988..4b0f07b286 100755 --- a/scripts/qualify-release.sh +++ b/scripts/qualify-release.sh @@ -29,6 +29,8 @@ TRAFFIC_STOP=/var/lib/ployz/qualification/traffic.stop REMOTE_RELEASE_ROOT=/var/lib/ployz/qualification/releases APP_URL=http://127.0.0.1:18082/identity APP_VALUE=qualify-persistent-data +APP_VOLUME=qualify-release_qualify-data +APP_VOLUME_MOUNT=/var/lib/ployz-volumes/$APP_VOLUME error() { echo "ERROR: $*" >&2; exit 1; } @@ -162,6 +164,18 @@ wait_for_application() { error "qualification application did not serve its persistent value" } +assert_provisioned_volume() { + local volumes + volumes=$("$PLOYZ" volume ls --context "$CONTEXT") + printf '%s\n' "$volumes" + printf '%s\n' "$volumes" | awk -F '\t' -v volume="$APP_VOLUME" ' + $2 == volume && $3 == "PROVISIONED" && $6 == "ployz" { found = 1 } + END { exit !found } + ' || error "$APP_VOLUME is not a provisioned Ployz Volume" + ssh_host "$first" "test \"\$(sudo docker volume inspect '$APP_VOLUME' --format '{{.Driver}}')\" = ployz" || error "$APP_VOLUME is not backed by the Ployz Docker driver" + ssh_host "$first" "sudo zfs list -H -o mountpoint | grep -Fqx '$APP_VOLUME_MOUNT'" || error "$APP_VOLUME has no mounted ZFS dataset" +} + start_traffic() { ssh_host "$first" "sudo rm -f '$TRAFFIC_STOP'" # shellcheck disable=SC2016 # The loop and its variables belong to the remote shell. @@ -309,9 +323,7 @@ ssh_host "$first" "sudo install -d -m 0755 /etc/systemd/system/ployz.service.d & echo "deploy persistent ZFS-backed application" "$PLOYZ" deploy --yes --context "$CONTEXT" -f "$COMPOSE_DIR/compose.yaml" -volumes=$("$PLOYZ" volume ls --context "$CONTEXT") -printf '%s\n' "$volumes" -printf '%s\n' "$volumes" | grep -Eq 'qualify-data' || error "named volume qualify-data did not appear in volume ls" +assert_provisioned_volume wait_for_application state_before=$(machine_state_signature) start_traffic @@ -387,6 +399,5 @@ traffic_successes=$(grep -c '^ok$' "$work/traffic.log" || true) [ "$traffic_successes" -ge 5 ] || error "continuous traffic collected only $traffic_successes successful probes" printf 'continuous application traffic: %s successful probes, 0 failures\n' "$traffic_successes" -volumes=$("$PLOYZ" volume ls --context "$CONTEXT") -printf '%s\n' "$volumes" | grep -Eq 'qualify-data' || error "named volume disappeared after upgrade" +assert_provisioned_volume echo "qualify-release passed on ${HOST_LIST[*]}: $source_version -> $target_version" diff --git a/scripts/qualify-release/compose.yaml b/scripts/qualify-release/compose.yaml index b243238c00..1020f247ba 100644 --- a/scripts/qualify-release/compose.yaml +++ b/scripts/qualify-release/compose.yaml @@ -1,6 +1,6 @@ services: data: - image: alpine:3.23.3 + image: busybox:1.37.0 command: - sh - -c @@ -9,5 +9,5 @@ services: - qualify-data:/data x-machines: [qualify-1] x-ports: [18082:8080/tcp@host] -volumes: - qualify-data: +x-volumes: + qualify-data: 256M diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index 253ca38c16..be04d98d28 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -16,6 +16,8 @@ if grep -qi vultr "$ROOT/scripts/qualify-release.sh" "$ROOT/docs/RELEASE.md"; th fi grep -Fq 'qualify-data' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no named volume" +grep -Fq 'x-volumes:' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture volume is not provisioned" +grep -Fq 'busybox:1.37.0' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no verified HTTP server image" grep -Fq '18082:8080/tcp@host' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no traffic endpoint" grep -Fq '/data/identity' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture does not serve persistent data" @@ -56,7 +58,7 @@ case "$1" in shift done printf '%s context=%s file=%s yes=%s\n' "$action" "$context" "$file" "$yes" >>"$LOG" - [ "$action" != volume ] || printf 'qualify-data\n' + [ "$action" != volume ] || printf 'MACHINE\tVOLUME\tTYPE\tQUOTA\tUSED\tDRIVER\nqualify-1\tqualify-release_qualify-data\tPROVISIONED\t268435456\t4096\tployz\n' ;; machine) action=$2 @@ -98,7 +100,7 @@ case "$1" in current=$(cat "$QUALIFY_STATE") printf 'upgrade-inspect machine=%s attempt=%s context=%s output=%s current=%s\n' "$machine" "$attempt" "$context" "$output" "$current" >>"$LOG" case "$current" in - target) printf '{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","version":"1.2.4"}\n' ;; + target) printf '{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","target":"1.2.4","version":"1.2.4"}\n' ;; failure) printf '{"outcome":"failed","attempt_id":"cccccccccccccccccccccccccccccccc","target":"1.2.4","stage":"restarting","error":"restart daemon failed"}\n' ;; *) exit 1 ;; esac @@ -188,7 +190,7 @@ case "$command" in *'ln -sfn'*'/failure'*'/current'*) printf 'failure\n' >"$QUALIFY_STATE" ;; *'upgrade-attempt.json'*) case "$(cat "$QUALIFY_STATE")" in - target) printf '{"requested":"1.2.4","attempt":{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","version":"1.2.4"}}\n' ;; + target) printf '{"requested":"1.2.4","attempt":{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","target":"1.2.4","version":"1.2.4"}}\n' ;; corrupt) printf '{"requested":"1.2.4","attempt":{"outcome":"failed","attempt_id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","target":"1.2.4","stage":"verifying","error":"checksum mismatch"}}\n' ;; failure) printf '{"requested":"1.2.4","attempt":{"outcome":"failed","attempt_id":"cccccccccccccccccccccccccccccccc","target":"1.2.4","stage":"restarting","error":"restart daemon failed"}}\n' ;; esac @@ -236,6 +238,8 @@ for reset in 0 1; do grep -Fxq 'upgrade-inspect machine=qualify-1 attempt=cccccccccccccccccccccccccccccccc context=qualify output=json current=failure' "$LOG" || fail "failed attempt was not inspected after repair" done grep -Fq 'ssh host=root@192.0.2.10 command=' "$SSH_LOG" || fail "SSH evidence path was not exercised" +grep -Fq 'docker volume inspect' "$SSH_LOG" || fail "qualification did not verify the Docker Volume driver" +grep -Fq 'zfs list -H -o mountpoint' "$SSH_LOG" || fail "qualification did not verify the ZFS dataset mount" grep -Fq 'scp ' "$SSH_LOG" || fail "target release was not staged with SCP" cat >"$TMP/bin/cargo" <<'CARGO' From 98f721066f82c8a1d8e683aeeebdb30b5be5db5f Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 03:58:54 +0000 Subject: [PATCH 21/30] Fix provisioned volume metadata registration --- crates/ployzd/src/docker/volume.rs | 84 +++++++++++--- crates/ployzd/src/docker/volume/tests.rs | 135 +++++++++++++++++++++++ 2 files changed, 202 insertions(+), 17 deletions(-) diff --git a/crates/ployzd/src/docker/volume.rs b/crates/ployzd/src/docker/volume.rs index 7506b4dcfd..d35b8c18d7 100644 --- a/crates/ployzd/src/docker/volume.rs +++ b/crates/ployzd/src/docker/volume.rs @@ -65,22 +65,53 @@ impl ContainerRuntime { match self.inspect_volume(machine_id, name).await { Ok(volume) => verify_volume(source, &volume), Err(error) if volume_not_found(&error) => { - let request = source - .to_create_volume_request() - .expect("managed Docker Volume sources have creation requests"); - match self.create_volume(machine_id, request).await { - Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), - Ok(CreateVolumeReport::Unverified { id, error }) => { - Err(Error::VolumeCreatedButUnverified { - id, - error: Box::new(error), - }) - } - Err(error) if volume_conflict(&error) => { - verify_volume(source, &self.inspect_volume(machine_id, name).await?) - } - Err(error) => Err(error), - } + self.create_managed_volume(machine_id, source).await + } + Err(error) => Err(error), + } + } + + async fn create_managed_volume( + &self, + machine_id: &MachineId, + source: &VolumeSource, + ) -> Result<(), Error> { + let name = source + .docker_volume_name() + .expect("managed Docker Volume sources have names"); + let request = source + .to_create_volume_request() + .expect("managed Docker Volume sources have creation requests"); + match self.create_volume(machine_id, request).await { + Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), + Ok(CreateVolumeReport::Unverified { id, error }) => { + Err(Error::VolumeCreatedButUnverified { + id, + error: Box::new(error), + }) + } + Err(error) if volume_conflict(&error) => { + verify_volume(source, &self.inspect_volume(machine_id, name).await?) + } + Err(error) => Err(error), + } + } + + async fn finish_prepared_volume( + &self, + machine_id: &MachineId, + source: &VolumeSource, + ) -> Result<(), Error> { + let name = source + .docker_volume_name() + .expect("provisioned Volume has a name"); + match self.inspect_volume(machine_id, name).await { + Ok(volume) if prepared_volume_needs_registration(source, &volume) => { + self.create_managed_volume(machine_id, source).await + } + Ok(volume) => verify_volume(source, &volume), + Err(error) if volume_not_found(&error) => { + self.create_managed_volume(machine_id, source).await } Err(error) => Err(error), } @@ -126,7 +157,7 @@ impl ContainerRuntime { ) -> Result<(), Error> { for spec in specs { for volume in spec.volume_graph().mounted_provisioned_volumes() { - self.ensure_volume_source(machine_id, &volume.source) + self.finish_prepared_volume(machine_id, &volume.source) .await?; } } @@ -332,6 +363,25 @@ fn verify_volume(source: &VolumeSource, observed: &DockerVolume) -> Result<(), E Ok(()) } +fn prepared_volume_needs_registration(source: &VolumeSource, observed: &DockerVolume) -> bool { + let ployz_core::RawVolumeSource::Provisioned { + name, + maximum_bytes, + .. + } = source.kind() + else { + return false; + }; + observed.id.name == *name + && observed.options.is_empty() + && observed.labels.is_empty() + && matches!( + &observed.storage, + DockerVolumeStorageObservation::Provisioned { bound_bytes, .. } + if bound_bytes.get() == maximum_bytes.get() + ) +} + fn volume_not_found(error: &Error) -> bool { docker_status(error, 404) } diff --git a/crates/ployzd/src/docker/volume/tests.rs b/crates/ployzd/src/docker/volume/tests.rs index 285c06399a..64addd88f6 100644 --- a/crates/ployzd/src/docker/volume/tests.rs +++ b/crates/ployzd/src/docker/volume/tests.rs @@ -297,6 +297,141 @@ async fn missing_provisioned_volume_uses_ployz_driver_bound_and_labels() { ); } +#[tokio::test] +async fn preparation_registers_metadata_for_storage_created_by_plugin() { + let (runtime, fake) = fake_runtime().await; + let machine_id = MachineId::random(); + let source = provisioned_source("prepared", 2_147_483_648); + fake.volumes.lock().unwrap().insert( + "app_prepared".into(), + serde_json::json!({ + "Name":"app_prepared", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_prepared", + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, + "Options":null, + "Labels":null + }), + ); + let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( + vec![source.clone()], + ))]; + + assert!(matches!( + runtime.ensure_volume_source(&machine_id, &source).await, + Err(Error::VolumeShapeMismatch { .. }) + )); + assert!( + fake.requests + .lock() + .unwrap() + .iter() + .all(|(method, _)| method != Method::POST) + ); + + runtime + .ensure_provisioned_volumes(&machine_id, &specs) + .await + .unwrap(); + + let create = fake + .request_bodies + .lock() + .unwrap() + .iter() + .find(|(path, _)| path.ends_with("/volumes/create")) + .map(|(_, request)| request.clone()) + .expect("Docker metadata registration request"); + assert_eq!( + create, + serde_json::json!({ + "Name":"app_prepared", + "Driver":"ployz", + "DriverOpts":{"size":"2147483648b"}, + "Labels":{"backup":"daily","ployz.managed":"","ployz.project.name":"app"} + }) + ); + let observed = runtime + .inspect_volume(&machine_id, source.docker_volume_name().unwrap()) + .await + .unwrap(); + assert!(source.matches_managed_volume(&observed)); +} + +#[tokio::test] +async fn preparation_refuses_mismatched_or_partially_registered_volumes() { + let (runtime, fake) = fake_runtime().await; + let cases = [ + ( + "wrong-driver", + serde_json::json!({ + "Name":"app_wrong-driver", + "Driver":"local", + "Mountpoint":"/var/lib/docker/volumes/app_wrong-driver/_data", + "Options":null, + "Labels":null + }), + ), + ( + "wrong-bound", + serde_json::json!({ + "Name":"app_wrong-bound", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_wrong-bound", + "Status":{"bound_bytes":1073741824_u64,"used_bytes":0}, + "Options":null, + "Labels":null + }), + ), + ( + "partial-options", + serde_json::json!({ + "Name":"app_partial-options", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_partial-options", + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, + "Options":{"size":"2147483648b"}, + "Labels":null + }), + ), + ( + "partial-labels", + serde_json::json!({ + "Name":"app_partial-labels", + "Driver":"ployz", + "Mountpoint":"/var/lib/ployz-volumes/app_partial-labels", + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, + "Options":null, + "Labels":{"ployz.managed":"","ployz.project.name":"app"} + }), + ), + ]; + + for (name, observed) in cases { + let source = provisioned_source(name, 2_147_483_648); + fake.volumes + .lock() + .unwrap() + .insert(format!("app_{name}"), observed); + let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( + vec![source], + ))]; + assert!(matches!( + runtime + .ensure_provisioned_volumes(&MachineId::random(), &specs) + .await, + Err(Error::VolumeShapeMismatch { .. }) + )); + } + assert!( + fake.requests + .lock() + .unwrap() + .iter() + .all(|(method, _)| method != Method::POST) + ); +} + #[tokio::test] async fn existing_managed_volume_allows_extra_labels() { let (runtime, fake) = fake_runtime().await; From 0c0f02c28e2afa5811421ea0c6ac6953a8256f6a Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:09:45 +0000 Subject: [PATCH 22/30] test(machine): await guarded local mutations --- crates/ployzd/tests/machine_record.rs | 33 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/crates/ployzd/tests/machine_record.rs b/crates/ployzd/tests/machine_record.rs index 773278bdd6..42e9e88ec9 100644 --- a/crates/ployzd/tests/machine_record.rs +++ b/crates/ployzd/tests/machine_record.rs @@ -130,8 +130,8 @@ fn sample_cloud_pairing() -> CloudPairing { .unwrap() } -#[test] -fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { +#[tokio::test] +async fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { let dir = TestDir::new("ployzd-initialize-cloud-pairing"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); @@ -147,6 +147,7 @@ fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { wireguard_mtu: None, cloud_pairing: Some(pairing.clone()), }) + .await .unwrap(); assert_eq!( @@ -171,8 +172,8 @@ fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { assert!(pairing_json.get("dial").is_none()); } -#[test] -fn set_cloud_pairing_after_initialize_persists() { +#[tokio::test] +async fn set_cloud_pairing_after_initialize_persists() { let dir = TestDir::new("ployzd-set-cloud-pairing"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); @@ -188,10 +189,14 @@ fn set_cloud_pairing_after_initialize_persists() { wireguard_mtu: None, cloud_pairing: None, }) + .await .unwrap(); assert_eq!(local.record().unwrap().cloud_pairing, None); - local.set_cloud_pairing(Some(pairing.clone())).unwrap(); + local + .set_cloud_pairing(Some(pairing.clone())) + .await + .unwrap(); assert_eq!( local.record().unwrap().cloud_pairing.as_ref(), Some(&pairing) @@ -201,8 +206,8 @@ fn set_cloud_pairing_after_initialize_persists() { assert_eq!(reopened.record().cloud_pairing.as_ref(), Some(&pairing)); } -#[test] -fn set_cloud_pairing_none_clears_persisted_pairing() { +#[tokio::test] +async fn set_cloud_pairing_none_clears_persisted_pairing() { let dir = TestDir::new("ployzd-clear-cloud-pairing"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); @@ -216,28 +221,30 @@ fn set_cloud_pairing_none_clears_persisted_pairing() { wireguard_mtu: None, cloud_pairing: Some(sample_cloud_pairing()), }) + .await .unwrap(); - local.set_cloud_pairing(None).unwrap(); + local.set_cloud_pairing(None).await.unwrap(); assert_eq!(local.record().unwrap().cloud_pairing, None); drop(local); let reopened = LocalMachineStore::open(&dir.0).unwrap(); assert_eq!(reopened.record().cloud_pairing, None); } -#[test] -fn set_cloud_pairing_before_initialize_is_not_participating() { +#[tokio::test] +async fn set_cloud_pairing_before_initialize_is_not_participating() { let dir = TestDir::new("ployzd-set-cloud-pairing-uninitialized"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); let local = LocalMachine::new(Arc::new(Mutex::new(store)), reset); let error = local .set_cloud_pairing(Some(sample_cloud_pairing())) + .await .unwrap_err(); assert!(matches!(error, LocalMachineError::NotParticipating)); } -#[test] -fn join_with_cloud_pairing_stores_the_same_two_fields() { +#[tokio::test] +async fn join_with_cloud_pairing_stores_the_same_two_fields() { let first_dir = TestDir::new("ployzd-join-cloud-pairing-first"); let mut first = LocalMachineStore::open(&first_dir.0).unwrap(); let initialized = first @@ -278,6 +285,7 @@ fn join_with_cloud_pairing_stores_the_same_two_fields() { wireguard_mtu: None, cloud_pairing: Some(pairing.clone()), }) + .await .unwrap(); assert_eq!( @@ -440,6 +448,7 @@ async fn inspect_reports_stored_cloud_pairing_without_the_secret() { wireguard_mtu: None, cloud_pairing: Some(sample_cloud_pairing()), }) + .await .unwrap(); let details = local.inspect(InspectRequest::default()).await.unwrap(); From 4d578c9a0794847ef33b541991ae4e85adacc8b0 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:18:44 +0000 Subject: [PATCH 23/30] fix(installer): reject nonstandard system paths --- crates/ployz/src/handlers/machine/upgrade.rs | 39 +++++++++++------- crates/ployzd/src/installer/mod.rs | 42 ++++++++++++++++++-- crates/ployzd/src/main.rs | 7 ++-- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/crates/ployz/src/handlers/machine/upgrade.rs b/crates/ployz/src/handlers/machine/upgrade.rs index 479ba16796..eea9f745ea 100644 --- a/crates/ployz/src/handlers/machine/upgrade.rs +++ b/crates/ployz/src/handlers/machine/upgrade.rs @@ -69,7 +69,7 @@ pub(in crate::handlers) fn upgrade(root: &ArgMatches) -> Result<(), Error> { let attempt = match run_one(client, machine, release.clone(), attempt_id).await { Ok(attempt) => attempt, Err(error) => { - print_unattempted(&machines[index + 1..], machine); + print_unattempted(machines.iter().skip(index.saturating_add(1)), machine); return Err(error); } }; @@ -77,11 +77,11 @@ pub(in crate::handlers) fn upgrade(root: &ArgMatches) -> Result<(), Error> { match attempt.outcome { MachineUpgradeOutcome::Succeeded { .. } => {} MachineUpgradeOutcome::Failed { error, .. } => { - print_unattempted(&machines[index + 1..], machine); + print_unattempted(machines.iter().skip(index.saturating_add(1)), machine); return Err(Error::usage(error)); } MachineUpgradeOutcome::Interrupted { .. } => { - print_unattempted(&machines[index + 1..], machine); + print_unattempted(machines.iter().skip(index.saturating_add(1)), machine); return Err(Error::usage(format!( "Machine {} upgrade was interrupted; {}", machine.name, @@ -260,15 +260,18 @@ fn print_attempt_target(machine: &str, attempt: &MachineUpgradeAttempt) { } } -fn print_unattempted(machines: &[Machine], after: &Machine) { +fn print_unattempted<'a>(machines: impl IntoIterator, after: &Machine) { for line in unattempted_lines(machines, after) { println!("{line}"); } } -fn unattempted_lines(machines: &[Machine], after: &Machine) -> Vec { +fn unattempted_lines<'a>( + machines: impl IntoIterator, + after: &Machine, +) -> Vec { machines - .iter() + .into_iter() .map(|machine| { format!( "Machine {} ({}): upgrade unattempted after {} ({})", @@ -410,17 +413,23 @@ mod tests { ]; let first = unattempted_lines(&machines[1..], &machines[0]); - assert_eq!(first.len(), 3); - assert!(first[0].contains("Machine b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)")); - assert!(first[0].contains("after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)")); - assert!(first[1].contains("Machine c (cccccccccccccccccccccccccccccccc)")); - assert!(first[2].contains("Machine d (dddddddddddddddddddddddddddddddd)")); + assert_eq!( + first, + [ + "Machine b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb): upgrade unattempted after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)", + "Machine c (cccccccccccccccccccccccccccccccc): upgrade unattempted after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)", + "Machine d (dddddddddddddddddddddddddddddddd): upgrade unattempted after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)", + ] + ); let middle = unattempted_lines(&machines[2..], &machines[1]); - assert_eq!(middle.len(), 2); - assert!(middle[0].contains("Machine c (cccccccccccccccccccccccccccccccc)")); - assert!(middle[0].contains("after b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)")); - assert!(middle[1].contains("Machine d (dddddddddddddddddddddddddddddddd)")); + assert_eq!( + middle, + [ + "Machine c (cccccccccccccccccccccccccccccccc): upgrade unattempted after b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)", + "Machine d (dddddddddddddddddddddddddddddddd): upgrade unattempted after b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)", + ] + ); } fn machine(id: char, subnet: u8) -> Machine { diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index c7758b1d92..eadfd07a5d 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -31,6 +31,9 @@ pub use self::release::{ReleaseRequest, ReleaseSource}; const PLOYZ_USER: &str = "ployz"; const DEFAULT_BIN_DIR: &str = "/usr/local/bin"; const DEFAULT_SYSTEMD_DIR: &str = "/etc/systemd/system"; +const DEFAULT_RUN_DIR: &str = "/run/ployz"; +/// Unix socket installed systemd services use for the local Machine API. +pub const DEFAULT_SOCKET_PATH: &str = "/run/ployz/ployz.sock"; /// Explicit host work associated with one Machine installation attempt. #[derive(Clone, Debug, Eq, PartialEq)] pub enum Preparation { @@ -93,6 +96,10 @@ pub enum Error { UnsupportedArchitecture(String), #[error("Ployz requires systemd")] SystemdRequired, + #[error( + "system installation requires --data-dir /var/lib/ployz and --socket /run/ployz/ployz.sock; received --data-dir {data_dir:?} and --socket {socket:?}" + )] + NonstandardPaths { data_dir: PathBuf, socket: PathBuf }, #[error("release selection: {0}")] ReleaseSelection(String), #[error("artifact verification: {0}")] @@ -142,7 +149,7 @@ impl InstallPaths { } } -/// Install using the daemon's configured data and runtime directories. +/// Install into global host locations for the standard Machine data and socket paths. /// /// # Errors /// @@ -150,9 +157,16 @@ impl InstallPaths { pub async fn install( request: InstallRequest, data_dir: impl Into, - run_dir: impl Into, + socket: impl Into, ) -> Result { - install_at(request, InstallPaths::system(data_dir, run_dir)).await + let data_dir = data_dir.into(); + let socket = socket.into(); + if data_dir != Path::new(crate::machine::DEFAULT_DATA_DIR) + || socket != Path::new(DEFAULT_SOCKET_PATH) + { + return Err(Error::NonstandardPaths { data_dir, socket }); + } + install_at(request, InstallPaths::system(data_dir, DEFAULT_RUN_DIR)).await } async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result { @@ -368,6 +382,28 @@ mod tests { use super::*; + #[tokio::test] + async fn system_install_rejects_nonstandard_machine_paths_before_mutation() { + let fixture = fixture("nonstandard-paths"); + let request = InstallRequest { + release: ReleaseRequest::Exact(Version::parse("1.2.3").unwrap()), + source: ReleaseSource::Local(fixture.path().join("release")), + preparation: Preparation::SoftwareOnly, + install_only: true, + }; + + for (data_dir, socket) in [ + (fixture.path(), Path::new(DEFAULT_SOCKET_PATH)), + (Path::new(crate::machine::DEFAULT_DATA_DIR), fixture.path()), + ] { + let error = install(request.clone(), data_dir, socket) + .await + .unwrap_err(); + assert!(matches!(error, Error::NonstandardPaths { .. })); + } + assert!(fs::read_dir(fixture.path()).unwrap().next().is_none()); + } + #[tokio::test] async fn installation_interface_contract() { if let Ok(case) = env::var("PLOYZ_INSTALLER_CONTRACT_CASE") { diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index 86bf2cbc51..384aff6880 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -16,13 +16,14 @@ use ployz_core::{DOCKER_NETWORK_CONFLICT_EXIT_STATUS, MachineUpgradeAttemptId, S use ployzd::{ daemon::{ContainerMode, Daemon, DaemonConfig, Error, wait_until_socket_accepts}, diag, - installer::{InstallRequest, Preparation, Readiness, ReleaseRequest, ReleaseSource}, + installer::{ + DEFAULT_SOCKET_PATH, InstallRequest, Preparation, Readiness, ReleaseRequest, ReleaseSource, + }, machine::DEFAULT_DATA_DIR, network::NetworkError, }; use tokio::io::{AsyncWriteExt, copy, stdin, stdout}; -const DEFAULT_SOCKET_PATH: &str = "/run/ployz/ployz.sock"; const DIAL_STDIO_SOCKET_TIMEOUT: Duration = Duration::from_secs(20); #[derive(Parser)] @@ -154,7 +155,7 @@ async fn run(args: Args) -> Result<(), Error> { release_dir, ) .map_err(Error::from)?; - let outcome = ployzd::installer::install(request, &args.data_dir, &run_dir) + let outcome = ployzd::installer::install(request, &args.data_dir, &args.socket) .await .map_err(io::Error::other)?; match outcome.readiness { From de09dea4fedefdd860b41fdc0b7c50ae2954a3b3 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:26:37 +0000 Subject: [PATCH 24/30] fix(upgrade): reject nonstandard activation paths --- crates/ployzd/src/installer/mod.rs | 29 +++++++++------ crates/ployzd/src/installer/upgrade.rs | 23 ++++++++++++ .../src/machine/local_machine/upgrade.rs | 2 ++ crates/ployzd/src/machine_api/local.rs | 2 +- crates/ployzd/src/machine_api/local_tests.rs | 35 +++++++++++++++++++ 5 files changed, 80 insertions(+), 11 deletions(-) diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index eadfd07a5d..454c3f9ea1 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -96,10 +96,8 @@ pub enum Error { UnsupportedArchitecture(String), #[error("Ployz requires systemd")] SystemdRequired, - #[error( - "system installation requires --data-dir /var/lib/ployz and --socket /run/ployz/ployz.sock; received --data-dir {data_dir:?} and --socket {socket:?}" - )] - NonstandardPaths { data_dir: PathBuf, socket: PathBuf }, + #[error("{0}")] + NonstandardPaths(String), #[error("release selection: {0}")] ReleaseSelection(String), #[error("artifact verification: {0}")] @@ -149,6 +147,21 @@ impl InstallPaths { } } +pub(crate) fn require_standard_machine_paths(data_dir: &Path, socket: &Path) -> Result<(), String> { + if data_dir == Path::new(crate::machine::DEFAULT_DATA_DIR) + && socket == Path::new(DEFAULT_SOCKET_PATH) + { + return Ok(()); + } + Err(format!( + "system installation requires --data-dir {} and --socket {}; received --data-dir {} and --socket {}", + crate::machine::DEFAULT_DATA_DIR, + DEFAULT_SOCKET_PATH, + data_dir.display(), + socket.display() + )) +} + /// Install into global host locations for the standard Machine data and socket paths. /// /// # Errors @@ -161,11 +174,7 @@ pub async fn install( ) -> Result { let data_dir = data_dir.into(); let socket = socket.into(); - if data_dir != Path::new(crate::machine::DEFAULT_DATA_DIR) - || socket != Path::new(DEFAULT_SOCKET_PATH) - { - return Err(Error::NonstandardPaths { data_dir, socket }); - } + require_standard_machine_paths(&data_dir, &socket).map_err(Error::NonstandardPaths)?; install_at(request, InstallPaths::system(data_dir, DEFAULT_RUN_DIR)).await } @@ -399,7 +408,7 @@ mod tests { let error = install(request.clone(), data_dir, socket) .await .unwrap_err(); - assert!(matches!(error, Error::NonstandardPaths { .. })); + assert!(matches!(error, Error::NonstandardPaths(_))); } assert!(fs::read_dir(fixture.path()).unwrap().next().is_none()); } diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs index eac00c1505..599cffa4ff 100644 --- a/crates/ployzd/src/installer/upgrade.rs +++ b/crates/ployzd/src/installer/upgrade.rs @@ -51,6 +51,9 @@ pub enum Error { /// The process-local qualification source was not a trusted absolute directory. #[error("invalid qualification release directory: {0}")] QualificationSource(String), + /// Global activation was requested from a daemon using unsupported Machine paths. + #[error("{0}")] + NonstandardPaths(String), /// The requested release could not be parsed or resolved. #[error("resolve Machine release: {0}")] Resolve(#[source] InstallError), @@ -215,6 +218,8 @@ pub async fn run_worker( data_dir: &Path, run_dir: &Path, ) -> Result<(), Error> { + super::require_standard_machine_paths(data_dir, &run_dir.join("ployz.sock")) + .map_err(Error::NonstandardPaths)?; let admission = mutation::MutationGate::new(run_dir, data_dir); let guard = admission.lock_installation()?; let mut stored = read(data_dir)?; @@ -476,6 +481,24 @@ mod tests { const CONTRACT_CASE: &str = "PLOYZ_UPGRADE_CONTRACT_CASE"; const CONTRACT_ROOT: &str = "PLOYZ_UPGRADE_CONTRACT_ROOT"; + #[tokio::test] + async fn worker_rejects_nonstandard_paths_before_reading_local_state() { + let root = tempfile::Builder::new() + .prefix("ployzd-upgrade-worker-paths-") + .tempdir() + .unwrap(); + let data_dir = root.path().join("data"); + let run_dir = root.path().join("run"); + + let error = run_worker(MachineUpgradeAttemptId::random(), &data_dir, &run_dir) + .await + .unwrap_err(); + + assert!(matches!(error, Error::NonstandardPaths(_))); + assert!(!data_dir.exists()); + assert!(!run_dir.exists()); + } + #[test] fn stored_attempt_round_trips_without_exposing_local_source_in_response() { let attempt_id = MachineUpgradeAttemptId::random(); diff --git a/crates/ployzd/src/machine/local_machine/upgrade.rs b/crates/ployzd/src/machine/local_machine/upgrade.rs index 68ace0d5d8..cd6f767803 100644 --- a/crates/ployzd/src/machine/local_machine/upgrade.rs +++ b/crates/ployzd/src/machine/local_machine/upgrade.rs @@ -36,6 +36,8 @@ impl LocalMachine { let _local = local .try_lock_owned() .map_err(|_| crate::mutation::Error::Busy)?; + crate::installer::require_standard_machine_paths(&data_dir, &run_dir.join("ployz.sock")) + .map_err(crate::installer::upgrade::Error::NonstandardPaths)?; let installation = gate.try_installation()?; crate::installer::upgrade::request_locked(request, &data_dir, &run_dir, &installation) .await diff --git a/crates/ployzd/src/machine_api/local.rs b/crates/ployzd/src/machine_api/local.rs index 7ea5db61de..542a2fc38a 100644 --- a/crates/ployzd/src/machine_api/local.rs +++ b/crates/ployzd/src/machine_api/local.rs @@ -949,7 +949,7 @@ fn upgrade_error(error: crate::installer::upgrade::Error) -> RpcError { Error::AttemptConflict(_) | Error::Busy | Error::Admission(mutation::Error::Busy) => { RpcErrorCode::Conflict } - Error::Resolve(_) => RpcErrorCode::InvalidArgument, + Error::Resolve(_) | Error::NonstandardPaths(_) => RpcErrorCode::InvalidArgument, Error::Read(_) | Error::Decode(_) | Error::Write(_) diff --git a/crates/ployzd/src/machine_api/local_tests.rs b/crates/ployzd/src/machine_api/local_tests.rs index cf17d89a3e..d50eefc9fb 100644 --- a/crates/ployzd/src/machine_api/local_tests.rs +++ b/crates/ployzd/src/machine_api/local_tests.rs @@ -160,6 +160,41 @@ async fn upgrade_and_machine_mutations_refuse_each_other_at_the_rpc_boundary() { std::fs::remove_dir_all(data_dir).unwrap(); } +#[tokio::test] +async fn upgrade_rpc_rejects_nonstandard_machine_paths_before_acceptance() { + let data_dir = + std::env::temp_dir().join(format!("ployzd-upgrade-rpc-paths-{}", MachineId::random())); + let store = Arc::new(Mutex::new(LocalMachineStore::open(&data_dir).unwrap())); + let service = MachineService::with_cluster(store, watch::channel(false).0, None); + + let response = service + .request_machine_upgrade(Request::new( + op::RequestMachineUpgrade::into_request(RequestMachineUpgradeRequest { + attempt_id: MachineUpgradeAttemptId::parse("b".repeat(32)).unwrap(), + release: MachineRelease::parse("1.2.3").unwrap(), + }) + .encode() + .unwrap(), + )) + .await + .unwrap() + .into_inner() + .decode_response() + .unwrap(); + + assert!(matches!( + response.body, + RpcResponseBody::Error(error) + if error.code == RpcErrorCode::InvalidArgument + && error.message.contains("system installation requires --data-dir") + )); + assert!(!data_dir.join("upgrade-attempt.json").exists()); + assert!(!data_dir.join(".upgrade-active").exists()); + + drop(service); + std::fs::remove_dir_all(data_dir).unwrap(); +} + #[test] fn not_allocator_does_not_allocate() { let RpcResponseBody::Error(error) = local_error(LocalMachineError::NotAllocator) From 54e7d41ed3967019102111d5c6241a05a4db7b4f Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:29:20 +0000 Subject: [PATCH 25/30] test(upgrade): isolate qualification from public DNS --- scripts/qualify-release.sh | 2 +- scripts/test-qualify-release.sh | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/qualify-release.sh b/scripts/qualify-release.sh index 4b0f07b286..9f19440e1c 100755 --- a/scripts/qualify-release.sh +++ b/scripts/qualify-release.sh @@ -268,7 +268,7 @@ export PLOYZ_RELEASE_DIR=$ARTIFACT_DIR first=${HOST_LIST[0]} echo "machine init $first at $source_version" -init_cmd=("$PLOYZ" machine init --version "$source_version" --storage zfs --name qualify-1 --context "$CONTEXT") +init_cmd=("$PLOYZ" machine init --version "$source_version" --storage zfs --name qualify-1 --context "$CONTEXT" --no-dns --no-ingress) if [ "$RESET" != 0 ]; then init_cmd+=(--yes) fi diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index be04d98d28..7c1163cd8c 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -65,7 +65,7 @@ case "$1" in shift 2 case "$action" in init|add) - reset=no key= target= context= no_install=no version= storage= name= + reset=no key= target= context= no_install=no version= storage= name= no_dns=no no_ingress=no while [ "$#" -gt 0 ]; do case "$1" in --yes) reset=yes ;; @@ -75,12 +75,14 @@ case "$1" in --storage) storage=$2; shift ;; --name) name=$2; shift ;; --no-install) no_install=yes ;; + --no-dns) no_dns=yes ;; + --no-ingress) no_ingress=yes ;; root@*) target=$1 ;; *) echo "unexpected machine argument: $1" >&2; exit 1 ;; esac shift done - printf '%s target=%s reset=%s key=%s context=%s no_install=%s version=%s storage=%s name=%s release=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" "$version" "$storage" "$name" "${PLOYZ_RELEASE_DIR:-}" >>"$LOG" + printf '%s target=%s reset=%s key=%s context=%s no_install=%s version=%s storage=%s name=%s no_dns=%s no_ingress=%s release=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" "$version" "$storage" "$name" "$no_dns" "$no_ingress" "${PLOYZ_RELEASE_DIR:-}" >>"$LOG" ;; upgrade) if [ "${1:-}" = inspect ]; then @@ -228,8 +230,8 @@ for reset in 0 1; do "$ROOT/scripts/qualify-release.sh" >/dev/null expected_reset=no [ "$reset" = 0 ] || expected_reset=yes - grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-1 release=$SOURCE" "$LOG" || fail "init lost its target, release, storage, name, version, reset policy, or SSH identity" - grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-2 release=$SOURCE" "$LOG" || fail "add lost its target, release, storage, name, version, or SSH identity" + grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-1 no_dns=yes no_ingress=yes release=$SOURCE" "$LOG" || fail "init lost its target, release, storage, name, version, reset policy, isolation policy, or SSH identity" + grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-2 no_dns=no no_ingress=no release=$SOURCE" "$LOG" || fail "add lost its target, release, storage, name, version, or SSH identity" grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "persistent-volume fixture was not deployed" grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=target' "$LOG" || fail "target upgrade was not requested" grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=corrupt' "$LOG" || fail "corrupt preflight was not requested" From 16fd272377b194fa40736cfaa4fa3aafa27dd38b Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:57:00 +0000 Subject: [PATCH 26/30] Let Docker own provisioned volume creation --- crates/ployzd/src/docker/test_support.rs | 12 +- crates/ployzd/src/docker/volume.rs | 84 +++------- crates/ployzd/src/docker/volume/tests.rs | 143 +----------------- .../src/machine/local_machine/container.rs | 9 +- crates/ployzd/src/volume_plugin.rs | 25 +-- crates/ployzd/src/volume_plugin/capacity.rs | 51 +++---- .../src/volume_plugin/tests/capacity_tests.rs | 59 +++++++- 7 files changed, 126 insertions(+), 257 deletions(-) diff --git a/crates/ployzd/src/docker/test_support.rs b/crates/ployzd/src/docker/test_support.rs index b0f1e4487d..5b17fbe057 100644 --- a/crates/ployzd/src/docker/test_support.rs +++ b/crates/ployzd/src/docker/test_support.rs @@ -228,14 +228,16 @@ async fn fake_docker( serde_json::json!({"bound_bytes":bound,"used_bytes":0}), ); } - fake.volumes - .lock() - .unwrap() - .insert(name.clone(), observed.clone()); + let mut response = observed.clone(); + if driver == ployz_core::PROVISIONED_VOLUME_DRIVER { + // Docker's plugin Create response lacks Status; the following inspect supplies it. + response.as_object_mut().unwrap().remove("Status"); + } + fake.volumes.lock().unwrap().insert(name.clone(), observed); if fake.fail_after_create.lock().unwrap().contains(&name) { fake.fail_inspect_once.lock().unwrap().insert(name.clone()); } - (StatusCode::CREATED, observed) + (StatusCode::CREATED, response) } } else if method == Method::POST && (path.ends_with("/start") || path.ends_with("/stop")) { (StatusCode::NO_CONTENT, serde_json::Value::Null) diff --git a/crates/ployzd/src/docker/volume.rs b/crates/ployzd/src/docker/volume.rs index d35b8c18d7..7506b4dcfd 100644 --- a/crates/ployzd/src/docker/volume.rs +++ b/crates/ployzd/src/docker/volume.rs @@ -65,53 +65,22 @@ impl ContainerRuntime { match self.inspect_volume(machine_id, name).await { Ok(volume) => verify_volume(source, &volume), Err(error) if volume_not_found(&error) => { - self.create_managed_volume(machine_id, source).await - } - Err(error) => Err(error), - } - } - - async fn create_managed_volume( - &self, - machine_id: &MachineId, - source: &VolumeSource, - ) -> Result<(), Error> { - let name = source - .docker_volume_name() - .expect("managed Docker Volume sources have names"); - let request = source - .to_create_volume_request() - .expect("managed Docker Volume sources have creation requests"); - match self.create_volume(machine_id, request).await { - Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), - Ok(CreateVolumeReport::Unverified { id, error }) => { - Err(Error::VolumeCreatedButUnverified { - id, - error: Box::new(error), - }) - } - Err(error) if volume_conflict(&error) => { - verify_volume(source, &self.inspect_volume(machine_id, name).await?) - } - Err(error) => Err(error), - } - } - - async fn finish_prepared_volume( - &self, - machine_id: &MachineId, - source: &VolumeSource, - ) -> Result<(), Error> { - let name = source - .docker_volume_name() - .expect("provisioned Volume has a name"); - match self.inspect_volume(machine_id, name).await { - Ok(volume) if prepared_volume_needs_registration(source, &volume) => { - self.create_managed_volume(machine_id, source).await - } - Ok(volume) => verify_volume(source, &volume), - Err(error) if volume_not_found(&error) => { - self.create_managed_volume(machine_id, source).await + let request = source + .to_create_volume_request() + .expect("managed Docker Volume sources have creation requests"); + match self.create_volume(machine_id, request).await { + Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), + Ok(CreateVolumeReport::Unverified { id, error }) => { + Err(Error::VolumeCreatedButUnverified { + id, + error: Box::new(error), + }) + } + Err(error) if volume_conflict(&error) => { + verify_volume(source, &self.inspect_volume(machine_id, name).await?) + } + Err(error) => Err(error), + } } Err(error) => Err(error), } @@ -157,7 +126,7 @@ impl ContainerRuntime { ) -> Result<(), Error> { for spec in specs { for volume in spec.volume_graph().mounted_provisioned_volumes() { - self.finish_prepared_volume(machine_id, &volume.source) + self.ensure_volume_source(machine_id, &volume.source) .await?; } } @@ -363,25 +332,6 @@ fn verify_volume(source: &VolumeSource, observed: &DockerVolume) -> Result<(), E Ok(()) } -fn prepared_volume_needs_registration(source: &VolumeSource, observed: &DockerVolume) -> bool { - let ployz_core::RawVolumeSource::Provisioned { - name, - maximum_bytes, - .. - } = source.kind() - else { - return false; - }; - observed.id.name == *name - && observed.options.is_empty() - && observed.labels.is_empty() - && matches!( - &observed.storage, - DockerVolumeStorageObservation::Provisioned { bound_bytes, .. } - if bound_bytes.get() == maximum_bytes.get() - ) -} - fn volume_not_found(error: &Error) -> bool { docker_status(error, 404) } diff --git a/crates/ployzd/src/docker/volume/tests.rs b/crates/ployzd/src/docker/volume/tests.rs index 64addd88f6..f9751fca94 100644 --- a/crates/ployzd/src/docker/volume/tests.rs +++ b/crates/ployzd/src/docker/volume/tests.rs @@ -297,141 +297,6 @@ async fn missing_provisioned_volume_uses_ployz_driver_bound_and_labels() { ); } -#[tokio::test] -async fn preparation_registers_metadata_for_storage_created_by_plugin() { - let (runtime, fake) = fake_runtime().await; - let machine_id = MachineId::random(); - let source = provisioned_source("prepared", 2_147_483_648); - fake.volumes.lock().unwrap().insert( - "app_prepared".into(), - serde_json::json!({ - "Name":"app_prepared", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_prepared", - "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, - "Options":null, - "Labels":null - }), - ); - let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( - vec![source.clone()], - ))]; - - assert!(matches!( - runtime.ensure_volume_source(&machine_id, &source).await, - Err(Error::VolumeShapeMismatch { .. }) - )); - assert!( - fake.requests - .lock() - .unwrap() - .iter() - .all(|(method, _)| method != Method::POST) - ); - - runtime - .ensure_provisioned_volumes(&machine_id, &specs) - .await - .unwrap(); - - let create = fake - .request_bodies - .lock() - .unwrap() - .iter() - .find(|(path, _)| path.ends_with("/volumes/create")) - .map(|(_, request)| request.clone()) - .expect("Docker metadata registration request"); - assert_eq!( - create, - serde_json::json!({ - "Name":"app_prepared", - "Driver":"ployz", - "DriverOpts":{"size":"2147483648b"}, - "Labels":{"backup":"daily","ployz.managed":"","ployz.project.name":"app"} - }) - ); - let observed = runtime - .inspect_volume(&machine_id, source.docker_volume_name().unwrap()) - .await - .unwrap(); - assert!(source.matches_managed_volume(&observed)); -} - -#[tokio::test] -async fn preparation_refuses_mismatched_or_partially_registered_volumes() { - let (runtime, fake) = fake_runtime().await; - let cases = [ - ( - "wrong-driver", - serde_json::json!({ - "Name":"app_wrong-driver", - "Driver":"local", - "Mountpoint":"/var/lib/docker/volumes/app_wrong-driver/_data", - "Options":null, - "Labels":null - }), - ), - ( - "wrong-bound", - serde_json::json!({ - "Name":"app_wrong-bound", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_wrong-bound", - "Status":{"bound_bytes":1073741824_u64,"used_bytes":0}, - "Options":null, - "Labels":null - }), - ), - ( - "partial-options", - serde_json::json!({ - "Name":"app_partial-options", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_partial-options", - "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, - "Options":{"size":"2147483648b"}, - "Labels":null - }), - ), - ( - "partial-labels", - serde_json::json!({ - "Name":"app_partial-labels", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_partial-labels", - "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, - "Options":null, - "Labels":{"ployz.managed":"","ployz.project.name":"app"} - }), - ), - ]; - - for (name, observed) in cases { - let source = provisioned_source(name, 2_147_483_648); - fake.volumes - .lock() - .unwrap() - .insert(format!("app_{name}"), observed); - let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( - vec![source], - ))]; - assert!(matches!( - runtime - .ensure_provisioned_volumes(&MachineId::random(), &specs) - .await, - Err(Error::VolumeShapeMismatch { .. }) - )); - } - assert!( - fake.requests - .lock() - .unwrap() - .iter() - .all(|(method, _)| method != Method::POST) - ); -} - #[tokio::test] async fn existing_managed_volume_allows_extra_labels() { let (runtime, fake) = fake_runtime().await; @@ -496,6 +361,14 @@ async fn existing_managed_volume_refuses_every_unsafe_shape_mismatch() { "Status":{"bound_bytes":1073741824,"used_bytes":0} }), ), + ( + provisioned_source("missing-metadata", 2_147_483_648), + serde_json::json!({ + "Name":"app_missing-metadata","Driver":"ployz","Mountpoint":"/var/lib/ployz-volumes/app_missing-metadata", + "Options":null,"Labels":null, + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0} + }), + ), ]; for (source, observed) in cases { let name = source.docker_volume_name().unwrap().to_string(); diff --git a/crates/ployzd/src/machine/local_machine/container.rs b/crates/ployzd/src/machine/local_machine/container.rs index d7d33c340b..3efcdd354f 100644 --- a/crates/ployzd/src/machine/local_machine/container.rs +++ b/crates/ployzd/src/machine/local_machine/container.rs @@ -23,7 +23,7 @@ impl LocalMachine { &self, specs: Vec, ) -> Result { - use ployz_core::{RawVolumeSource, RpcError}; + use ployz_core::RawVolumeSource; let admission = self.lock_store()?.admission_lock.clone(); let guard = admission.lock_owned().await; let local = self.clone(); @@ -71,12 +71,7 @@ impl LocalMachine { crate::storage::plugin("Storage.Prepare", &requested).await?; containers .ensure_provisioned_volumes(&machine.id, &specs) - .await - .map_err(|error| { - let mut error = RpcError::from(&error); - error.details = serde_json::json!({ "prepared_volumes": names }); - error - })?; + .await?; Ok(ployz_core::PreparedVolumes { names }) }) .await? diff --git a/crates/ployzd/src/volume_plugin.rs b/crates/ployzd/src/volume_plugin.rs index 9cd7d04741..aee870f139 100644 --- a/crates/ployzd/src/volume_plugin.rs +++ b/crates/ployzd/src/volume_plugin.rs @@ -121,17 +121,7 @@ impl VolumeStorage { let volume = format!("{root}/{name}"); if let Some(existing) = Self::dataset(&datasets, pool, name)? { - existing.require_mountpoint(&name.mountpoint())?; - existing.require_writable()?; - return if existing.refquota == requested { - Ok(()) - } else { - Err(format!( - "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", - existing.refquota - ) - .into()) - }; + return existing.require_requested(name, requested); } if matches!(origin, CapacityAdmission::Required) { @@ -345,6 +335,19 @@ impl Dataset { } self.require_mountpoint(&name.mountpoint()) } + + fn require_requested(&self, name: &DockerVolumeName, requested: u64) -> Result<()> { + self.require_mountpoint(&name.mountpoint())?; + self.require_writable()?; + if self.refquota == requested { + return Ok(()); + } + Err(format!( + "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", + self.refquota + ) + .into()) + } } fn parse_zfs_bytes(value: &str) -> Result { diff --git a/crates/ployzd/src/volume_plugin/capacity.rs b/crates/ployzd/src/volume_plugin/capacity.rs index b091bac650..a6da5cc6a1 100644 --- a/crates/ployzd/src/volume_plugin/capacity.rs +++ b/crates/ployzd/src/volume_plugin/capacity.rs @@ -4,7 +4,7 @@ use axum::{Json, extract::State}; use ployz_core::{ProvisionedVolumeMaximumBytes, StorageCapacity, StorageCapacityError}; use std::collections::BTreeMap; -use super::{CapacityAdmission, DockerVolumeName, VolumeStorage}; +use super::{DockerVolumeName, VolumeStorage}; type Volumes = BTreeMap; @@ -78,52 +78,41 @@ impl VolumeStorage { if requested.is_empty() { return Ok(Vec::new()); } + let existing_pool = self.pool.one_usable().await.map_err(storage_error)?; + if let Some(pool) = &existing_pool { + let datasets = self.datasets(pool).await.map_err(storage_error)?; + for (name, maximum) in requested { + let plugin_name = name + .as_str() + .parse::() + .map_err(storage_error)?; + if let Some(existing) = + Self::dataset(&datasets, pool, &plugin_name).map_err(storage_error)? + { + existing + .require_requested(&plugin_name, maximum.get()) + .map_err(storage_error)?; + } + } + } let commitment = capacity .volumes .values() .map(|maximum| maximum.get()) .try_fold(budget.additional_commitment_bytes, u64::checked_add) .ok_or_else(|| unknown("Volume commitments overflow u64"))?; - let pool = match self.pool.one_usable().await.map_err(storage_error)? { + match existing_pool { Some(pool) => { self.pool .ensure_capacity(&pool, commitment, capacity.unmanaged_used_bytes) .await .map_err(storage_error)?; - pool } None => { self.pool.create(commitment).await.map_err(storage_error)?; - self.one_pool().await.map_err(storage_error)? - } - }; - let mut prepared = Vec::new(); - for (name, maximum) in requested { - let plugin_name = name - .as_str() - .parse::() - .map_err(storage_error)?; - // The whole batch has physical backing; each dataset records its durable commitment. - if let Err(error) = self - .create_volume( - &pool, - &plugin_name, - maximum.get(), - CapacityAdmission::Ensured, - ) - .await - { - let mut error = storage_error(error); - error - .details - .as_object_mut() - .expect("storage errors have object details") - .insert("prepared_volumes".into(), serde_json::json!(prepared)); - return Err(error); } - prepared.push(name.clone()); } - Ok(prepared) + Ok(requested.keys().cloned().collect()) } } diff --git a/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs b/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs index d957e9503b..cdb2729bce 100644 --- a/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs +++ b/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs @@ -71,7 +71,42 @@ async fn storage_inspection_preserves_unrecoverable_backing() { } #[tokio::test] -async fn batch_preparation_checks_total_before_allocating_and_reuses_committed_volumes() { +async fn batch_preparation_rejects_an_existing_read_only_volume_before_dataset_creation() { + let test = TestDir::new(); + for marker in ["root", "volume", "readonly-volume"] { + fs::write(test.0.join(marker), "").unwrap(); + } + let (zpool, zfs) = fake_zfs(&test.0, USABLE_POOL); + let socket = test.0.join("plugin.sock"); + let server = tokio::spawn(serve( + UnixListener::bind(&socket).unwrap(), + VolumeStorage::with_programs(zpool, zfs), + )); + + let response = post( + &socket, + "/Storage.Prepare", + json!({"data":1073741824u64,"other":1073741824u64}), + ) + .await; + + assert!( + response + .pointer("/Err/message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("read-only")), + "{response}" + ); + assert!( + !fs::read_to_string(test.0.join("commands")) + .unwrap() + .contains("zfs create") + ); + server.abort(); +} + +#[tokio::test] +async fn batch_preparation_reserves_capacity_before_docker_creates_volumes() { let test = TestDir::new(); let socket = test.0.join("plugin.sock"); let server = tokio::spawn(serve( @@ -103,6 +138,28 @@ async fn batch_preparation_checks_total_before_allocating_and_reuses_committed_v post(&socket, "/Storage.Prepare", requested.clone()).await, json!({"Ok":["data","other"]}) ); + assert_eq!( + post(&socket, "/VolumeDriver.Get", json!({"Name":"data"})).await, + json!({"Err":"Provisioned Volume data does not exist"}) + ); + assert_eq!( + post(&socket, "/VolumeDriver.List", json!({})).await, + json!({"Volumes":[],"Err":""}) + ); + let capacity = post(&socket, "/Storage.Inspect", json!(null)).await; + assert_eq!(capacity.pointer("/Ok/volumes").unwrap(), &json!({})); + + for (name, size) in [("data", "1073741824b"), ("other", "2147483648b")] { + assert_eq!( + post( + &socket, + "/VolumeDriver.Create", + json!({"Name":name,"Opts":{"size":size}}), + ) + .await, + json!({"Err":""}) + ); + } let capacity = post(&socket, "/Storage.Inspect", json!(null)).await; assert_eq!( capacity.pointer("/Ok/volumes").unwrap(), From 8df1743bebd741bf304cafead59ff3321a905ec4 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 04:57:00 +0000 Subject: [PATCH 27/30] Let Docker own provisioned volume creation --- crates/ployzd/src/docker/test_support.rs | 12 +- crates/ployzd/src/docker/volume.rs | 84 +++------- crates/ployzd/src/docker/volume/tests.rs | 143 +----------------- .../src/machine/local_machine/container.rs | 9 +- crates/ployzd/src/volume_plugin/capacity.rs | 51 +++---- crates/ployzd/src/volume_plugin/storage.rs | 25 +-- .../src/volume_plugin/tests/capacity_tests.rs | 59 +++++++- 7 files changed, 126 insertions(+), 257 deletions(-) diff --git a/crates/ployzd/src/docker/test_support.rs b/crates/ployzd/src/docker/test_support.rs index b0f1e4487d..5b17fbe057 100644 --- a/crates/ployzd/src/docker/test_support.rs +++ b/crates/ployzd/src/docker/test_support.rs @@ -228,14 +228,16 @@ async fn fake_docker( serde_json::json!({"bound_bytes":bound,"used_bytes":0}), ); } - fake.volumes - .lock() - .unwrap() - .insert(name.clone(), observed.clone()); + let mut response = observed.clone(); + if driver == ployz_core::PROVISIONED_VOLUME_DRIVER { + // Docker's plugin Create response lacks Status; the following inspect supplies it. + response.as_object_mut().unwrap().remove("Status"); + } + fake.volumes.lock().unwrap().insert(name.clone(), observed); if fake.fail_after_create.lock().unwrap().contains(&name) { fake.fail_inspect_once.lock().unwrap().insert(name.clone()); } - (StatusCode::CREATED, observed) + (StatusCode::CREATED, response) } } else if method == Method::POST && (path.ends_with("/start") || path.ends_with("/stop")) { (StatusCode::NO_CONTENT, serde_json::Value::Null) diff --git a/crates/ployzd/src/docker/volume.rs b/crates/ployzd/src/docker/volume.rs index d35b8c18d7..7506b4dcfd 100644 --- a/crates/ployzd/src/docker/volume.rs +++ b/crates/ployzd/src/docker/volume.rs @@ -65,53 +65,22 @@ impl ContainerRuntime { match self.inspect_volume(machine_id, name).await { Ok(volume) => verify_volume(source, &volume), Err(error) if volume_not_found(&error) => { - self.create_managed_volume(machine_id, source).await - } - Err(error) => Err(error), - } - } - - async fn create_managed_volume( - &self, - machine_id: &MachineId, - source: &VolumeSource, - ) -> Result<(), Error> { - let name = source - .docker_volume_name() - .expect("managed Docker Volume sources have names"); - let request = source - .to_create_volume_request() - .expect("managed Docker Volume sources have creation requests"); - match self.create_volume(machine_id, request).await { - Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), - Ok(CreateVolumeReport::Unverified { id, error }) => { - Err(Error::VolumeCreatedButUnverified { - id, - error: Box::new(error), - }) - } - Err(error) if volume_conflict(&error) => { - verify_volume(source, &self.inspect_volume(machine_id, name).await?) - } - Err(error) => Err(error), - } - } - - async fn finish_prepared_volume( - &self, - machine_id: &MachineId, - source: &VolumeSource, - ) -> Result<(), Error> { - let name = source - .docker_volume_name() - .expect("provisioned Volume has a name"); - match self.inspect_volume(machine_id, name).await { - Ok(volume) if prepared_volume_needs_registration(source, &volume) => { - self.create_managed_volume(machine_id, source).await - } - Ok(volume) => verify_volume(source, &volume), - Err(error) if volume_not_found(&error) => { - self.create_managed_volume(machine_id, source).await + let request = source + .to_create_volume_request() + .expect("managed Docker Volume sources have creation requests"); + match self.create_volume(machine_id, request).await { + Ok(CreateVolumeReport::Verified { volume }) => verify_volume(source, &volume), + Ok(CreateVolumeReport::Unverified { id, error }) => { + Err(Error::VolumeCreatedButUnverified { + id, + error: Box::new(error), + }) + } + Err(error) if volume_conflict(&error) => { + verify_volume(source, &self.inspect_volume(machine_id, name).await?) + } + Err(error) => Err(error), + } } Err(error) => Err(error), } @@ -157,7 +126,7 @@ impl ContainerRuntime { ) -> Result<(), Error> { for spec in specs { for volume in spec.volume_graph().mounted_provisioned_volumes() { - self.finish_prepared_volume(machine_id, &volume.source) + self.ensure_volume_source(machine_id, &volume.source) .await?; } } @@ -363,25 +332,6 @@ fn verify_volume(source: &VolumeSource, observed: &DockerVolume) -> Result<(), E Ok(()) } -fn prepared_volume_needs_registration(source: &VolumeSource, observed: &DockerVolume) -> bool { - let ployz_core::RawVolumeSource::Provisioned { - name, - maximum_bytes, - .. - } = source.kind() - else { - return false; - }; - observed.id.name == *name - && observed.options.is_empty() - && observed.labels.is_empty() - && matches!( - &observed.storage, - DockerVolumeStorageObservation::Provisioned { bound_bytes, .. } - if bound_bytes.get() == maximum_bytes.get() - ) -} - fn volume_not_found(error: &Error) -> bool { docker_status(error, 404) } diff --git a/crates/ployzd/src/docker/volume/tests.rs b/crates/ployzd/src/docker/volume/tests.rs index 64addd88f6..f9751fca94 100644 --- a/crates/ployzd/src/docker/volume/tests.rs +++ b/crates/ployzd/src/docker/volume/tests.rs @@ -297,141 +297,6 @@ async fn missing_provisioned_volume_uses_ployz_driver_bound_and_labels() { ); } -#[tokio::test] -async fn preparation_registers_metadata_for_storage_created_by_plugin() { - let (runtime, fake) = fake_runtime().await; - let machine_id = MachineId::random(); - let source = provisioned_source("prepared", 2_147_483_648); - fake.volumes.lock().unwrap().insert( - "app_prepared".into(), - serde_json::json!({ - "Name":"app_prepared", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_prepared", - "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, - "Options":null, - "Labels":null - }), - ); - let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( - vec![source.clone()], - ))]; - - assert!(matches!( - runtime.ensure_volume_source(&machine_id, &source).await, - Err(Error::VolumeShapeMismatch { .. }) - )); - assert!( - fake.requests - .lock() - .unwrap() - .iter() - .all(|(method, _)| method != Method::POST) - ); - - runtime - .ensure_provisioned_volumes(&machine_id, &specs) - .await - .unwrap(); - - let create = fake - .request_bodies - .lock() - .unwrap() - .iter() - .find(|(path, _)| path.ends_with("/volumes/create")) - .map(|(_, request)| request.clone()) - .expect("Docker metadata registration request"); - assert_eq!( - create, - serde_json::json!({ - "Name":"app_prepared", - "Driver":"ployz", - "DriverOpts":{"size":"2147483648b"}, - "Labels":{"backup":"daily","ployz.managed":"","ployz.project.name":"app"} - }) - ); - let observed = runtime - .inspect_volume(&machine_id, source.docker_volume_name().unwrap()) - .await - .unwrap(); - assert!(source.matches_managed_volume(&observed)); -} - -#[tokio::test] -async fn preparation_refuses_mismatched_or_partially_registered_volumes() { - let (runtime, fake) = fake_runtime().await; - let cases = [ - ( - "wrong-driver", - serde_json::json!({ - "Name":"app_wrong-driver", - "Driver":"local", - "Mountpoint":"/var/lib/docker/volumes/app_wrong-driver/_data", - "Options":null, - "Labels":null - }), - ), - ( - "wrong-bound", - serde_json::json!({ - "Name":"app_wrong-bound", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_wrong-bound", - "Status":{"bound_bytes":1073741824_u64,"used_bytes":0}, - "Options":null, - "Labels":null - }), - ), - ( - "partial-options", - serde_json::json!({ - "Name":"app_partial-options", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_partial-options", - "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, - "Options":{"size":"2147483648b"}, - "Labels":null - }), - ), - ( - "partial-labels", - serde_json::json!({ - "Name":"app_partial-labels", - "Driver":"ployz", - "Mountpoint":"/var/lib/ployz-volumes/app_partial-labels", - "Status":{"bound_bytes":2147483648_u64,"used_bytes":0}, - "Options":null, - "Labels":{"ployz.managed":"","ployz.project.name":"app"} - }), - ), - ]; - - for (name, observed) in cases { - let source = provisioned_source(name, 2_147_483_648); - fake.volumes - .lock() - .unwrap() - .insert(format!("app_{name}"), observed); - let specs = [ployz_core::ServiceStorageSpec::from(&spec_with_sources( - vec![source], - ))]; - assert!(matches!( - runtime - .ensure_provisioned_volumes(&MachineId::random(), &specs) - .await, - Err(Error::VolumeShapeMismatch { .. }) - )); - } - assert!( - fake.requests - .lock() - .unwrap() - .iter() - .all(|(method, _)| method != Method::POST) - ); -} - #[tokio::test] async fn existing_managed_volume_allows_extra_labels() { let (runtime, fake) = fake_runtime().await; @@ -496,6 +361,14 @@ async fn existing_managed_volume_refuses_every_unsafe_shape_mismatch() { "Status":{"bound_bytes":1073741824,"used_bytes":0} }), ), + ( + provisioned_source("missing-metadata", 2_147_483_648), + serde_json::json!({ + "Name":"app_missing-metadata","Driver":"ployz","Mountpoint":"/var/lib/ployz-volumes/app_missing-metadata", + "Options":null,"Labels":null, + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0} + }), + ), ]; for (source, observed) in cases { let name = source.docker_volume_name().unwrap().to_string(); diff --git a/crates/ployzd/src/machine/local_machine/container.rs b/crates/ployzd/src/machine/local_machine/container.rs index 671ef572e8..0912d5ffd7 100644 --- a/crates/ployzd/src/machine/local_machine/container.rs +++ b/crates/ployzd/src/machine/local_machine/container.rs @@ -25,7 +25,7 @@ impl LocalMachine { &self, specs: Vec, ) -> Result { - use ployz_core::{RawVolumeSource, RpcError}; + use ployz_core::RawVolumeSource; let local = self.clone(); self.finish_mutation(async move { let containers = local.containers.as_ref().ok_or(Error::DockerUnavailable)?; @@ -70,12 +70,7 @@ impl LocalMachine { crate::storage::plugin("Storage.Prepare", &requested).await?; containers .ensure_provisioned_volumes(&machine.id, &specs) - .await - .map_err(|error| { - let mut error = RpcError::from(&error); - error.details = serde_json::json!({ "prepared_volumes": names }); - error - })?; + .await?; Ok(ployz_core::PreparedVolumes { names }) }) .await diff --git a/crates/ployzd/src/volume_plugin/capacity.rs b/crates/ployzd/src/volume_plugin/capacity.rs index 5bf3bc391c..51ab1d21aa 100644 --- a/crates/ployzd/src/volume_plugin/capacity.rs +++ b/crates/ployzd/src/volume_plugin/capacity.rs @@ -4,7 +4,7 @@ use axum::{Json, extract::State}; use ployz_core::{ProvisionedVolumeMaximumBytes, StorageCapacity, StorageCapacityError}; use std::collections::BTreeMap; -use super::{CapacityAdmission, DockerVolumeName, VolumeStorage}; +use super::{DockerVolumeName, VolumeStorage}; type Volumes = BTreeMap; @@ -104,52 +104,41 @@ impl VolumeStorage { if requested.is_empty() { return Ok(Vec::new()); } + let existing_pool = self.pool.one_usable().await.map_err(storage_error)?; + if let Some(pool) = &existing_pool { + let datasets = self.datasets(pool).await.map_err(storage_error)?; + for (name, maximum) in requested { + let plugin_name = name + .as_str() + .parse::() + .map_err(storage_error)?; + if let Some(existing) = + Self::dataset(&datasets, pool, &plugin_name).map_err(storage_error)? + { + existing + .require_requested(&plugin_name, maximum.get()) + .map_err(storage_error)?; + } + } + } let commitment = capacity .volumes .values() .map(|maximum| maximum.get()) .try_fold(budget.additional_commitment_bytes, u64::checked_add) .ok_or_else(|| unknown("Volume commitments overflow u64"))?; - let pool = match self.pool.one_usable().await.map_err(storage_error)? { + match existing_pool { Some(pool) => { self.pool .ensure_capacity(&pool, commitment, capacity.unmanaged_used_bytes) .await .map_err(storage_error)?; - pool } None => { self.pool.create(commitment).await.map_err(storage_error)?; - self.one_pool().await.map_err(storage_error)? - } - }; - let mut prepared = Vec::new(); - for (name, maximum) in requested { - let plugin_name = name - .as_str() - .parse::() - .map_err(storage_error)?; - // The whole batch has physical backing; each dataset records its durable commitment. - if let Err(error) = self - .create_volume( - &pool, - &plugin_name, - maximum.get(), - CapacityAdmission::Ensured, - ) - .await - { - let mut error = storage_error(error); - error - .details - .as_object_mut() - .expect("storage errors have object details") - .insert("prepared_volumes".into(), serde_json::json!(prepared)); - return Err(error); } - prepared.push(name.clone()); } - Ok(prepared) + Ok(requested.keys().cloned().collect()) } } diff --git a/crates/ployzd/src/volume_plugin/storage.rs b/crates/ployzd/src/volume_plugin/storage.rs index 044c0bc6d8..a324026a4d 100644 --- a/crates/ployzd/src/volume_plugin/storage.rs +++ b/crates/ployzd/src/volume_plugin/storage.rs @@ -78,17 +78,7 @@ impl VolumeStorage { let volume = format!("{root}/{name}"); if let Some(existing) = Self::dataset(&datasets, pool, name)? { - existing.require_mountpoint(&name.mountpoint())?; - existing.require_writable()?; - return if existing.refquota == requested { - Ok(()) - } else { - Err(format!( - "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", - existing.refquota - ) - .into()) - }; + return existing.require_requested(name, requested); } if matches!(origin, CapacityAdmission::Required) { @@ -302,6 +292,19 @@ impl Dataset { } self.require_mountpoint(&name.mountpoint()) } + + pub(super) fn require_requested(&self, name: &DockerVolumeName, requested: u64) -> Result<()> { + self.require_mountpoint(&name.mountpoint())?; + self.require_writable()?; + if self.refquota == requested { + return Ok(()); + } + Err(format!( + "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", + self.refquota + ) + .into()) + } } fn parse_zfs_bytes(value: &str) -> Result { diff --git a/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs b/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs index d957e9503b..cdb2729bce 100644 --- a/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs +++ b/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs @@ -71,7 +71,42 @@ async fn storage_inspection_preserves_unrecoverable_backing() { } #[tokio::test] -async fn batch_preparation_checks_total_before_allocating_and_reuses_committed_volumes() { +async fn batch_preparation_rejects_an_existing_read_only_volume_before_dataset_creation() { + let test = TestDir::new(); + for marker in ["root", "volume", "readonly-volume"] { + fs::write(test.0.join(marker), "").unwrap(); + } + let (zpool, zfs) = fake_zfs(&test.0, USABLE_POOL); + let socket = test.0.join("plugin.sock"); + let server = tokio::spawn(serve( + UnixListener::bind(&socket).unwrap(), + VolumeStorage::with_programs(zpool, zfs), + )); + + let response = post( + &socket, + "/Storage.Prepare", + json!({"data":1073741824u64,"other":1073741824u64}), + ) + .await; + + assert!( + response + .pointer("/Err/message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("read-only")), + "{response}" + ); + assert!( + !fs::read_to_string(test.0.join("commands")) + .unwrap() + .contains("zfs create") + ); + server.abort(); +} + +#[tokio::test] +async fn batch_preparation_reserves_capacity_before_docker_creates_volumes() { let test = TestDir::new(); let socket = test.0.join("plugin.sock"); let server = tokio::spawn(serve( @@ -103,6 +138,28 @@ async fn batch_preparation_checks_total_before_allocating_and_reuses_committed_v post(&socket, "/Storage.Prepare", requested.clone()).await, json!({"Ok":["data","other"]}) ); + assert_eq!( + post(&socket, "/VolumeDriver.Get", json!({"Name":"data"})).await, + json!({"Err":"Provisioned Volume data does not exist"}) + ); + assert_eq!( + post(&socket, "/VolumeDriver.List", json!({})).await, + json!({"Volumes":[],"Err":""}) + ); + let capacity = post(&socket, "/Storage.Inspect", json!(null)).await; + assert_eq!(capacity.pointer("/Ok/volumes").unwrap(), &json!({})); + + for (name, size) in [("data", "1073741824b"), ("other", "2147483648b")] { + assert_eq!( + post( + &socket, + "/VolumeDriver.Create", + json!({"Name":name,"Opts":{"size":size}}), + ) + .await, + json!({"Err":""}) + ); + } let capacity = post(&socket, "/Storage.Inspect", json!(null)).await; assert_eq!( capacity.pointer("/Ok/volumes").unwrap(), From d692ea3023a98d1fb9b9a6d5f852bd52b1240f39 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 06:06:39 +0000 Subject: [PATCH 28/30] fix(upgrade): close implementation review findings --- Cargo.lock | 1 - crates/ployz/src/handlers/machine/upgrade.rs | 50 +++------ crates/ployz/tests/setup_cli.rs | 31 +----- crates/ployzd/Cargo.toml | 1 - crates/ployzd/src/build.rs | 45 +++++++- crates/ployzd/src/build/tests.rs | 105 ++++++++++++++++++ crates/ployzd/src/installer/host.rs | 13 +-- crates/ployzd/src/installer/mod.rs | 69 +++++------- crates/ployzd/src/installer/release.rs | 67 ++++++----- crates/ployzd/src/installer/storage.rs | 2 +- crates/ployzd/src/installer/upgrade.rs | 15 +-- crates/ployzd/src/machine/local_machine.rs | 15 ++- .../src/machine/local_machine/upgrade.rs | 2 +- crates/ployzd/src/machine/mod.rs | 2 +- crates/ployzd/src/machine_api/local.rs | 1 + crates/ployzd/src/main.rs | 21 ++-- scripts/qualify-release.sh | 50 ++++++++- .../qualify-release/cloud-enroll-fixture.py | 97 ++++++++++++++++ scripts/test-qualify-release.sh | 57 +++++++++- 19 files changed, 467 insertions(+), 177 deletions(-) create mode 100644 scripts/qualify-release/cloud-enroll-fixture.py diff --git a/Cargo.lock b/Cargo.lock index 5524b7248c..caf192026e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2287,7 +2287,6 @@ dependencies = [ "reqwest", "rusqlite", "sd-notify", - "semver", "serde", "serde_json", "sha2", diff --git a/crates/ployz/src/handlers/machine/upgrade.rs b/crates/ployz/src/handlers/machine/upgrade.rs index eea9f745ea..d0c9392e7e 100644 --- a/crates/ployz/src/handlers/machine/upgrade.rs +++ b/crates/ployz/src/handlers/machine/upgrade.rs @@ -159,7 +159,14 @@ async fn run_one( attempt_id, release, }; - let accepted = match request_until(deadline, client, &target, request).await { + let accepted = match client + .request_upgrade( + request, + &target, + deadline.saturating_duration_since(Instant::now()), + ) + .await + { Ok(accepted) => accepted, Err(crate::setup_retry::Error::Permanent(error)) => return Err(error.into()), Err(crate::setup_retry::Error::Exhausted(error)) => { @@ -178,7 +185,14 @@ async fn run_one( { return Err(uncertain_timeout(machine, attempt_id)); } - let observed = inspect_until(deadline, client, &target, attempt_id) + let observed = client + .inspect_upgrade( + InspectMachineUpgradeRequest { + attempt_id: Some(attempt_id), + }, + &target, + deadline.saturating_duration_since(Instant::now()), + ) .await .map_err(|error| match error { crate::setup_retry::Error::Permanent(error) => error.into(), @@ -192,38 +206,6 @@ async fn run_one( } } -async fn request_until( - deadline: Instant, - client: &mut impl UpgradeRequests, - target: &MachineTarget, - request: RequestMachineUpgradeRequest, -) -> Result> { - client - .request_upgrade( - request, - target, - deadline.saturating_duration_since(Instant::now()), - ) - .await -} - -async fn inspect_until( - deadline: Instant, - client: &mut impl UpgradeRequests, - target: &MachineTarget, - attempt_id: MachineUpgradeAttemptId, -) -> Result> { - client - .inspect_upgrade( - InspectMachineUpgradeRequest { - attempt_id: Some(attempt_id), - }, - target, - deadline.saturating_duration_since(Instant::now()), - ) - .await -} - fn print_attempt(machine: &Machine, attempt: &MachineUpgradeAttempt) { print_attempt_target(&format!("{} ({})", machine.name, machine.id), attempt); } diff --git a/crates/ployz/tests/setup_cli.rs b/crates/ployz/tests/setup_cli.rs index 0db5740831..3e6c7276c3 100644 --- a/crates/ployz/tests/setup_cli.rs +++ b/crates/ployz/tests/setup_cli.rs @@ -8,27 +8,6 @@ use std::{ }; use sha2::{Digest, Sha256}; -use uuid::Uuid; - -struct Fixture(PathBuf); - -impl Fixture { - fn new() -> Self { - let path = env::temp_dir().join(format!("ployz-setup-cli-{}", Uuid::new_v4())); - fs::create_dir(&path).unwrap(); - Self(path) - } - - fn path(&self) -> &Path { - &self.0 - } -} - -impl Drop for Fixture { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.0); - } -} fn write_executable(path: &Path, body: &str) { fs::write(path, body).unwrap(); @@ -97,7 +76,7 @@ fn run(args: &[&str], root: &Path, release: &Path, bin: &Path) -> Output { #[test] fn local_setup_runs_the_verified_bootstrap_installer() { - let fixture = Fixture::new(); + let fixture = tempfile::tempdir().unwrap(); let log = fixture.path().join("commands.log"); let release = release_fixture(fixture.path(), &log); let bin = fixture.path().join("bin"); @@ -136,7 +115,7 @@ fn local_setup_runs_the_verified_bootstrap_installer() { #[test] fn corrupt_local_bootstrap_is_rejected_before_execution() { - let fixture = Fixture::new(); + let fixture = tempfile::tempdir().unwrap(); let log = fixture.path().join("commands.log"); let release = release_fixture(fixture.path(), &log); fs::write(release.join(daemon_archive()), b"corrupt").unwrap(); @@ -174,7 +153,7 @@ fn corrupt_local_bootstrap_is_rejected_before_execution() { #[test] fn ssh_setup_transfers_one_bootstrap_and_no_install_skips_it() { - let fixture = Fixture::new(); + let fixture = tempfile::tempdir().unwrap(); let log = fixture.path().join("commands.log"); let release = release_fixture(fixture.path(), &log); let bin = fixture.path().join("bin"); @@ -285,7 +264,7 @@ esac #[test] fn remote_preflight_timeout_kills_the_child_and_cleans_the_stage() { - let fixture = Fixture::new(); + let fixture = tempfile::tempdir().unwrap(); let log = fixture.path().join("commands.log"); let release = release_fixture(fixture.path(), &fixture.path().join("installer.log")); let bin = fixture.path().join("bin"); @@ -351,7 +330,7 @@ esac #[test] fn remote_cleanup_failure_is_returned_after_successful_installation() { - let fixture = Fixture::new(); + let fixture = tempfile::tempdir().unwrap(); let release = release_fixture(fixture.path(), &fixture.path().join("installer.log")); let bin = fixture.path().join("bin"); fs::create_dir(&bin).unwrap(); diff --git a/crates/ployzd/Cargo.toml b/crates/ployzd/Cargo.toml index 955c87c52b..52bf466d5d 100644 --- a/crates/ployzd/Cargo.toml +++ b/crates/ployzd/Cargo.toml @@ -35,7 +35,6 @@ rusqlite = { version = "0.40.2", default-features = false, features = ["bundled" sd-notify = "0.5.0" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -semver = "1.0.28" sha2 = "0.11.0" shell-words = "1.1.1" thiserror = "2.0.20" diff --git a/crates/ployzd/src/build.rs b/crates/ployzd/src/build.rs index dba2ce00dc..ca39be1d39 100644 --- a/crates/ployzd/src/build.rs +++ b/crates/ployzd/src/build.rs @@ -21,19 +21,26 @@ use tonic::Status; #[derive(Default)] struct AttemptState { evidence: Mutex, - retained: Mutex>, + retained: Mutex>, +} + +struct RetainedBuild { + // Field order keeps installation exclusion until Docker tag cleanup finishes. + _retention: ployz_build::ImageRetention, + _installation: crate::mutation::MutationGuard, } pub(crate) fn start( machine_id: MachineId, requests: impl Stream> + Send + Unpin + 'static, runner: std::sync::Arc, + local: crate::machine::LocalMachine, ) -> RpcStream { let (events, receiver) = mpsc::channel(8); tokio::spawn(async move { let shutdown = runner.shutdown.clone(); let state = Arc::new(AttemptState::default()); - let outcome = attempt(machine_id, requests, &events, runner, state.clone()).await; + let outcome = attempt(machine_id, requests, &events, runner, local, state.clone()).await; let message = "Build terminal report exceeds the response size limit"; let fallback = match &outcome { Outcome::Unknown { stage, .. } => Outcome::Unknown { @@ -50,7 +57,8 @@ pub(crate) fn start( remote::encode(&Event::Finished(fallback)).expect("bounded terminal fallback") }); let _ = tokio::time::timeout(Duration::from_secs(5), events.send(Ok(payload))).await; - // Admission is already released. Only image retention follows this stream. + // Local mutation serialization is already released. Installation exclusion follows + // retained image cleanup because releasing temporary tags mutates Docker. if state .retained .lock() @@ -70,6 +78,7 @@ async fn attempt( mut requests: impl Stream> + Send + Unpin + 'static, events: &mpsc::Sender>, runner: Arc, + local: crate::machine::LocalMachine, state: Arc, ) -> Outcome { let start = tokio::time::timeout(Duration::from_secs(10), requests.next()).await; @@ -119,6 +128,28 @@ async fn attempt( Err(reason) => return failed(Stage::Queued, reason.to_string()).with_work(work), }; let policy = runner.policy.clone(); + let mutation = { + let admission = local.admit_mutation(); + tokio::pin!(admission); + tokio::select! { + biased; + () = runner.shutdown.cancelled() => return failed(Stage::Admission, "Build daemon stopped before admission").with_work(work), + () = events.closed() => return failed(Stage::Admission, "Build client disconnected before admission").with_work(work), + frame = requests.next() => { + let reason = match frame { + Some(Ok(payload)) if matches!(remote::decode::(&payload), Ok(Input::Cancel)) => "Build cancelled before admission", + None | Some(Err(_)) => "Build client disconnected before admission", + Some(Ok(_)) => "Build upload before admission is forbidden", + }; + return failed(Stage::Admission, reason).with_work(work); + } + () = tokio::time::sleep(policy.active_timeout) => return failed(Stage::Admission, "Build admission timed out waiting for another Machine mutation").with_work(work), + result = &mut admission => match result { + Ok(admission) => admission, + Err(error) => return failed(Stage::Admission, error.to_string()).with_work(work), + }, + } + }; let admission = match tokio::task::spawn_blocking({ let policy = policy.clone(); move || Admission::try_acquire_with(&policy) @@ -148,7 +179,7 @@ async fn attempt( let mut execution = tokio::task::spawn_blocking(move || { let _permit = permit; receive_and_execute( - machine_id, definition, source, admission, policy, &output, &observed, + machine_id, definition, source, admission, mutation, policy, &output, &observed, ) }); // Dropping the input pump closes the upload channel. A receiver blocked @@ -221,6 +252,7 @@ fn receive_and_execute( mut definition: Definition, mut source: mpsc::Receiver, admission: Admission, + mutation: crate::machine::MutationAdmission, policy: HostPolicy, events: &mpsc::Sender>, state: &AttemptState, @@ -294,7 +326,10 @@ fn receive_and_execute( match result { Ok(completed) => { if !definition.retained_tags.is_empty() { - *state.retained.lock().expect("Build retention lock") = Some(completed.retention); + *state.retained.lock().expect("Build retention lock") = Some(RetainedBuild { + _retention: completed.retention, + _installation: mutation.into_installation_guard(), + }); } match definition.output { Output::Load => Outcome::Images { diff --git a/crates/ployzd/src/build/tests.rs b/crates/ployzd/src/build/tests.rs index c2780363c3..991ec7f568 100644 --- a/crates/ployzd/src/build/tests.rs +++ b/crates/ployzd/src/build/tests.rs @@ -21,6 +21,7 @@ struct Fixture { root: PathBuf, policy: HostPolicy, cluster: tokio::task::JoinHandle<()>, + local: crate::machine::LocalMachine, machine: ployz_core::Machine, address: String, server: tokio::task::JoinHandle>, @@ -74,6 +75,7 @@ impl Fixture { update(&mut policy); let shutdown = tokio_util::sync::CancellationToken::new(); service.builds = Runner::new(policy.clone(), shutdown.clone()).unwrap(); + let local = service.local(); write_docker(&policy.docker, &root); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = format!("http://{}", listener.local_addr().unwrap()); @@ -86,6 +88,7 @@ impl Fixture { root, policy, cluster, + local, machine, address, server, @@ -199,6 +202,63 @@ async fn admitted_upload_queues_competitors_and_disconnect_releases_unused_owner assert!(matches!(event(&mut admitted).await, Event::Admitted { .. })); } +#[tokio::test] +async fn active_build_refuses_an_upgrade_request() { + let fixture = Fixture::new().await; + let (sender, mut response) = fixture.request(Output::Load).await; + assert!(matches!(event(&mut response).await, Event::Admitted { .. })); + + let error = fixture + .local + .request_upgrade(ployz_core::RequestMachineUpgradeRequest { + attempt_id: ployz_core::MachineUpgradeAttemptId::random(), + release: ployz_core::MachineRelease::parse("1.2.3").unwrap(), + }) + .await + .unwrap_err(); + assert!(matches!( + error, + crate::machine::LocalMachineError::Admission(crate::mutation::Error::Busy) + )); + + let mut ordinary = Box::pin(fixture.local.set_cloud_pairing(None)); + assert!( + tokio::time::timeout(Duration::from_millis(100), ordinary.as_mut()) + .await + .is_err(), + "ordinary mutation ran concurrently with active Build execution" + ); + drop(sender); + let _ = terminal(&mut response).await; + tokio::time::timeout(Duration::from_secs(2), ordinary) + .await + .expect("ordinary mutation stayed blocked after Build execution ended") + .unwrap(); +} + +#[tokio::test] +async fn durable_upgrade_marker_refuses_build_before_execution() { + let fixture = Fixture::new().await; + let marker = fixture.root.join("machine/.upgrade-active"); + fs::write(&marker, "active-upgrade").unwrap(); + + let (_sender, mut rejected) = fixture.request(Output::Load).await; + assert!(matches!( + terminal(&mut rejected).await, + Outcome::Failed { + stage: Stage::Admission, + .. + } + )); + assert!(!fixture.root.join("executed").exists()); + + fs::remove_file(marker).unwrap(); + let (sender, mut admitted) = fixture.request(Output::Load).await; + assert!(matches!(event(&mut admitted).await, Event::Admitted { .. })); + drop(sender); + let _ = terminal(&mut admitted).await; +} + #[tokio::test] async fn captured_build_crosses_owned_rpc_and_returns_only_remote_image_evidence() { let fixture = Fixture::new().await; @@ -248,6 +308,50 @@ async fn captured_build_crosses_owned_rpc_and_returns_only_remote_image_evidence assert!(Admission::try_acquire_with(&fixture.policy).is_ok()); } +#[tokio::test] +async fn retained_build_images_do_not_block_same_machine_mutation() { + let fixture = Fixture::new().await; + let client = ployz::connect::connect( + Path::new("/missing-test-config"), + Some(&fixture.address.replace("http://", "tcp://")), + None, + ) + .await + .unwrap(); + let retained = fixture + .capture() + .execute_remote_images( + &client, + fixture.machine.id, + tokio_util::sync::CancellationToken::new(), + |_| {}, + ) + .await + .unwrap(); + + tokio::time::timeout( + Duration::from_secs(2), + fixture.local.set_cloud_pairing(None), + ) + .await + .expect("retained Build stream held the local mutation mutex") + .unwrap(); + + let error = fixture + .local + .request_upgrade(ployz_core::RequestMachineUpgradeRequest { + attempt_id: ployz_core::MachineUpgradeAttemptId::random(), + release: ployz_core::MachineRelease::parse("1.2.3").unwrap(), + }) + .await + .unwrap_err(); + assert!(matches!( + error, + crate::machine::LocalMachineError::Admission(crate::mutation::Error::Busy) + )); + drop(retained); +} + #[tokio::test] async fn oversized_diagnostics_still_return_a_definite_terminal_failure() { let fixture = Fixture::new().await; @@ -401,6 +505,7 @@ async fn upload_timeout_stops_before_execution_and_releases_admission() { fixture.machine.id, ReceiverStream::new(receiver), Runner::new(policy, Default::default()).unwrap(), + fixture.local.clone(), ); assert!(matches!( remote::decode::(&responses.next().await.unwrap().unwrap()).unwrap(), diff --git a/crates/ployzd/src/installer/host.rs b/crates/ployzd/src/installer/host.rs index ee03efb3cd..4f6e04681d 100644 --- a/crates/ployzd/src/installer/host.rs +++ b/crates/ployzd/src/installer/host.rs @@ -8,11 +8,10 @@ use std::{ process::{Command, Stdio}, }; -use semver::Version; use tonic::transport::Endpoint; use crate::filesystem::atomic_write; -use ployz_core::{DescribeContractRequest, MachineRpcClient, op}; +use ployz_core::{DescribeContractRequest, MachineRpcClient, MachineVersion, op}; use super::release::{fetch, installed_release}; use super::{Error, InstallPaths, PLOYZ_USER, command_exists, run_apt, run_host, systemctl}; @@ -268,7 +267,7 @@ pub(super) async fn install_docker(paths: &InstallPaths) -> Result<(), Error> { pub(super) async fn verify_running_daemon( paths: &InstallPaths, - target: &Version, + target: &MachineVersion, ) -> Result<(), Error> { systemctl( "check daemon readiness", @@ -311,7 +310,7 @@ pub(super) async fn verify_running_daemon( verify_daemon_contract(&paths.run_dir.join("ployz.sock"), target).await } -async fn verify_daemon_contract(socket: &Path, target: &Version) -> Result<(), Error> { +async fn verify_daemon_contract(socket: &Path, target: &MachineVersion) -> Result<(), Error> { let endpoint = Endpoint::from_shared(format!("unix:{}", socket.display())).map_err(|error| { Error::Verification(format!("invalid Machine API socket address: {error}")) @@ -337,8 +336,8 @@ async fn verify_daemon_contract(socket: &Path, target: &Version) -> Result<(), E require_running_version(&response.daemon_version, target) } -fn require_running_version(observed: &str, target: &Version) -> Result<(), Error> { - let observed = Version::parse(observed).map_err(|_| { +fn require_running_version(observed: &str, target: &MachineVersion) -> Result<(), Error> { + let observed = MachineVersion::parse(observed).map_err(|_| { Error::Verification(format!( "running Machine API reported invalid daemon version {observed:?}" )) @@ -358,7 +357,7 @@ mod tests { #[test] fn running_machine_api_version_must_match_the_target() { - let target = Version::parse("1.2.3-beta.4").unwrap(); + let target = MachineVersion::parse("1.2.3-beta.4").unwrap(); assert!(require_running_version("1.2.3-beta.4", &target).is_ok()); assert!(matches!( require_running_version("1.2.3-beta.3", &target), diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs index 454c3f9ea1..52ab56acaf 100644 --- a/crates/ployzd/src/installer/mod.rs +++ b/crates/ployzd/src/installer/mod.rs @@ -34,9 +34,11 @@ const DEFAULT_SYSTEMD_DIR: &str = "/etc/systemd/system"; const DEFAULT_RUN_DIR: &str = "/run/ployz"; /// Unix socket installed systemd services use for the local Machine API. pub const DEFAULT_SOCKET_PATH: &str = "/run/ployz/ployz.sock"; -/// Explicit host work associated with one Machine installation attempt. +/// Mutually exclusive host work for one Machine installation attempt. #[derive(Clone, Debug, Eq, PartialEq)] -pub enum Preparation { +pub enum InstallMode { + /// Write the release and units without starting systemd or preparing the host. + InstallationOnly, /// Replace the daemon only after confirming the Machine is already prepared. SoftwareOnly, /// Prepare the host and optionally configure its requested storage. @@ -55,10 +57,8 @@ pub struct InstallRequest { pub release: ReleaseRequest, /// Trusted release source. Published releases never accept caller-provided URLs. pub source: ReleaseSource, - /// Explicit host preparation or ordinary software-only replacement. - pub preparation: Preparation, - /// Write the release and units without starting systemd. - pub install_only: bool, + /// Installation-only, software-only replacement, or full host preparation. + pub mode: InstallMode, } /// What installation directly observed after activation. @@ -194,36 +194,36 @@ async fn install_locked( _lock: mutation::InstallationGuard, mut progress: impl FnMut(MachineUpgradeStage) -> Result<(), Error>, ) -> Result { - verify_system(request.install_only)?; + let installation_only = matches!(request.mode, InstallMode::InstallationOnly); + verify_system(installation_only)?; let target = resolve_release(&request.release, &request.source).await?; progress(MachineUpgradeStage::Preparing)?; - if !request.install_only { - match &request.preparation { - Preparation::SoftwareOnly => verify_software_prerequisites(&paths)?, - Preparation::PrepareHost { - storage, - group_user, - } => { - prepare_storage(*storage, &paths)?; - install_prerequisites()?; - let inherited_group = sudo_user(); - create_user_and_directories( - group_user.as_deref().or(inherited_group.as_deref()), - &paths, - )?; - } + match &request.mode { + InstallMode::InstallationOnly => {} + InstallMode::SoftwareOnly => verify_software_prerequisites(&paths)?, + InstallMode::PrepareHost { + storage, + group_user, + } => { + prepare_storage(*storage, &paths)?; + install_prerequisites()?; + let inherited_group = sudo_user(); + create_user_and_directories( + group_user.as_deref().or(inherited_group.as_deref()), + &paths, + )?; } } let mut restart_required = !paths.systemd_dir.join("ployz.service").is_file(); restart_required |= install_binaries(&request.source, &paths, &target, &mut progress).await?; - install_systemd(&paths, request.install_only)?; - if !request.install_only && matches!(request.preparation, Preparation::PrepareHost { .. }) { + install_systemd(&paths, installation_only)?; + if matches!(request.mode, InstallMode::PrepareHost { .. }) { install_docker(&paths).await?; } - let readiness = if request.install_only { + let readiness = if installation_only { Readiness::InstallationOnly } else { if restart_required { @@ -386,7 +386,7 @@ mod tests { process::Command, }; - use semver::Version; + use ployz_core::MachineVersion; use tempfile::TempDir; use super::*; @@ -395,10 +395,9 @@ mod tests { async fn system_install_rejects_nonstandard_machine_paths_before_mutation() { let fixture = fixture("nonstandard-paths"); let request = InstallRequest { - release: ReleaseRequest::Exact(Version::parse("1.2.3").unwrap()), + release: ReleaseRequest::Exact(MachineVersion::parse("1.2.3").unwrap()), source: ReleaseSource::Local(fixture.path().join("release")), - preparation: Preparation::SoftwareOnly, - install_only: true, + mode: InstallMode::InstallationOnly, }; for (data_dir, socket) in [ @@ -527,17 +526,9 @@ mod tests { } let request = InstallRequest { - release: ReleaseRequest::Exact(Version::parse("1.2.3").unwrap()), + release: ReleaseRequest::Exact(MachineVersion::parse("1.2.3").unwrap()), source: ReleaseSource::Local(root.join("release")), - preparation: if case == "install-only" { - Preparation::PrepareHost { - storage: StorageChoice::None, - group_user: None, - } - } else { - Preparation::SoftwareOnly - }, - install_only: true, + mode: InstallMode::InstallationOnly, }; let result = if case == "busy" { let admission = mutation::MutationGate::new(&paths.run_dir, &paths.data_dir); diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs index 2e36f1d2fc..53f538e197 100644 --- a/crates/ployzd/src/installer/release.rs +++ b/crates/ployzd/src/installer/release.rs @@ -13,13 +13,12 @@ use std::{ #[cfg(not(test))] use std::os::unix::fs::chown; -use semver::Version; use sha2::{Digest, Sha256}; use tempfile::TempDir; use tokio::{process::Command, time::timeout}; use super::{Error, InstallPaths, daemon_archive, run_command}; -use ployz_core::MachineUpgradeStage; +use ployz_core::{MachineUpgradeStage, MachineVersion}; const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; const CHANNEL_URL: &str = "https://ployz.sh"; @@ -37,7 +36,7 @@ pub enum ReleaseRequest { /// Resolve the beta channel once for this installation attempt. Beta, /// Install this exact published daemon version. - Exact(Version), + Exact(MachineVersion), } impl FromStr for ReleaseRequest { @@ -49,7 +48,11 @@ impl FromStr for ReleaseRequest { "" | "latest" | "stable" => Ok(Self::Stable), "beta" => Ok(Self::Beta), "nightly" => Err(Error::Nightly), - value => parse_release(value).map(Self::Exact), + value => MachineVersion::parse(value.to_owned()) + .map(Self::Exact) + .map_err(|_| Error::InvalidVersion { + value: value.to_owned(), + }), } } } @@ -88,7 +91,7 @@ impl ReleaseSource { async fn release_file( &self, - target: &Version, + target: &MachineVersion, file: &str, stage: &'static str, ) -> Result, Error> { @@ -105,28 +108,24 @@ impl ReleaseSource { pub(super) async fn resolve_release( request: &ReleaseRequest, source: &ReleaseSource, -) -> Result { +) -> Result { match request { - ReleaseRequest::Stable => parse_release(source.channel("stable").await?.trim()), - ReleaseRequest::Beta => parse_release(source.channel("beta").await?.trim()), + ReleaseRequest::Stable => { + let value = source.channel("stable").await?; + parse_channel_version(value.trim()) + } + ReleaseRequest::Beta => { + let value = source.channel("beta").await?; + parse_channel_version(value.trim()) + } ReleaseRequest::Exact(version) => Ok(version.clone()), } } -fn parse_release(value: &str) -> Result { - let version = Version::parse(value).map_err(|_| Error::InvalidVersion { - value: value.into(), - })?; - let pre = version.pre.as_str(); - let beta_is_valid = pre.strip_prefix("beta.").is_some_and(|number| { - !number.is_empty() && number.bytes().all(|byte| byte.is_ascii_digit()) - }); - if !version.build.is_empty() || (!pre.is_empty() && !beta_is_valid) { - return Err(Error::InvalidVersion { - value: value.into(), - }); - } - Ok(version) +fn parse_channel_version(value: &str) -> Result { + MachineVersion::parse(value.to_owned()).map_err(|_| Error::InvalidVersion { + value: value.to_owned(), + }) } pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { @@ -151,14 +150,14 @@ pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}"))) } -fn release_url(target: &Version, file: &str) -> String { +fn release_url(target: &MachineVersion, file: &str) -> String { format!("{RELEASE_REPOSITORY}/releases/download/v{target}/{file}") } pub(super) async fn install_binaries( source: &ReleaseSource, paths: &InstallPaths, - target: &Version, + target: &MachineVersion, progress: &mut impl FnMut(MachineUpgradeStage) -> Result<(), Error>, ) -> Result { let installed = installed_release(&paths.bin_dir.join("ployzd")).await?; @@ -195,13 +194,13 @@ pub(super) async fn install_binaries( fn replacement_required( source: &ReleaseSource, - installed: Option<&Version>, - target: &Version, + installed: Option<&MachineVersion>, + target: &MachineVersion, ) -> bool { source.is_local() || installed.is_none_or(|installed| installed != target) } -pub(super) async fn installed_release(path: &Path) -> Result, Error> { +pub(super) async fn installed_release(path: &Path) -> Result, Error> { if !path.is_file() { return Ok(None); } @@ -212,14 +211,14 @@ pub(super) async fn installed_release(path: &Path) -> Result, Er output.status ))); } - parse_release(String::from_utf8_lossy(&output.stdout).trim()) + MachineVersion::parse(String::from_utf8_lossy(&output.stdout).trim().to_owned()) .map(Some) .map_err(|_| Error::Verification("installed daemon reported an invalid version".into())) } async fn release_checksum( source: &ReleaseSource, - target: &Version, + target: &MachineVersion, archive: &str, ) -> Result { let checksums = source @@ -271,7 +270,7 @@ fn extract_archive(archive: &Path, destination: &Path) -> Result<(), Error> { Ok(()) } -async fn verify_executable(path: &Path, target: &Version) -> Result<(), Error> { +async fn verify_executable(path: &Path, target: &MachineVersion) -> Result<(), Error> { fs::set_permissions(path, fs::Permissions::from_mode(0o755)).map_err(|source| Error::Io { stage: "mark staged daemon executable", source, @@ -284,7 +283,7 @@ async fn verify_executable(path: &Path, target: &Version) -> Result<(), Error> { ))); } let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - if observed == target.to_string() { + if observed == target.as_str() { Ok(()) } else { Err(Error::Verification(format!( @@ -439,7 +438,7 @@ mod tests { assert!(matches!("beta".parse(), Ok(ReleaseRequest::Beta))); assert_eq!( "v1.2.3-beta.4".parse::().unwrap(), - ReleaseRequest::Exact(Version::parse("1.2.3-beta.4").unwrap()) + ReleaseRequest::Exact(MachineVersion::parse("1.2.3-beta.4").unwrap()) ); for invalid in ["1.2", "1.2.3-rc.1", "1.2.3-beta.x", "1.2.3+build.1"] { assert!(invalid.parse::().is_err(), "{invalid}"); @@ -459,7 +458,7 @@ mod tests { #[test] fn resolved_channels_replace_the_exact_target_and_skip_only_the_same_target() { - let target = Version::parse("1.2.3").unwrap(); + let target = MachineVersion::parse("1.2.3").unwrap(); assert!(!replacement_required( &ReleaseSource::Published, Some(&target), @@ -467,7 +466,7 @@ mod tests { )); assert!(replacement_required( &ReleaseSource::Published, - Some(&Version::parse("1.2.4").unwrap()), + Some(&MachineVersion::parse("1.2.4").unwrap()), &target )); assert!(replacement_required( diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs index 32283ba53a..4267ed8e31 100644 --- a/crates/ployzd/src/installer/storage.rs +++ b/crates/ployzd/src/installer/storage.rs @@ -128,7 +128,7 @@ fn require_host_root_reserve(allocation: u64) -> Result<(), Error> { stage: "inspect host-root capacity for ZFS storage preparation", source, })?; - let reserve = (size / 4).max(10 * 1024 * 1024 * 1024); + let reserve = ployz_core::storage_host_reserve(size); if available < reserve.saturating_add(allocation) { return Err(Error::Command { stage: "prepare ZFS storage".into(), diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs index 599cffa4ff..975f5a75fb 100644 --- a/crates/ployzd/src/installer/upgrade.rs +++ b/crates/ployzd/src/installer/upgrade.rs @@ -8,15 +8,14 @@ use std::{ use ployz_core::{ MachineRelease, MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeOutcome, - MachineUpgradeStage, MachineVersion, RequestMachineUpgradeRequest, + MachineUpgradeStage, RequestMachineUpgradeRequest, }; -use semver::Version; use serde::{Deserialize, Serialize}; use thiserror::Error; use tokio::{process::Command, time::timeout}; use super::{ - Error as InstallError, InstallPaths, InstallRequest, Preparation, ReleaseRequest, ReleaseSource, + Error as InstallError, InstallMode, InstallPaths, InstallRequest, ReleaseRequest, ReleaseSource, }; use crate::mutation; @@ -150,8 +149,6 @@ pub(crate) async fn request_locked( let target = super::release::resolve_release(&release, &source) .await .map_err(Error::Resolve)?; - let target = MachineVersion::parse(target.to_string()) - .expect("the installer accepts only supported Machine versions"); let mut stored = StoredAttempt { requested: request.release, source, @@ -227,8 +224,6 @@ pub async fn run_worker( return Err(Error::NotActive(attempt_id)); } let target = stored.attempt.target.clone(); - let target_version = Version::parse(target.as_str()) - .expect("a MachineVersion contains a supported semantic version"); let mut stage = MachineUpgradeStage::Preparing; stored.attempt.outcome = MachineUpgradeOutcome::Running { stage: stage.clone(), @@ -237,10 +232,9 @@ pub async fn run_worker( let result = super::install_locked( InstallRequest { - release: ReleaseRequest::Exact(target_version), + release: ReleaseRequest::Exact(target.clone()), source: stored.source.clone(), - preparation: Preparation::SoftwareOnly, - install_only: false, + mode: InstallMode::SoftwareOnly, }, InstallPaths::system(data_dir, run_dir), guard, @@ -477,6 +471,7 @@ fn write(data_dir: &Path, stored: &StoredAttempt) -> Result<(), Error> { #[cfg(test)] mod tests { use super::*; + use ployz_core::MachineVersion; const CONTRACT_CASE: &str = "PLOYZ_UPGRADE_CONTRACT_CASE"; const CONTRACT_ROOT: &str = "PLOYZ_UPGRADE_CONTRACT_ROOT"; diff --git a/crates/ployzd/src/machine/local_machine.rs b/crates/ployzd/src/machine/local_machine.rs index 6bad65db8a..69066a2670 100644 --- a/crates/ployzd/src/machine/local_machine.rs +++ b/crates/ployzd/src/machine/local_machine.rs @@ -127,11 +127,22 @@ pub enum Error { Upgrade(#[from] crate::installer::upgrade::Error), } -struct MutationAdmission { +pub(crate) struct MutationAdmission { _local: OwnedMutexGuard<()>, _installation: crate::mutation::MutationGuard, } +impl MutationAdmission { + pub(crate) fn into_installation_guard(self) -> crate::mutation::MutationGuard { + let Self { + _local, + _installation, + } = self; + drop(_local); + _installation + } +} + impl LocalMachine { /// An uninitialized Local Machine with no Cluster or Docker collaborators. #[must_use] @@ -190,7 +201,7 @@ impl LocalMachine { self.store.lock().map_err(|_| Error::LockPoisoned) } - async fn admit_mutation(&self) -> Result { + pub(crate) async fn admit_mutation(&self) -> Result { let (local, installation) = { let store = self.lock_store()?; (store.admission_lock.clone(), store.mutation_gate.clone()) diff --git a/crates/ployzd/src/machine/local_machine/upgrade.rs b/crates/ployzd/src/machine/local_machine/upgrade.rs index cd6f767803..5935e3e46d 100644 --- a/crates/ployzd/src/machine/local_machine/upgrade.rs +++ b/crates/ployzd/src/machine/local_machine/upgrade.rs @@ -36,9 +36,9 @@ impl LocalMachine { let _local = local .try_lock_owned() .map_err(|_| crate::mutation::Error::Busy)?; + let installation = gate.try_installation()?; crate::installer::require_standard_machine_paths(&data_dir, &run_dir.join("ployz.sock")) .map_err(crate::installer::upgrade::Error::NonstandardPaths)?; - let installation = gate.try_installation()?; crate::installer::upgrade::request_locked(request, &data_dir, &run_dir, &installation) .await .map_err(Into::into) diff --git a/crates/ployzd/src/machine/mod.rs b/crates/ployzd/src/machine/mod.rs index b0932ef60d..81e6aec8ee 100644 --- a/crates/ployzd/src/machine/mod.rs +++ b/crates/ployzd/src/machine/mod.rs @@ -32,8 +32,8 @@ mod store; use store::PENDING_RESET_FILE_NAME; pub use store::{LocalMachineStore, StoreError}; -pub(crate) use local_machine::RuntimeWatchTelemetry; pub use local_machine::{Error as LocalMachineError, LocalMachine}; +pub(crate) use local_machine::{MutationAdmission, RuntimeWatchTelemetry}; #[cfg(test)] mod register_tests; diff --git a/crates/ployzd/src/machine_api/local.rs b/crates/ployzd/src/machine_api/local.rs index 1a473ebce4..257fdaf739 100644 --- a/crates/ployzd/src/machine_api/local.rs +++ b/crates/ployzd/src/machine_api/local.rs @@ -566,6 +566,7 @@ impl MachineRpc for MachineService { self.local_record()?.id(), request.into_inner(), self.builds.clone(), + self.local.clone(), ))) } diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index 384aff6880..8dc2a493d3 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -17,7 +17,7 @@ use ployzd::{ daemon::{ContainerMode, Daemon, DaemonConfig, Error, wait_until_socket_accepts}, diag, installer::{ - DEFAULT_SOCKET_PATH, InstallRequest, Preparation, Readiness, ReleaseRequest, ReleaseSource, + DEFAULT_SOCKET_PATH, InstallMode, InstallRequest, Readiness, ReleaseRequest, ReleaseSource, }, machine::DEFAULT_DATA_DIR, network::NetworkError, @@ -214,7 +214,9 @@ fn install_request( "--install-only cannot be combined with --group-user", )); } - let preparation = if software_only { + let mode = if install_only { + InstallMode::InstallationOnly + } else if software_only { if storage != StorageChoice::None { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -227,9 +229,9 @@ fn install_request( "--software-only cannot be combined with --group-user", )); } - Preparation::SoftwareOnly + InstallMode::SoftwareOnly } else { - Preparation::PrepareHost { + InstallMode::PrepareHost { storage, group_user, } @@ -237,8 +239,7 @@ fn install_request( Ok(InstallRequest { release, source: release_dir.map_or(ReleaseSource::Published, ReleaseSource::Local), - preparation, - install_only, + mode, }) } @@ -332,10 +333,10 @@ mod tests { } #[test] - fn install_cli_builds_one_explicit_preparation_mode() { + fn install_cli_builds_one_explicit_mode() { let replacement = install_request("1.2.3".into(), StorageChoice::None, true, true, None, None).unwrap(); - assert!(matches!(replacement.preparation, Preparation::SoftwareOnly)); + assert!(matches!(replacement.mode, InstallMode::InstallationOnly)); let host = install_request( "1.2.3".into(), @@ -347,8 +348,8 @@ mod tests { ) .unwrap(); assert!(matches!( - host.preparation, - Preparation::PrepareHost { + host.mode, + InstallMode::PrepareHost { storage: StorageChoice::Zfs, group_user: Some(_) } diff --git a/scripts/qualify-release.sh b/scripts/qualify-release.sh index 9f19440e1c..6c08a0b3ce 100755 --- a/scripts/qualify-release.sh +++ b/scripts/qualify-release.sh @@ -25,12 +25,15 @@ RESET=${PLOYZ_QUALIFY_RESET:-0} SSH_KEY=${PLOYZ_QUALIFY_SSH_KEY:-} CONFIG_DIR= TRAFFIC_PID= +ENROLL_FIXTURE_PID= +ENROLL_PORT= TRAFFIC_STOP=/var/lib/ployz/qualification/traffic.stop REMOTE_RELEASE_ROOT=/var/lib/ployz/qualification/releases APP_URL=http://127.0.0.1:18082/identity APP_VALUE=qualify-persistent-data APP_VOLUME=qualify-release_qualify-data APP_VOLUME_MOUNT=/var/lib/ployz-volumes/$APP_VOLUME +PAIRING='{"relayUrl":"http://127.0.0.1:1/","secret":"qualification-synthetic-pairing-secret-v1"}' error() { echo "ERROR: $*" >&2; exit 1; } @@ -108,6 +111,37 @@ encoded = json.dumps(stable(value), sort_keys=True, separators=(",", ":")).encod print(hashlib.sha256(encoded).hexdigest())' } +machine_cloud_pairing() { + ssh_host "$first" sudo cat /var/lib/ployz/machine.json | python3 -c 'import json, sys +value = json.load(sys.stdin)["cloud_pairing"] +print(json.dumps(value, sort_keys=True, separators=(",", ":")))' +} + +start_enroll_fixture() { + local port_file=$work/cloud-enroll.port i + python3 "$COMPOSE_DIR/cloud-enroll-fixture.py" "$port_file" "$work/cloud-enroll-evidence.jsonl" >"$work/cloud-enroll.log" 2>&1 & + ENROLL_FIXTURE_PID=$! + for ((i = 0; i < 50; i++)); do + if [ -s "$port_file" ]; then + ENROLL_PORT=$(cat "$port_file") + return + fi + kill -0 "$ENROLL_FIXTURE_PID" 2>/dev/null || break + sleep 0.1 + done + cat "$work/cloud-enroll.log" >&2 || true + error "Cloud enrollment fixture did not start" +} + +finish_enroll_fixture() { + if ! wait "$ENROLL_FIXTURE_PID"; then + cat "$work/cloud-enroll.log" >&2 || true + error "Cloud enrollment fixture rejected the enrollment contract" + fi + ENROLL_FIXTURE_PID= + [ "$(wc -l <"$work/cloud-enroll-evidence.jsonl")" -eq 2 ] || error "Cloud enrollment fixture did not observe exactly two requests" +} + read_receipt() { ssh_host "$first" 'sudo cat /var/lib/ployz/upgrade-attempt.json 2>/dev/null || true' } @@ -194,6 +228,11 @@ stop_traffic() { cleanup() { stop_traffic + if [ -n "$ENROLL_FIXTURE_PID" ]; then + kill "$ENROLL_FIXTURE_PID" >/dev/null 2>&1 || true + wait "$ENROLL_FIXTURE_PID" >/dev/null 2>&1 || true + ENROLL_FIXTURE_PID= + fi [ -z "${work:-}" ] || rm -rf "$work" [ -z "$CONFIG_DIR" ] || rm -rf "$CONFIG_DIR" } @@ -244,7 +283,7 @@ if [ "$DRY_RUN" != 0 ]; then else echo "reset: no (initialized hosts fail without PLOYZ_QUALIFY_RESET=1)" fi - echo "steps: normal ZFS machine init/add from the source release; persistent application traffic; target upgrade with client disconnect and reconnect; corrupt preflight; failed activation; explicit previous-binary repair" + echo "steps: normal ZFS machine init/add from the source release; synthetic loopback Cloud Pairing; persistent application traffic; target upgrade with client disconnect and reconnect; corrupt preflight; failed activation; explicit previous-binary repair" exit 0 fi @@ -288,6 +327,13 @@ while [ "$i" -lt "${#HOST_LIST[@]}" ]; do i=$((i + 1)) done +echo "establish a synthetic non-null Cloud Pairing through Cloud enrollment" +start_enroll_fixture +"$PLOYZ" cloud enroll pmet_qualification --name qualify-1 --storage none --no-dns --no-ingress --cloud-url "http://127.0.0.1:$ENROLL_PORT" --context "$CONTEXT" +finish_enroll_fixture +pairing_before=$(machine_cloud_pairing) +[ "$pairing_before" = "$PAIRING" ] || error "Machine did not persist the exact synthetic Cloud Pairing" + machine_arch=$(ssh_host "$first" uname -m) machine_archive=$(daemon_archive "$machine_arch") need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR "$machine_archive" @@ -344,6 +390,7 @@ require_field "$success_inspection" version "$target_version" assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" wait_for_application [ "$(machine_state_signature)" = "$state_before" ] || error "Machine identity, pairing, or configuration changed during upgrade" +[ "$(machine_cloud_pairing)" = "$pairing_before" ] || error "Cloud Pairing changed during upgrade" echo "reject a checksum-corrupt target before activation" select_remote_release corrupt @@ -389,6 +436,7 @@ require_field "$failure_inspection" stage "$failure_stage" assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" wait_for_application [ "$(machine_state_signature)" = "$state_before" ] || error "Machine identity, pairing, or configuration changed after explicit repair" +[ "$(machine_cloud_pairing)" = "$pairing_before" ] || error "Cloud Pairing changed after explicit repair" stop_traffic if grep -q '^failure:' "$work/traffic.log"; then diff --git a/scripts/qualify-release/cloud-enroll-fixture.py b/scripts/qualify-release/cloud-enroll-fixture.py new file mode 100644 index 0000000000..5be3e18e67 --- /dev/null +++ b/scripts/qualify-release/cloud-enroll-fixture.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Bounded loopback Cloud enrollment fixture for release qualification.""" + +import http.server +import json +import re +import sys + +port_file, evidence_file = sys.argv[1:] +token = "pmet_qualification" +secret = "qualification-synthetic-pairing-secret-v1" +expected_paths = [f"/api/enroll/{token}", f"/api/enroll/{token}/callback"] + + +class Handler(http.server.BaseHTTPRequestHandler): + requests = 0 + failure = None + + def do_POST(self): + try: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > 1024 * 1024: + raise AssertionError(f"invalid request size {length}") + body = json.loads(self.rfile.read(length)) + expected = expected_paths[Handler.requests] + if self.path != expected: + raise AssertionError( + f"request {Handler.requests + 1} path {self.path!r}, expected {expected!r}" + ) + if Handler.requests == 0: + if body.get("protocolVersion") != 2: + raise AssertionError("enroll protocolVersion was not 2") + if body.get("name") != "qualify-1": + raise AssertionError("enroll Machine Name was not qualify-1") + if body.get("requestedStorage") != "none": + raise AssertionError("enroll requestedStorage was not none") + if not isinstance(body.get("publicKey"), str) or not body["publicKey"]: + raise AssertionError("enroll publicKey was empty") + response = { + "kind": "initialize", + "resumed": True, + "storage": "none", + "pairing": { + "relayUrl": "http://127.0.0.1:1", + "secret": secret, + }, + } + evidence = {"request": "enroll", "identity": "accepted"} + else: + machine_id = body.get("machineId") + valid_machine_id = isinstance( + machine_id, str + ) and re.fullmatch(r"[0-9a-f]{32}", machine_id) + if not valid_machine_id: + raise AssertionError( + "callback Machine ID was not 32 lowercase hex characters" + ) + if body.get("pairingCredential") != secret: + raise AssertionError("callback Pairing Credential changed") + response = {} + evidence = { + "request": "callback", + "machineId": machine_id, + "pairingCredential": "matched", + } + with open(evidence_file, "a", encoding="utf-8") as evidence_stream: + evidence_stream.write(json.dumps(evidence, sort_keys=True) + "\n") + payload = json.dumps(response, separators=(",", ":")).encode() + self.send_response(200) + except Exception as error: + Handler.failure = str(error) + payload = json.dumps({"error": str(error)}).encode() + self.send_response(400) + Handler.requests += 1 + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_): + pass + + +server = http.server.HTTPServer(("127.0.0.1", 0), Handler) +server.timeout = 30 +with open(port_file, "w", encoding="ascii") as output: + output.write(str(server.server_port)) +for _ in expected_paths: + server.handle_request() +server.server_close() +if Handler.requests != len(expected_paths): + raise SystemExit( + f"Cloud enrollment fixture received {Handler.requests} of {len(expected_paths)} requests" + ) +if Handler.failure is not None: + raise SystemExit(Handler.failure) diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index 7c1163cd8c..8d88c1fda7 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -17,9 +17,6 @@ fi grep -Fq 'qualify-data' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no named volume" grep -Fq 'x-volumes:' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture volume is not provisioned" -grep -Fq 'busybox:1.37.0' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no verified HTTP server image" -grep -Fq '18082:8080/tcp@host' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no traffic endpoint" -grep -Fq '/data/identity' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture does not serve persistent data" if "$ROOT/scripts/qualify-release.sh" >/dev/null 2>&1; then fail "qualify-release accepted empty hosts" @@ -60,6 +57,50 @@ case "$1" in printf '%s context=%s file=%s yes=%s\n' "$action" "$context" "$file" "$yes" >>"$LOG" [ "$action" != volume ] || printf 'MACHINE\tVOLUME\tTYPE\tQUOTA\tUSED\tDRIVER\nqualify-1\tqualify-release_qualify-data\tPROVISIONED\t268435456\t4096\tployz\n' ;; + cloud) + [ "$2" = enroll ] || { echo "unexpected cloud action: $2" >&2; exit 1; } + token=$3 + shift 3 + name= storage= context= cloud_url= no_dns=no no_ingress=no + while [ "$#" -gt 0 ]; do + case "$1" in + --name) name=$2; shift ;; + --storage) storage=$2; shift ;; + --context) context=$2; shift ;; + --cloud-url) cloud_url=$2; shift ;; + --no-dns) no_dns=yes ;; + --no-ingress) no_ingress=yes ;; + *) echo "unexpected cloud enroll argument: $1" >&2; exit 1 ;; + esac + shift + done + printf 'cloud-enroll token=%s name=%s storage=%s context=%s cloud_url=%s no_dns=%s no_ingress=%s\n' "$token" "$name" "$storage" "$context" "$cloud_url" "$no_dns" "$no_ingress" >>"$LOG" + python3 - "$cloud_url" "$token" <<'PY' +import json +import sys +import urllib.request + +origin, token = sys.argv[1:] +url = f"{origin}/api/enroll/{token}" +identity = { + "protocolVersion": 2, + "name": "qualify-1", + "requestedStorage": "none", + "publicKey": "synthetic-public-key", +} +request = urllib.request.Request(url, json.dumps(identity).encode(), {"Content-Type": "application/json"}) +with urllib.request.urlopen(request) as response: + pairing = json.load(response)["pairing"] +callback = { + "machineId": "11111111111111111111111111111111", + "pairingCredential": pairing["secret"], +} +request = urllib.request.Request(f"{url}/callback", json.dumps(callback).encode(), {"Content-Type": "application/json"}) +with urllib.request.urlopen(request) as response: + json.load(response) +PY + : >"$QUALIFY_PAIRED" + ;; machine) action=$2 shift 2 @@ -198,7 +239,12 @@ case "$command" in esac ;; *'sudo cat /var/lib/ployz/machine.json'*) - printf '{"body":{"phase":"participating","machine":{"id":"11111111111111111111111111111111","runtime":{"daemon_version":"1.2.3"}}},"wireguard_private_key":"secret","cloud_pairing":null,"selected_endpoints":{}}\n' + if [ -f "$QUALIFY_PAIRED" ]; then + pairing='{"relayUrl":"http://127.0.0.1:1/","secret":"qualification-synthetic-pairing-secret-v1"}' + else + pairing=null + fi + printf '{"body":{"phase":"participating","machine":{"id":"11111111111111111111111111111111","runtime":{"daemon_version":"1.2.3"}}},"wireguard_private_key":"secret","cloud_pairing":%s,"selected_endpoints":{}}\n' "$pairing" ;; *'while [ ! -e /var/lib/ployz/qualification/traffic.stop'*) printf 'ok\nok\nok\nok\nok\nok\n' @@ -221,9 +267,11 @@ fi printf 'scp %s\n' "$*" >>"$SSH_LOG" SCP chmod 0755 "$TMP/bin/scp" +export QUALIFY_PAIRED=$TMP/paired for reset in 0 1; do : >"$LOG" + rm -f "$QUALIFY_PAIRED" PATH="$TMP/bin:$PATH" PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$SOURCE" \ PLOYZ_UPGRADE_ARTIFACT_DIR="$TARGET" PLOYZ_QUALIFY_SSH_KEY=/tmp/qualify-key \ PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ @@ -232,6 +280,7 @@ for reset in 0 1; do [ "$reset" = 0 ] || expected_reset=yes grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-1 no_dns=yes no_ingress=yes release=$SOURCE" "$LOG" || fail "init lost its target, release, storage, name, version, reset policy, isolation policy, or SSH identity" grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-2 no_dns=no no_ingress=no release=$SOURCE" "$LOG" || fail "add lost its target, release, storage, name, version, or SSH identity" + grep -Eq '^cloud-enroll token=pmet_qualification name=qualify-1 storage=none context=qualify cloud_url=http://127\.0\.0\.1:[0-9]+ no_dns=yes no_ingress=yes$' "$LOG" || fail "Cloud Pairing fixture was not exercised through the source CLI" grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "persistent-volume fixture was not deployed" grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=target' "$LOG" || fail "target upgrade was not requested" grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=corrupt' "$LOG" || fail "corrupt preflight was not requested" From 70b9fa1fb0463bb51814056f14eea1613fae8888 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 06:14:25 +0000 Subject: [PATCH 29/30] refactor(build): group execution admission --- crates/ployzd/src/build.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/ployzd/src/build.rs b/crates/ployzd/src/build.rs index ca39be1d39..79d6a54fef 100644 --- a/crates/ployzd/src/build.rs +++ b/crates/ployzd/src/build.rs @@ -30,6 +30,11 @@ struct RetainedBuild { _installation: crate::mutation::MutationGuard, } +struct ExecutionAdmission { + build: Admission, + machine: crate::machine::MutationAdmission, +} + pub(crate) fn start( machine_id: MachineId, requests: impl Stream> + Send + Unpin + 'static, @@ -179,7 +184,16 @@ async fn attempt( let mut execution = tokio::task::spawn_blocking(move || { let _permit = permit; receive_and_execute( - machine_id, definition, source, admission, mutation, policy, &output, &observed, + machine_id, + definition, + source, + ExecutionAdmission { + build: admission, + machine: mutation, + }, + policy, + &output, + &observed, ) }); // Dropping the input pump closes the upload channel. A receiver blocked @@ -251,12 +265,15 @@ fn receive_and_execute( machine_id: MachineId, mut definition: Definition, mut source: mpsc::Receiver, - admission: Admission, - mutation: crate::machine::MutationAdmission, + admission: ExecutionAdmission, policy: HostPolicy, events: &mpsc::Sender>, state: &AttemptState, ) -> Outcome { + let ExecutionAdmission { + build: admission, + machine: mutation, + } = admission; let cancellation = admission.cancellation(); let deadline = std::time::Instant::now() + admission.remaining(); let mut upload = match admission.upload() { From ad510052f12053a79fb5ce5ae0779fc0cb3f4e15 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 9 Sep 2026 07:23:32 +0000 Subject: [PATCH 30/30] fix(installer): own launches and retain live smoke backing --- crates/ployzd/src/installer/storage.rs | 86 ++++++++++----- crates/ployzd/src/installer/storage/tests.rs | 102 ++++++++++++++++++ crates/ployzd/src/installer/upgrade.rs | 93 ++++++++++++++-- .../src/machine/local_machine/upgrade.rs | 2 +- 4 files changed, 243 insertions(+), 40 deletions(-) create mode 100644 crates/ployzd/src/installer/storage/tests.rs diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs index 4267ed8e31..8ec86822ff 100644 --- a/crates/ployzd/src/installer/storage.rs +++ b/crates/ployzd/src/installer/storage.rs @@ -333,36 +333,49 @@ fn validate_zfs() -> Result<(), Error> { let stage = staging_directory(Path::new("/var/tmp"))?; let backing = stage.path().join("backing"); write_private(&backing, b"", "create ZFS smoke backing file")?; - // The installer lock serializes smoke Pools, and the process ID is the shell installer's - // existing collision boundary. - let pool = format!("ployz-smoke-{}", std::process::id()); + let mut allocate = Command::new("fallocate"); + allocate + .args(["-l", &ZFS_SMOKE_BYTES.to_string()]) + .arg(&backing); + run_command("preallocate ZFS smoke backing file", &mut allocate)?; + let metadata = backing.metadata().map_err(|source| Error::Io { + stage: "inspect ZFS smoke backing file", + source, + })?; + if metadata.blocks().saturating_mul(POSIX_STAT_BLOCK_BYTES) < ZFS_SMOKE_BYTES { + return Err(Error::Verification(format!( + "ZFS smoke backing file {} is sparse", + backing.display() + ))); + } + let root = fs::metadata("/").map_err(|source| Error::Io { + stage: "inspect host-root filesystem", + source, + })?; + if metadata.dev() != root.dev() { + return Err(Error::Verification(format!( + "ZFS smoke backing file {} is not on the host root filesystem", + backing.display() + ))); + } + validate_zfs_pool(stage) +} + +fn validate_zfs_pool(stage: tempfile::TempDir) -> Result<(), Error> { + let pool = format!( + "ployz-smoke-{}", + stage + .path() + .file_name() + .expect("temporary directory name") + .to_string_lossy() + ); + // ZFS may own this backing file as soon as creation is attempted. A persistent path + // cannot unlink it on error or unwind; only positively verified cleanup may delete it. + let directory = stage.keep(); + let backing = directory.join("backing"); let mut pool_created = false; let result = (|| { - let mut allocate = Command::new("fallocate"); - allocate - .args(["-l", &ZFS_SMOKE_BYTES.to_string()]) - .arg(&backing); - run_command("preallocate ZFS smoke backing file", &mut allocate)?; - let metadata = backing.metadata().map_err(|source| Error::Io { - stage: "inspect ZFS smoke backing file", - source, - })?; - if metadata.blocks().saturating_mul(POSIX_STAT_BLOCK_BYTES) < ZFS_SMOKE_BYTES { - return Err(Error::Verification(format!( - "ZFS smoke backing file {} is sparse", - backing.display() - ))); - } - let root = fs::metadata("/").map_err(|source| Error::Io { - stage: "inspect host-root filesystem", - source, - })?; - if metadata.dev() != root.dev() { - return Err(Error::Verification(format!( - "ZFS smoke backing file {} is not on the host root filesystem", - backing.display() - ))); - } let mut create = Command::new("zpool"); create .args(["create", "-f", "-m", "none", "-o", "cachefile=none", &pool]) @@ -381,7 +394,19 @@ fn validate_zfs() -> Result<(), Error> { )?; Ok(()) })(); - let cleanup = cleanup_zfs_smoke(&pool, &backing, pool_created); + let cleanup = cleanup_zfs_smoke(&pool, &backing, pool_created) + .and_then(|()| { + fs::remove_dir(&directory).map_err(|source| Error::Io { + stage: "remove temporary ZFS smoke directory", + source, + }) + }) + .map_err(|error| { + Error::Verification(format!( + "{error}; ZFS smoke backing directory retained at {} for inspection", + directory.display() + )) + }); match (result, cleanup) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) => Err(error), @@ -392,6 +417,9 @@ fn validate_zfs() -> Result<(), Error> { } } +#[cfg(test)] +mod tests; + fn cleanup_zfs_smoke(pool: &str, backing: &Path, pool_created: bool) -> Result<(), Error> { let existing_pool = if pool_created { true diff --git a/crates/ployzd/src/installer/storage/tests.rs b/crates/ployzd/src/installer/storage/tests.rs new file mode 100644 index 0000000000..25447358e2 --- /dev/null +++ b/crates/ployzd/src/installer/storage/tests.rs @@ -0,0 +1,102 @@ +//! Smoke Pool lifecycle contracts with isolated ZFS command adapters. + +use super::*; +use std::{env, os::unix::fs::PermissionsExt}; + +#[test] +fn smoke_pool_cleanup_preserves_backing_until_absence_is_proven() { + const CASE: &str = "PLOYZ_SMOKE_CLEANUP_CASE"; + const ROOT: &str = "PLOYZ_SMOKE_CLEANUP_ROOT"; + if let Ok(case) = env::var(CASE) { + let root = std::path::PathBuf::from(env::var_os(ROOT).unwrap()); + let stage = tempfile::tempdir_in(&root).unwrap(); + let directory = stage.path().to_owned(); + let backing = directory.join("backing"); + fs::write(&backing, b"pool data").unwrap(); + let result = validate_zfs_pool(stage); + if case == "success" || case == "query-failed" { + assert!(!directory.exists()); + assert_eq!(result.is_ok(), case == "success"); + } else { + assert!( + result + .unwrap_err() + .to_string() + .contains(&directory.display().to_string()) + ); + assert_eq!(fs::read(&backing).unwrap(), b"pool data", "{case}"); + } + fs::write(root.join("completed"), case).unwrap(); + return; + } + + for case in [ + "success", + "query-failed", + "destroy-failed", + "pool-inspection-failed", + "pool-remains", + "dataset-inspection-failed", + "dataset-remains", + "create-unknown", + ] { + let root = tempfile::tempdir().unwrap(); + let commands = root.path().join("commands"); + fs::create_dir(&commands).unwrap(); + for (name, body) in [ + ( + "zpool", + r#" +case "$1" in + create) + for arg do previous=${last-}; last=$arg; done + echo "$previous" > "$PLOYZ_SMOKE_CLEANUP_ROOT/pool" + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != create-unknown ] ;; + destroy) [ "$PLOYZ_SMOKE_CLEANUP_CASE" != destroy-failed ] ;; + list) + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != create-unknown ] || exit 1 + if [ "$2" = -Hp ]; then + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != query-failed ] + else + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != pool-inspection-failed ] || exit 1 + if [ "$PLOYZ_SMOKE_CLEANUP_CASE" = pool-remains ]; then + /bin/cat "$PLOYZ_SMOKE_CLEANUP_ROOT/pool" + fi + fi ;; +esac +"#, + ), + ( + "zfs", + r#" +if [ "$2" = -H ]; then + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != dataset-inspection-failed ] || exit 1 + if [ "$PLOYZ_SMOKE_CLEANUP_CASE" = dataset-remains ]; then + /bin/cat "$PLOYZ_SMOKE_CLEANUP_ROOT/pool" + fi +fi +"#, + ), + ] { + let path = commands.join(name); + fs::write(&path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + let output = Command::new(env::current_exe().unwrap()) + .args(["--exact", "installer::storage::tests::smoke_pool_cleanup_preserves_backing_until_absence_is_proven", "--nocapture"]) + .env(CASE, case) + .env(ROOT, root.path()) + .env("PATH", commands) + .output() + .unwrap(); + assert!( + output.status.success(), + "{case}: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::read_to_string(root.path().join("completed")).unwrap(), + case + ); + } +} diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs index 975f5a75fb..376560ddcd 100644 --- a/crates/ployzd/src/installer/upgrade.rs +++ b/crates/ployzd/src/installer/upgrade.rs @@ -114,13 +114,28 @@ pub(crate) fn existing_request( Ok(Some(stored.attempt)) } -/// Persist an accepted attempt and launch its old-daemon worker while installation admission is -/// held. Reusing the same attempt ID returns the same attempt. -pub(crate) async fn request_locked( +/// Own acceptance and worker launch independently of the requesting client's lifetime. +/// Dropping the returned future only stops waiting; the task retains exclusive admission. +pub(crate) fn request( + request: RequestMachineUpgradeRequest, + data_dir: PathBuf, + run_dir: PathBuf, + guard: mutation::InstallationGuard, +) -> impl Future> { + let task = tokio::spawn(async move { + let _guard = guard; + request_locked(request, &data_dir, &run_dir).await + }); + async move { + task.await + .map_err(|error| Error::Launch(format!("upgrade acceptance task failed: {error}")))? + } +} + +async fn request_locked( request: RequestMachineUpgradeRequest, data_dir: &Path, run_dir: &Path, - _guard: &mutation::InstallationGuard, ) -> Result { let admission = mutation::MutationGate::new(run_dir, data_dir); if let Some(mut stored) = read_optional(data_dir)? { @@ -537,6 +552,8 @@ mod tests { "interrupted", "inspection-unknown", "launch-failed", + "cancelled-launch", + "cancelled-launch-failed", ] { let root = tempfile::Builder::new() .prefix(&format!("ployzd-upgrade-{case}-")) @@ -546,7 +563,9 @@ mod tests { fs::create_dir(&commands).unwrap(); write_script( &commands.join("systemd-run"), - if case == "launch-failed" { + if case.starts_with("cancelled-launch") { + "echo started > \"$PLOYZ_UPGRADE_CONTRACT_ROOT/launch-started\"\nwhile [ ! -f \"$PLOYZ_UPGRADE_CONTRACT_ROOT/launch-release\" ]; do /bin/sleep 0.01; done\nprintf '%s\\n' \"$*\" >> \"$PLOYZ_UPGRADE_COMMAND_LOG\"\nif [ \"$PLOYZ_UPGRADE_CONTRACT_CASE\" = cancelled-launch-failed ]; then echo worker launch refused >&2; exit 1; fi" + } else if case == "launch-failed" { "echo worker launch refused >&2; exit 1" } else { "printf '%s\\n' \"$*\" >> \"$PLOYZ_UPGRADE_COMMAND_LOG\"" @@ -601,10 +620,62 @@ mod tests { release: MachineRelease::parse("1.2.3").unwrap(), }; let guard = admission.try_installation().unwrap(); - let accepted = request_locked(request.clone(), &data, &run, &guard) + if case.starts_with("cancelled-launch") { + let launch = tokio::spawn({ + let data = data.clone(); + let run = run.clone(); + let request = request.clone(); + async move { super::request(request, data, run, guard).await } + }); + timeout(Duration::from_secs(5), async { + while !root.join("launch-started").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("launch did not reach the cancellation window"); + launch.abort(); + assert!(launch.await.unwrap_err().is_cancelled()); + assert!(admission.active().unwrap()); + assert!(matches!( + admission.try_installation(), + Err(mutation::Error::Busy) + )); + fs::write(root.join("launch-release"), "continue").unwrap(); + timeout(Duration::from_secs(5), async { + while admission.try_installation().is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("detached launch did not finish"); + let observed = existing_request(&request, &data).unwrap().unwrap(); + if case == "cancelled-launch-failed" { + assert!(matches!( + observed.outcome, + MachineUpgradeOutcome::Failed { + stage: MachineUpgradeStage::Launching, + .. + } + )); + assert!(!admission.active().unwrap()); + assert!(admission.try_mutation().is_ok()); + } else { + assert!(matches!(observed.outcome, MachineUpgradeOutcome::Accepted)); + assert!(admission.active().unwrap()); + } + assert_eq!( + fs::read_to_string(root.join("commands.log")) + .unwrap() + .lines() + .count(), + 1 + ); + return; + } + let accepted = super::request(request.clone(), data.clone(), run.clone(), guard) .await .unwrap(); - drop(guard); match case { "retry-active" => { @@ -628,18 +699,20 @@ mod tests { let guard = admission.try_installation().unwrap(); assert_eq!( - request_locked(request, &data, &run, &guard).await.unwrap(), + super::request(request, data.clone(), run.clone(), guard) + .await + .unwrap(), accepted ); + let guard = admission.try_installation().unwrap(); let other = RequestMachineUpgradeRequest { attempt_id: MachineUpgradeAttemptId::parse("b".repeat(32)).unwrap(), release: MachineRelease::parse("1.2.3").unwrap(), }; assert!(matches!( - request_locked(other, &data, &run, &guard).await, + super::request(other, data.clone(), run.clone(), guard).await, Err(Error::Busy) )); - drop(guard); let log = fs::read_to_string(root.join("commands.log")).unwrap(); assert_eq!(log.lines().count(), 1, "{log}"); for required in [ diff --git a/crates/ployzd/src/machine/local_machine/upgrade.rs b/crates/ployzd/src/machine/local_machine/upgrade.rs index 5935e3e46d..14541d2c9b 100644 --- a/crates/ployzd/src/machine/local_machine/upgrade.rs +++ b/crates/ployzd/src/machine/local_machine/upgrade.rs @@ -39,7 +39,7 @@ impl LocalMachine { let installation = gate.try_installation()?; crate::installer::require_standard_machine_paths(&data_dir, &run_dir.join("ployz.sock")) .map_err(crate::installer::upgrade::Error::NonstandardPaths)?; - crate::installer::upgrade::request_locked(request, &data_dir, &run_dir, &installation) + crate::installer::upgrade::request(request, data_dir, run_dir, installation) .await .map_err(Into::into) }