diff --git a/.github/workflows/release-contracts.yml b/.github/workflows/release-contracts.yml index 13ef3336ce..9446456664 100644 --- a/.github/workflows/release-contracts.yml +++ b/.github/workflows/release-contracts.yml @@ -39,7 +39,6 @@ on: - .github/workflows/release-contracts.yml - .goreleaser.yaml - install.sh - - scripts/install.sh - scripts/uninstall.sh - scripts/verify-release.sh - scripts/pack-release.sh diff --git a/.goreleaser.yaml b/.goreleaser.yaml index cddc6c8cb8..ca7b1e8506 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -117,7 +117,7 @@ archives: checksum: name_template: checksums.txt - ids: [ployz] + ids: [ployz, ployzd] changelog: disable: true diff --git a/Cargo.lock b/Cargo.lock index 10661d8886..caf192026e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,6 +834,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -2144,6 +2150,7 @@ dependencies = [ "shell-words", "signal-hook", "sigpipe", + "tempfile", "thiserror 2.0.20", "tokio", "tokio-stream", @@ -2189,6 +2196,7 @@ dependencies = [ "base64", "ipnet", "prost", + "semver", "serde", "serde_json", "thiserror 2.0.20", @@ -2283,6 +2291,7 @@ dependencies = [ "serde_json", "sha2", "shell-words", + "tempfile", "thiserror 2.0.20", "tokio", "tokio-stream", @@ -3174,6 +3183,19 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "termcolor" version = "1.4.1" diff --git a/crates/ployz-core/Cargo.toml b/crates/ployz-core/Cargo.toml index 8ed533f5e1..f77bf5784a 100644 --- a/crates/ployz-core/Cargo.toml +++ b/crates/ployz-core/Cargo.toml @@ -11,6 +11,7 @@ ipnet = { version = "2.12.1", features = ["serde"] } prost = "0.13.5" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" +semver = "1.0.28" thiserror = "2.0.20" ts-rs.workspace = true url = { version = "2.5.8", features = ["serde"] } diff --git a/crates/ployz-core/build.rs b/crates/ployz-core/build.rs index 8d8ff8c3a1..d47ae5b889 100644 --- a/crates/ployz-core/build.rs +++ b/crates/ployz-core/build.rs @@ -1,6 +1,7 @@ include!("src/rpc_catalog.rs"); fn main() { + println!("cargo:rerun-if-changed=src/rpc_catalog.rs"); macro_rules! build_service { ( package $package:literal diff --git a/crates/ployz-core/src/rpc.rs b/crates/ployz-core/src/rpc.rs index 90220e69cb..3e58df36b2 100644 --- a/crates/ployz-core/src/rpc.rs +++ b/crates/ployz-core/src/rpc.rs @@ -6,6 +6,9 @@ use std::{ }; use ts_rs::TS; +mod upgrade; +pub use upgrade::*; + use ipnet::Ipv4Net; use prost::Message; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::DeserializeOwned}; @@ -839,6 +842,7 @@ define_responses! { Domain(Domain) => "domain"; DomainRecords(DomainRecords) => "domain_records"; MachineUpdated(MachineUpdated) => "machine_updated"; + MachineUpgradeAttempt(MachineUpgradeAttempt) => "machine_upgrade_attempt"; LocalMachineRemoved(LocalMachineRemoved) => "local_machine_removed"; MachineRemoved(MachineRemoved) => "machine_removed"; WireGuardInspected(WireGuardInspected) => "wireguard_inspected"; diff --git a/crates/ployz-core/src/rpc/upgrade.rs b/crates/ployz-core/src/rpc/upgrade.rs new file mode 100644 index 0000000000..ba34adeafc --- /dev/null +++ b/crates/ployz-core/src/rpc/upgrade.rs @@ -0,0 +1,87 @@ +//! Typed requests and durable outcomes for one Machine upgrade. + +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +/// Request one exact or channel-selected Machine release under a stable retry identity. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct RequestMachineUpgradeRequest { + /// Identity reused by every retry of the same request. + pub attempt_id: crate::MachineUpgradeAttemptId, + /// Exact version or supported release channel to resolve once. + pub release: crate::MachineRelease, +} + +/// Read the latest attempt, optionally requiring one exact retry identity. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct InspectMachineUpgradeRequest { + /// Required attempt identity, or the latest local attempt when absent. + #[serde(default)] + pub attempt_id: Option, +} + +crate::value::open_string_enum!(MachineUpgradeStage, Unknown { + Launching => "launching", + Preparing => "preparing", + Acquiring => "acquiring", + Verifying => "verifying", + Activating => "activating", + Restarting => "restarting", + Readiness => "readiness", +}); + +/// Durable local evidence for one bounded Machine upgrade attempt. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TS)] +pub struct MachineUpgradeAttempt { + /// Stable identity assigned before the request is dispatched. + pub attempt_id: crate::MachineUpgradeAttemptId, + /// Exact version resolved before any host mutation. + pub target: crate::MachineVersion, + /// State-specific evidence for the attempt. + #[serde(flatten)] + #[ts(flatten)] + pub outcome: MachineUpgradeOutcome, +} + +/// State-specific evidence for a Machine upgrade attempt. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, TS)] +#[serde(tag = "outcome", rename_all = "snake_case")] +pub enum MachineUpgradeOutcome { + /// The durable receipt and active marker exist and worker launch was accepted. + Accepted, + /// The independent worker reached this installation stage. + Running { + /// Latest stage durably recorded by the worker. + stage: MachineUpgradeStage, + }, + /// The target daemon was observed running and ready through its local API. + Succeeded { + /// Exact version reported by the ready, activated daemon. + version: crate::MachineVersion, + }, + /// The worker recorded a terminal installation failure. + Failed { + /// First installation stage that did not complete. + stage: MachineUpgradeStage, + /// Operator-facing failure evidence. + error: String, + }, + /// A positively stopped worker left no terminal result. + Interrupted { + /// Last durable stage before the worker stopped. + stage: MachineUpgradeStage, + }, +} + +impl MachineUpgradeAttempt { + /// Whether the attempt has durable terminal evidence. + #[must_use] + pub fn is_terminal(&self) -> bool { + matches!( + self.outcome, + MachineUpgradeOutcome::Succeeded { .. } + | MachineUpgradeOutcome::Failed { .. } + | MachineUpgradeOutcome::Interrupted { .. } + ) + } +} diff --git a/crates/ployz-core/src/rpc_catalog.rs b/crates/ployz-core/src/rpc_catalog.rs index 82f5b1dc30..5bda656876 100644 --- a/crates/ployz-core/src/rpc_catalog.rs +++ b/crates/ployz-core/src/rpc_catalog.rs @@ -44,6 +44,8 @@ macro_rules! rpc_catalog { SetCloudPairing: (set_cloud_pairing, "SetCloudPairing", SetCloudPairingRequest, "set_cloud_pairing", CloudPairingSet, SET_CLOUD_PAIRING_CAPABILITY, "ployz.machine.set-cloud-pairing.v1", Always), ListMachines: (list_machines, "ListMachines", ListMachinesRequest, "list_machines", MachineList, LIST_MACHINES_CAPABILITY, "ployz.machine.list.v1", Always), UpdateMachine: (update_machine, "UpdateMachine", UpdateMachineRequest, "update_machine", MachineUpdated, UPDATE_MACHINE_CAPABILITY, "ployz.machine.update.v1", Always), + RequestMachineUpgrade: (request_machine_upgrade, "RequestMachineUpgrade", RequestMachineUpgradeRequest, "request_machine_upgrade", MachineUpgradeAttempt, REQUEST_MACHINE_UPGRADE_CAPABILITY, "ployz.machine.upgrade.request.v1", Always), + InspectMachineUpgrade: (inspect_machine_upgrade, "InspectMachineUpgrade", InspectMachineUpgradeRequest, "inspect_machine_upgrade", MachineUpgradeAttempt, INSPECT_MACHINE_UPGRADE_CAPABILITY, "ployz.machine.upgrade.inspect.v1", Always), RemoveLocalMachine: (remove_local_machine, "RemoveLocalMachine", RemoveLocalMachineRequest, "remove_local_machine", LocalMachineRemoved, REMOVE_LOCAL_MACHINE_CAPABILITY, "ployz.machine.remove-local.v1", Always), RemoveMachine: (remove_machine, "RemoveMachine", RemoveMachineRequest, "remove_machine", MachineRemoved, REMOVE_MACHINE_CAPABILITY, "ployz.machine.remove.v1", Always), InspectWireguard: (inspect_wireguard, "InspectWireguard", InspectWireGuardRequest, "inspect_wireguard", WireGuardInspected, INSPECT_WIREGUARD_CAPABILITY, "ployz.wireguard.inspect.v1", Always), diff --git a/crates/ployz-core/src/value.rs b/crates/ployz-core/src/value.rs index 91f9b14275..9078e4de40 100644 --- a/crates/ployz-core/src/value.rs +++ b/crates/ployz-core/src/value.rs @@ -63,6 +63,18 @@ fn is_hostname(value: &str) -> bool { (1..=253).contains(&value.len()) && value.split('.').all(is_dns_label) } +fn is_machine_version(value: &str) -> bool { + let Ok(version) = semver::Version::parse(value) else { + return false; + }; + let pre = version.pre.as_str(); + version.build.is_empty() + && (pre.is_empty() + || pre.strip_prefix("beta.").is_some_and(|number| { + !number.is_empty() && number.bytes().all(|byte| byte.is_ascii_digit()) + })) +} + macro_rules! hex_id_newtype { ($(#[$attribute:meta])* $name:ident, $label:literal, $len:expr, $expected:literal) => { $(#[$attribute])* @@ -208,6 +220,13 @@ hex_id_newtype!( 32, "32 lowercase hexadecimal characters" ); +hex_id_newtype!( + /// Identity of one bounded Machine upgrade attempt. + MachineUpgradeAttemptId, + "Machine upgrade attempt ID", + 32, + "32 lowercase hexadecimal characters" +); hex_id_newtype!( ServiceId, "Service ID", @@ -237,6 +256,16 @@ impl MachineId { } } +impl MachineUpgradeAttemptId { + /// Random 32-character lowercase hexadecimal attempt ID. + #[must_use] + pub fn random() -> Self { + let mut hex = [0_u8; 32]; + uuid::Uuid::new_v4().simple().encode_lower(&mut hex); + Self(hex) + } +} + impl ServiceId { #[must_use] pub fn random() -> Self { @@ -287,6 +316,20 @@ validated_string_newtype!( "a 1-63 character lowercase DNS label", |value| is_dns_label(value) ); +validated_string_newtype!( + /// A trusted release selector: stable, beta, or an exact supported version. + MachineRelease, + "Machine release", + "stable, beta, X.Y.Z, or X.Y.Z-beta.N", + |value| matches!(value, "stable" | "beta") || is_machine_version(value) +); +validated_string_newtype!( + /// One exact supported Machine release version. + MachineVersion, + "Machine version", + "X.Y.Z or X.Y.Z-beta.N", + |value| is_machine_version(value) +); validated_string_newtype!( DockerVolumeName, "Docker Volume name", @@ -878,4 +921,24 @@ mod tests { assert_eq!(format!("{token:?}"), "CloudEnrollToken(..)"); assert!(!format!("{token:?}").contains("pmet_test")); } + + #[test] + fn machine_release_accepts_only_channels_and_supported_exact_versions() { + for release in ["stable", "beta", "1.2.3", "1.2.3-beta.4"] { + assert!(MachineRelease::parse(release).is_ok(), "{release}"); + } + for release in [ + "nightly", + "v1.2.3", + "1.2", + "1.2.3-alpha.1", + "1.2.3+build", + "1.2.3-beta.01", + "https://example.test/release", + ] { + assert!(MachineRelease::parse(release).is_err(), "{release}"); + assert!(MachineVersion::parse(release).is_err(), "{release}"); + } + assert!(MachineVersion::parse("stable").is_err()); + } } diff --git a/crates/ployz-core/tests/capabilities.rs b/crates/ployz-core/tests/capabilities.rs index 902a028990..2decdce03e 100644 --- a/crates/ployz-core/tests/capabilities.rs +++ b/crates/ployz-core/tests/capabilities.rs @@ -4,16 +4,16 @@ use ployz_core::{ CapabilityName, DESCRIBE_CONTRACT_CAPABILITY, ENSURE_GLOBAL_SLOT_CAPABILITY, ENSURE_IMAGE_INGEST_CAPABILITY, EXEC_CONTAINER_CAPABILITY, GET_DOMAIN_CAPABILITY, GET_INGRESS_PROXY_CONFIG_CAPABILITY, INITIALIZE_MACHINE_CAPABILITY, - INSPECT_CONTAINER_CAPABILITY, INSPECT_MACHINE_CAPABILITY, INSPECT_STORAGE_CAPABILITY, - INSPECT_VOLUME_CAPABILITY, INSPECT_WIREGUARD_CAPABILITY, JOIN_MACHINE_CAPABILITY, - LIST_CONTAINERS_CAPABILITY, LIST_IMAGES_CAPABILITY, LIST_MACHINES_CAPABILITY, - LIST_VOLUMES_CAPABILITY, MACHINE_LOGS_CAPABILITY, MACHINE_STORAGE_OBSERVATION_CAPABILITY, - MACHINE_TOKEN_CAPABILITY, PREPARE_VOLUMES_CAPABILITY, PULL_IMAGE_FROM_MACHINE_CAPABILITY, - REGISTER_MACHINE_CAPABILITY, RELEASE_DOMAIN_CAPABILITY, REMOVE_CONTAINER_CAPABILITY, - REMOVE_LOCAL_MACHINE_CAPABILITY, REMOVE_MACHINE_CAPABILITY, REMOVE_VOLUME_CAPABILITY, - RESERVE_DOMAIN_CAPABILITY, RESET_MACHINE_CAPABILITY, RUNTIME_WATCH_CAPABILITY, Rpc, - SET_CLOUD_PAIRING_CAPABILITY, START_CONTAINER_CAPABILITY, STOP_CONTAINER_CAPABILITY, - UPDATE_MACHINE_CAPABILITY, op, + INSPECT_CONTAINER_CAPABILITY, INSPECT_MACHINE_CAPABILITY, INSPECT_MACHINE_UPGRADE_CAPABILITY, + INSPECT_STORAGE_CAPABILITY, INSPECT_VOLUME_CAPABILITY, INSPECT_WIREGUARD_CAPABILITY, + JOIN_MACHINE_CAPABILITY, LIST_CONTAINERS_CAPABILITY, LIST_IMAGES_CAPABILITY, + LIST_MACHINES_CAPABILITY, LIST_VOLUMES_CAPABILITY, MACHINE_LOGS_CAPABILITY, + MACHINE_STORAGE_OBSERVATION_CAPABILITY, MACHINE_TOKEN_CAPABILITY, PREPARE_VOLUMES_CAPABILITY, + PULL_IMAGE_FROM_MACHINE_CAPABILITY, REGISTER_MACHINE_CAPABILITY, RELEASE_DOMAIN_CAPABILITY, + REMOVE_CONTAINER_CAPABILITY, REMOVE_LOCAL_MACHINE_CAPABILITY, REMOVE_MACHINE_CAPABILITY, + REMOVE_VOLUME_CAPABILITY, REQUEST_MACHINE_UPGRADE_CAPABILITY, RESERVE_DOMAIN_CAPABILITY, + RESET_MACHINE_CAPABILITY, RUNTIME_WATCH_CAPABILITY, Rpc, SET_CLOUD_PAIRING_CAPABILITY, + START_CONTAINER_CAPABILITY, STOP_CONTAINER_CAPABILITY, UPDATE_MACHINE_CAPABILITY, op, }; /// Capability constants are generated from the catalog, so a typo would stay @@ -35,6 +35,14 @@ fn catalogued_capabilities_keep_stable_spellings() { "ployz.machine.set-cloud-pairing.v1", ), (LIST_MACHINES_CAPABILITY, "ployz.machine.list.v1"), + ( + REQUEST_MACHINE_UPGRADE_CAPABILITY, + "ployz.machine.upgrade.request.v1", + ), + ( + INSPECT_MACHINE_UPGRADE_CAPABILITY, + "ployz.machine.upgrade.inspect.v1", + ), ( MACHINE_STORAGE_OBSERVATION_CAPABILITY, "ployz.machine.storage-observation.v1", @@ -112,6 +120,8 @@ fn advertised_capability_groups_match_the_frozen_catalog() { "ployz.machine.set-cloud-pairing.v1", "ployz.machine.list.v1", "ployz.machine.update.v1", + "ployz.machine.upgrade.request.v1", + "ployz.machine.upgrade.inspect.v1", "ployz.machine.remove-local.v1", "ployz.machine.remove.v1", "ployz.wireguard.inspect.v1", @@ -200,4 +210,12 @@ fn unary_grpc_paths_stay_on_the_machine_rpc_service() { op::EnsureGlobalSlot::PATH, "/ployz.rpc.v1.MachineRpc/EnsureGlobalSlot" ); + assert_eq!( + op::RequestMachineUpgrade::PATH, + "/ployz.rpc.v1.MachineRpc/RequestMachineUpgrade" + ); + assert_eq!( + op::InspectMachineUpgrade::PATH, + "/ployz.rpc.v1.MachineRpc/InspectMachineUpgrade" + ); } diff --git a/crates/ployz-core/tests/contract.rs b/crates/ployz-core/tests/contract.rs index c8a8e61326..790c3bba56 100644 --- a/crates/ployz-core/tests/contract.rs +++ b/crates/ployz-core/tests/contract.rs @@ -14,16 +14,18 @@ use ployz_core::{ GetContainerObservationsRequest, GetIngressProxyConfigRequest, HealthObservation, HttpProtocol, ImageIngestDestination, ImageIngestOpened, ImageIngestReason, ImagePulled, ImageSummary, IngressHost, IngressHostname, IngressProxyConfig, IngressProxyFragment, - InspectWireGuardRequest, LIST_IMAGES_CAPABILITY, ListImagesRequest, MANAGED_LABEL, - MachineFailure, MachineGateway, MachineId, MachineImages, MachineName, MachineSubnet, - MachineSuccess, MachineTokenRequest, MachineUpdate, ManagementAddress, NameMatches, - OpaquePayload, PROJECT_NAME_LABEL, PROTOCOL_MAJOR, PULL_IMAGE_FROM_MACHINE_CAPABILITY, - PartialResult, PortPublication, ProjectName, PublicIpDiscovery, PublicIpUpdate, - PullImageFromMachineRequest, QualifiedService, RESET_MACHINE_CAPABILITY, - RemoveLocalMachineRequest, RemoveMachineRequest, RequestedServiceSpec, ReserveDomainRequest, - ResetAccepted, ResetRequest, ResolvedServiceSpec, ResponseKind, RpcError, RpcErrorCode, - RpcRequestBody, RpcResponse, RpcResponseBody, ServiceId, ServiceName, UpdateMachineRequest, - VolumeSource, encode_grpc_frame, grpc_frames, op, + InspectMachineUpgradeRequest, InspectWireGuardRequest, LIST_IMAGES_CAPABILITY, + ListImagesRequest, MANAGED_LABEL, MachineFailure, MachineGateway, MachineId, MachineImages, + MachineName, MachineRelease, MachineSubnet, MachineSuccess, MachineTokenRequest, MachineUpdate, + MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeOutcome, MachineUpgradeStage, + MachineVersion, ManagementAddress, NameMatches, OpaquePayload, PROJECT_NAME_LABEL, + PROTOCOL_MAJOR, PULL_IMAGE_FROM_MACHINE_CAPABILITY, PartialResult, PortPublication, + ProjectName, PublicIpDiscovery, PublicIpUpdate, PullImageFromMachineRequest, QualifiedService, + RESET_MACHINE_CAPABILITY, RemoveLocalMachineRequest, RemoveMachineRequest, + RequestMachineUpgradeRequest, RequestedServiceSpec, ReserveDomainRequest, ResetAccepted, + ResetRequest, ResolvedServiceSpec, ResponseKind, RpcError, RpcErrorCode, RpcRequestBody, + RpcResponse, RpcResponseBody, ServiceId, ServiceName, UpdateMachineRequest, VolumeSource, + encode_grpc_frame, grpc_frames, op, }; use prost::Message; use serde_json::{Value, json}; @@ -1474,6 +1476,57 @@ fn machine_administration_requests_round_trip_as_typed_payloads() { } } +#[test] +fn machine_upgrade_request_and_receipt_have_one_typed_wire_contract() { + let attempt_id = MachineUpgradeAttemptId::parse(MACHINE_ID).unwrap(); + let request = RequestMachineUpgradeRequest { + attempt_id, + release: MachineRelease::parse("beta").unwrap(), + }; + assert_eq!( + op::RequestMachineUpgrade::into_request(request.clone()) + .encode() + .unwrap() + .decode_request() + .unwrap() + .body, + RpcRequestBody::RequestMachineUpgrade(request) + ); + let inspect = InspectMachineUpgradeRequest { + attempt_id: Some(attempt_id), + }; + assert_eq!( + op::InspectMachineUpgrade::into_request(inspect.clone()) + .encode() + .unwrap() + .decode_request() + .unwrap() + .body, + RpcRequestBody::InspectMachineUpgrade(inspect) + ); + + let attempt = MachineUpgradeAttempt { + attempt_id, + target: MachineVersion::parse("1.2.3-beta.4").unwrap(), + outcome: MachineUpgradeOutcome::Failed { + stage: MachineUpgradeStage::Verifying, + error: "checksum mismatch".into(), + }, + }; + let response = RpcResponse::from(attempt.clone()); + assert_eq!( + response + .clone() + .decode::() + .unwrap(), + attempt + ); + assert_eq!( + response.decode::().unwrap(), + attempt + ); +} + #[test] fn service_ingress_proxy_fragment_is_a_validated_caddy_string() { let spec: RequestedServiceSpec = serde_json::from_value(json!({ diff --git a/crates/ployz/Cargo.toml b/crates/ployz/Cargo.toml index e2d7c582fc..7938527221 100644 --- a/crates/ployz/Cargo.toml +++ b/crates/ployz/Cargo.toml @@ -30,6 +30,7 @@ shell-words = "1.1.1" rustix = { version = "1.1.4", features = ["fs", "process", "termios"] } sigpipe = "0.1.3" thiserror = "2.0.20" +tempfile = "3.27.0" tokio = { version = "1.53.1", features = ["io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = "0.1.19" tokio-util = "0.7.19" diff --git a/crates/ployz/src/cli/mod.rs b/crates/ployz/src/cli/mod.rs index 84ca9ea081..8228e280fb 100644 --- a/crates/ployz/src/cli/mod.rs +++ b/crates/ployz/src/cli/mod.rs @@ -380,6 +380,7 @@ fn machine() -> Command { ), ) .subcommand(base("rtt", "Show round-trip times")) + .subcommand(machine_upgrade()) .subcommand( base("update", "Update machine configuration") .arg(value("name", None)) @@ -389,6 +390,33 @@ fn machine() -> Command { ) } +fn machine_upgrade() -> Command { + base("upgrade", "Upgrade explicitly selected machines") + .arg_required_else_help(true) + .subcommand_negates_reqs(true) + .args_conflicts_with_subcommands(true) + .subcommand_precedence_over_arg(true) + .arg( + positional("version", true) + .value_name("VERSION") + .value_parser(clap::value_parser!(ployz_core::MachineRelease)), + ) + .arg( + many("machine", Some('m')) + .required(true) + .help("Machine name or ID; repeat for an explicit sequence"), + ) + .subcommand( + base("inspect", "Inspect one Machine upgrade attempt") + .arg(positional("machine", true)) + .arg( + value("attempt", None) + .value_parser(clap::value_parser!(ployz_core::MachineUpgradeAttemptId)), + ) + .arg(json_output()), + ) +} + fn provisioning_flags(command: Command) -> Command { command .arg(value("name", Some('n'))) diff --git a/crates/ployz/src/cluster.rs b/crates/ployz/src/cluster.rs index f3aa2857cc..b482be18ec 100644 --- a/crates/ployz/src/cluster.rs +++ b/crates/ployz/src/cluster.rs @@ -133,7 +133,27 @@ impl Client { request: T::Request, target: Option<&MachineTarget>, ) -> Result { - let payload = T::into_request(request).encode()?; + self.call_repeatable_for::(request, target, crate::setup_retry::WAIT) + .await + .map_err(|error| match error { + crate::setup_retry::Error::Permanent(error) => error, + crate::setup_retry::Error::Exhausted(message) => { + ConnectError::Attempt(message.into()) + } + }) + } + + /// Retry a read or stable-identity request for the caller's remaining time budget. + pub(crate) async fn call_repeatable_for( + &mut self, + request: T::Request, + target: Option<&MachineTarget>, + wait: Duration, + ) -> Result> { + let payload = T::into_request(request) + .encode() + .map_err(ConnectError::from) + .map_err(crate::setup_retry::Error::Permanent)?; let mut redial = false; let operation = T::PATH.rsplit('/').next().unwrap_or(T::PATH); let destination = target.map_or_else( @@ -144,7 +164,7 @@ impl Client { crate::setup_retry::run( self, &progress, - crate::setup_retry::WAIT, + wait, ConnectError::is_setup_retryable, async |client| { let reconnect = redial; @@ -160,10 +180,6 @@ impl Client { }, ) .await - .map_err(|error| match error { - crate::setup_retry::Error::Permanent(error) => error, - crate::setup_retry::Error::Exhausted(message) => ConnectError::Attempt(message.into()), - }) } /// Setup mutations must not be replayed after a lost response. diff --git a/crates/ployz/src/context.rs b/crates/ployz/src/context.rs index 8bc053bebd..5104d0a6df 100644 --- a/crates/ployz/src/context.rs +++ b/crates/ployz/src/context.rs @@ -543,6 +543,7 @@ impl<'de> Deserialize<'de> for Connection { pub struct SshDestination { value: String, target: String, + copy_target: String, port: Option, } @@ -565,9 +566,15 @@ impl SshDestination { return Err(ConnectionError::SshDestination(value)); } let target = format!("{user}@{host}"); + let copy_target = if host.contains(':') && !host.starts_with('[') { + format!("{user}@[{host}]") + } else { + target.clone() + }; Ok(Self { value, target, + copy_target, port, }) } @@ -582,6 +589,12 @@ impl SshDestination { &self.target } + /// Return the SCP target, with IPv6 hosts bracketed to distinguish its path separator. + #[must_use] + pub(crate) fn copy_target(&self) -> &str { + &self.copy_target + } + #[must_use] pub fn port(&self) -> Option { self.port diff --git a/crates/ployz/src/handlers/cloud.rs b/crates/ployz/src/handlers/cloud.rs index af490186bf..1e1620a863 100644 --- a/crates/ployz/src/handlers/cloud.rs +++ b/crates/ployz/src/handlers/cloud.rs @@ -1,6 +1,6 @@ //! `ployz cloud enroll`: enroll `initialize` or `join` on this Machine. -use std::time::Duration; +use std::{future::Future, time::Duration}; use clap::ArgMatches; use ipnet::Ipv4Net; @@ -16,24 +16,33 @@ use crate::connect::{Client, ConnectError}; use crate::context::{ContextError, Transport}; pub(super) fn enroll(root: &ArgMatches) -> Result<(), Error> { - enroll_with_installer(root, &|| { - crate::provisioning::provision_local(StorageChoice::None).map_err(Into::into) + enroll_with_installer(root, &|storage| async move { + if storage == StorageChoice::Zfs { + crate::provisioning::provision_local(env!("CARGO_PKG_VERSION"), storage).await?; + } else { + crate::provisioning::synchronize_local_daemon().await?; + } + Ok(()) }) } /// Run Cloud enrollment, installing this CLI's daemon version with `install`. /// -/// `install` prepares no storage; tests substitute it to avoid provisioning a -/// real daemon. +/// `install` receives `none` for software-only synchronization and `zfs` for +/// storage preparation; tests substitute it to avoid provisioning a real daemon. /// /// # Errors /// /// Returns the same CLI failure as [`enroll`]. #[doc(hidden)] -pub fn enroll_with_installer( +pub fn enroll_with_installer( root: &ArgMatches, - install: &dyn Fn() -> Result<(), Error>, -) -> Result<(), Error> { + install: &Install, +) -> Result<(), Error> +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let matches = leaf_matches(root); let token = CloudEnrollToken::parse(required(matches, "token")?)?; let cloud_url = matches @@ -60,7 +69,7 @@ pub fn enroll_with_installer( let (details, machine_token, name, outcome) = enroll_current_identity(&mut client, requested_name, requested_storage, &url).await?; match outcome { - Outcome::Join(join) => enroll_join(matches, client, details, *join).await, + Outcome::Join(join) => enroll_join(matches, client, details, *join, install).await, Outcome::Initialize { mode, pairing, @@ -78,6 +87,7 @@ pub fn enroll_with_installer( storage, cloud_url, &token, + install, ) .await } @@ -112,12 +122,17 @@ async fn enroll_current_identity( Ok((details, machine_token, name, outcome)) } -async fn enroll_join( +async fn enroll_join( matches: &ArgMatches, mut client: Client, details: MachineDetails, join: Join, -) -> Result<(), Error> { + install: &Install, +) -> Result<(), Error> +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let assigned = join.registration.assigned_machine.clone(); if already_assigned(&details, &assigned) { println!("Initialised Machine {} ({})", assigned.name, assigned.id); @@ -131,7 +146,7 @@ async fn enroll_join( client, ) .await?; - provision_storage(&client, join.storage)?; + client = provision_storage(matches, client, join.storage, install).await?; crate::handlers::machine::join( &mut client, JoinRequest { @@ -171,7 +186,7 @@ enum FounderLocalState { clippy::too_many_arguments, reason = "the founder tail consumes the existing cloud-enroll command interface" )] -async fn enroll_founder( +async fn enroll_founder( matches: &ArgMatches, mut client: Client, details: MachineDetails, @@ -183,7 +198,12 @@ async fn enroll_founder( storage: StorageChoice, cloud_url: &str, token: &CloudEnrollToken, -) -> Result<(), Error> { + install: &Install, +) -> Result<(), Error> +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let state = match (mode, details.phase) { (InitializeMode::Resume, LocalMachinePhase::Participating) => FounderLocalState::Resume { machine: Box::new(details.machine.ok_or_else(|| { @@ -223,7 +243,7 @@ async fn enroll_founder( client, ) .await?; - provision_storage(&client, storage)?; + client = provision_storage(matches, client, storage, install).await?; let initialized = crate::handlers::machine::initialize( &mut client, InitializeRequest { @@ -277,10 +297,19 @@ async fn enroll_founder( Ok(()) } -fn provision_storage(client: &Client, storage: StorageChoice) -> Result<(), Error> { +async fn provision_storage( + matches: &ArgMatches, + client: Client, + storage: StorageChoice, + install: &Install, +) -> Result +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ crate::provisioning::announce_storage(storage); if storage != StorageChoice::Zfs { - return Ok(()); + return Ok(client); } if !matches!(client.connection().transport(), Transport::Unix(_)) { return Err(Error::usage(format!( @@ -288,15 +317,19 @@ fn provision_storage(client: &Client, storage: StorageChoice) -> Result<(), Erro client.connection() ))); } - crate::provisioning::provision_local(storage)?; - Ok(()) + install(storage).await?; + wait_matching_daemon(matches).await } -async fn synchronize_daemon( +async fn synchronize_daemon( matches: &ArgMatches, mut client: Client, - install: &dyn Fn() -> Result<(), Error>, -) -> Result { + install: &Install, +) -> Result +where + Install: Fn(StorageChoice) -> InstallFuture, + InstallFuture: Future>, +{ let daemon = client .call_repeatable::(DescribeContractRequest {}, None) .await?; @@ -309,7 +342,11 @@ async fn synchronize_daemon( client.connection() ))); } - install()?; + install(StorageChoice::None).await?; + wait_matching_daemon(matches).await +} + +async fn wait_matching_daemon(matches: &ArgMatches) -> Result { let mut client = wait_client(matches).await?; let daemon = client .call_repeatable::(DescribeContractRequest {}, None) @@ -337,7 +374,11 @@ async fn connect_machine(matches: &ArgMatches) -> Result { { Ok(client) => Ok(client), Err(ConnectError::Context(ContextError::NoConfig)) => { - crate::provisioning::provision_local(ployz_core::StorageChoice::None)?; + crate::provisioning::provision_local( + env!("CARGO_PKG_VERSION"), + ployz_core::StorageChoice::None, + ) + .await?; wait_client(matches).await } Err(error) => Err(error.into()), diff --git a/crates/ployz/src/handlers/machine.rs b/crates/ployz/src/handlers/machine.rs index 39bd984bab..999257afde 100644 --- a/crates/ployz/src/handlers/machine.rs +++ b/crates/ployz/src/handlers/machine.rs @@ -18,6 +18,7 @@ mod helpers; mod init; mod inspect; mod remove; +mod upgrade; pub(super) use add::add; pub(super) use helpers::{ @@ -26,6 +27,7 @@ pub(super) use helpers::{ pub(super) use init::init; pub(super) use inspect::{inspect, list, rtt, wireguard_show}; pub(super) use remove::remove; +pub(super) use upgrade::{inspect as inspect_upgrade, upgrade}; const DEFAULT_WIREGUARD_PORT: u16 = 51820; diff --git a/crates/ployz/src/handlers/machine/add.rs b/crates/ployz/src/handlers/machine/add.rs index cee88c789a..f63d7d4c9d 100644 --- a/crates/ployz/src/handlers/machine/add.rs +++ b/crates/ployz/src/handlers/machine/add.rs @@ -27,14 +27,15 @@ pub(in crate::handlers) fn add(root: &ArgMatches) -> Result<(), Error> { let wireguard_mtu = matches.get_one::("wg-mtu").copied(); let yes = matches.get_flag("yes"); let storage = crate::provisioning::resolve_storage(matches)?; - if !matches.get_flag("no-install") { - crate::provisioning::provision(matches, storage)?; - } - - let assigned = runtime()?.block_on(async { + let no_install = matches.get_flag("no-install"); + let runtime = runtime()?; + let assigned = runtime.block_on(async { + if !no_install { + crate::provisioning::provision(matches, storage).await?; + } let mut entry = connect_client(matches, options.context()).await?; let visible = entry.machines().await?; - let mut target_client = if matches.get_flag("no-install") { + let mut target_client = if no_install { helpers::connect_direct(matches, &connection).await? } else { helpers::reconnect_direct(matches, &connection).await? @@ -100,13 +101,13 @@ pub(in crate::handlers) fn add(root: &ArgMatches) -> Result<(), Error> { config.save()?; println!("{}", added_machine_line(&assigned)); - runtime()?.block_on(helpers::wait_direct_participating( + runtime.block_on(helpers::wait_direct_participating( matches, &connection, "added Machine did not become ready", ))?; - let catch_up = runtime()?.block_on(async { + let catch_up = runtime.block_on(async { let mut entry = super::super::reconnect_client(matches, options.context()).await?; Ok::<_, Error>( crate::global_catch_up::catch_up_globals(&mut entry, &assigned, !deploy_ingress).await, @@ -120,7 +121,7 @@ pub(in crate::handlers) fn add(root: &ArgMatches) -> Result<(), Error> { crate::global_catch_up::joined_catch_up_error(error) ))); } - let dns_result = runtime()?.block_on(async { + let dns_result = runtime.block_on(async { let mut entry = super::super::reconnect_client(matches, options.context()).await?; crate::dns::update_records_if_reserved(&mut entry).await?; Ok::<_, Error>(()) diff --git a/crates/ployz/src/handlers/machine/init.rs b/crates/ployz/src/handlers/machine/init.rs index 2cede88c59..f3876a82d7 100644 --- a/crates/ployz/src/handlers/machine/init.rs +++ b/crates/ployz/src/handlers/machine/init.rs @@ -50,15 +50,18 @@ pub(in crate::handlers) fn init(root: &ArgMatches) -> Result<(), Error> { let yes = matches.get_flag("yes"); let no_install = matches.get_flag("no-install"); let storage = crate::provisioning::resolve_storage(matches)?; - if !no_install { - if local { - crate::provisioning::provision_local(storage)?; - } else { - crate::provisioning::provision(matches, storage)?; + let version = matches + .get_one::("version") + .expect("version has a default"); + let runtime = runtime()?; + let (machine, connection) = runtime.block_on(async { + if !no_install { + if local { + crate::provisioning::provision_local(version, storage).await?; + } else { + crate::provisioning::provision(matches, storage).await?; + } } - } - - let (machine, connection) = runtime()?.block_on(async { let mut target = if !no_install { helpers::reconnect_direct(matches, &connection).await? } else { @@ -117,7 +120,7 @@ pub(in crate::handlers) fn init(root: &ArgMatches) -> Result<(), Error> { &context_name, &["machine", "inspect", machine.name.as_str()], ); - runtime()?.block_on(async { + runtime.block_on(async { let mut ready = helpers::wait_direct_participating(matches, &connection, "initial Machine did not become ready") .await.map_err(|error| Error::usage(format!("Machine initialized; startup incomplete: {error}\nInspect with: {inspect_recovery}")))?; diff --git a/crates/ployz/src/handlers/machine/remove.rs b/crates/ployz/src/handlers/machine/remove.rs index a64d86f47f..80dc96f41e 100644 --- a/crates/ployz/src/handlers/machine/remove.rs +++ b/crates/ployz/src/handlers/machine/remove.rs @@ -112,7 +112,7 @@ pub(in crate::handlers) fn remove(root: &ArgMatches) -> Result<(), Error> { }) } -fn select_machine( +pub(super) fn select_machine( machines: &[ployz_core::MachineObservation], selector: &str, ) -> Result { diff --git a/crates/ployz/src/handlers/machine/upgrade.rs b/crates/ployz/src/handlers/machine/upgrade.rs new file mode 100644 index 0000000000..d0c9392e7e --- /dev/null +++ b/crates/ployz/src/handlers/machine/upgrade.rs @@ -0,0 +1,428 @@ +//! Explicit, bounded Machine upgrade sequencing and inspection. + +use std::{collections::BTreeSet, time::Duration}; + +use clap::ArgMatches; +use ployz_core::{ + InspectMachineUpgradeRequest, Machine, MachineRelease, MachineTarget, MachineUpgradeAttempt, + MachineUpgradeAttemptId, MachineUpgradeOutcome, RequestMachineUpgradeRequest, op, +}; +use tokio::time::Instant; + +use crate::{cluster::Client, connect::ConnectError}; + +use super::super::{Error, leaf_matches, string_values, with_client}; + +const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(16 * 60); +const POLL_INTERVAL: Duration = Duration::from_secs(1); + +trait UpgradeRequests { + async fn request_upgrade( + &mut self, + request: RequestMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result>; + + async fn inspect_upgrade( + &mut self, + request: InspectMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result>; +} + +impl UpgradeRequests for Client { + async fn request_upgrade( + &mut self, + request: RequestMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result> { + self.call_repeatable_for::(request, Some(target), wait) + .await + } + + async fn inspect_upgrade( + &mut self, + request: InspectMachineUpgradeRequest, + target: &MachineTarget, + wait: Duration, + ) -> Result> { + self.call_repeatable_for::(request, Some(target), wait) + .await + } +} + +pub(in crate::handlers) fn upgrade(root: &ArgMatches) -> Result<(), Error> { + let matches = leaf_matches(root); + let release = matches + .get_one::("version") + .cloned() + .ok_or_else(|| Error::usage("upgrade version is required"))?; + let selectors = string_values(matches, "machine"); + with_client(root, |client| { + Box::pin(async move { + let machines = selected_machines(client, &selectors).await?; + for (index, machine) in machines.iter().enumerate() { + let attempt_id = MachineUpgradeAttemptId::random(); + let attempt = match run_one(client, machine, release.clone(), attempt_id).await { + Ok(attempt) => attempt, + Err(error) => { + print_unattempted(machines.iter().skip(index.saturating_add(1)), machine); + return Err(error); + } + }; + print_attempt(machine, &attempt); + match attempt.outcome { + MachineUpgradeOutcome::Succeeded { .. } => {} + MachineUpgradeOutcome::Failed { error, .. } => { + print_unattempted(machines.iter().skip(index.saturating_add(1)), machine); + return Err(Error::usage(error)); + } + MachineUpgradeOutcome::Interrupted { .. } => { + print_unattempted(machines.iter().skip(index.saturating_add(1)), machine); + return Err(Error::usage(format!( + "Machine {} upgrade was interrupted; {}", + machine.name, + journal_hint(attempt_id) + ))); + } + MachineUpgradeOutcome::Accepted | MachineUpgradeOutcome::Running { .. } => { + unreachable!("run_one returns only terminal evidence") + } + } + } + Ok(()) + }) + }) +} + +pub(in crate::handlers) fn inspect(root: &ArgMatches) -> Result<(), Error> { + let matches = leaf_matches(root); + let target = MachineTarget::parse( + matches + .get_one::("machine") + .ok_or_else(|| Error::usage("machine is required"))?, + )?; + let attempt_id = matches + .get_one::("attempt") + .copied(); + let json = matches.get_one::("output").is_some(); + with_client(root, |client| { + Box::pin(async move { + let attempt = client + .call_repeatable::( + InspectMachineUpgradeRequest { attempt_id }, + Some(&target), + ) + .await?; + if json { + println!("{}", serde_json::to_string_pretty(&attempt)?); + } else { + print_attempt_target(target.as_str(), &attempt); + } + Ok(()) + }) + }) +} + +async fn selected_machines( + client: &mut Client, + selectors: &[String], +) -> Result, Error> { + let visible = client.machines().await?; + let mut selected = Vec::with_capacity(selectors.len()); + let mut ids = BTreeSet::new(); + for selector in selectors { + let machine = super::remove::select_machine(&visible, selector)?; + if !ids.insert(machine.id) { + return Err(Error::usage(format!( + "Machine {} was selected more than once", + machine.name + ))); + } + selected.push(machine); + } + Ok(selected) +} + +async fn run_one( + client: &mut impl UpgradeRequests, + machine: &Machine, + release: MachineRelease, + attempt_id: MachineUpgradeAttemptId, +) -> Result { + let target = MachineTarget::from(&machine.id); + let deadline = Instant::now() + OBSERVATION_TIMEOUT; + let request = RequestMachineUpgradeRequest { + attempt_id, + release, + }; + let accepted = match client + .request_upgrade( + request, + &target, + deadline.saturating_duration_since(Instant::now()), + ) + .await + { + Ok(accepted) => accepted, + Err(crate::setup_retry::Error::Permanent(error)) => return Err(error.into()), + Err(crate::setup_retry::Error::Exhausted(error)) => { + return Err(uncertain(machine, attempt_id, error)); + } + }; + print_attempt(machine, &accepted); + if accepted.is_terminal() { + return Ok(accepted); + } + + loop { + if tokio::time::timeout_at(deadline, tokio::time::sleep(POLL_INTERVAL)) + .await + .is_err() + { + return Err(uncertain_timeout(machine, attempt_id)); + } + let observed = client + .inspect_upgrade( + InspectMachineUpgradeRequest { + attempt_id: Some(attempt_id), + }, + &target, + deadline.saturating_duration_since(Instant::now()), + ) + .await + .map_err(|error| match error { + crate::setup_retry::Error::Permanent(error) => error.into(), + crate::setup_retry::Error::Exhausted(error) => { + uncertain(machine, attempt_id, error) + } + })?; + if observed.is_terminal() { + return Ok(observed); + } + } +} + +fn print_attempt(machine: &Machine, attempt: &MachineUpgradeAttempt) { + print_attempt_target(&format!("{} ({})", machine.name, machine.id), attempt); +} + +fn print_attempt_target(machine: &str, attempt: &MachineUpgradeAttempt) { + let MachineUpgradeAttempt { + attempt_id, + target, + outcome, + } = attempt; + match outcome { + MachineUpgradeOutcome::Accepted => println!( + "Machine {machine}: upgrade {attempt_id} accepted for {target}; {}", + journal_hint(*attempt_id) + ), + MachineUpgradeOutcome::Running { stage } => println!( + "Machine {machine}: upgrade {attempt_id} is {} for {target}; {}", + stage.as_str(), + journal_hint(*attempt_id) + ), + MachineUpgradeOutcome::Succeeded { version } => { + println!("Machine {machine}: upgrade {attempt_id} succeeded; running version {version}") + } + MachineUpgradeOutcome::Failed { stage, error } => println!( + "Machine {machine}: upgrade {attempt_id} failed at {} for {target}: {error}; {}", + stage.as_str(), + journal_hint(*attempt_id) + ), + MachineUpgradeOutcome::Interrupted { stage } => println!( + "Machine {machine}: upgrade {attempt_id} was interrupted at {} for {target}; {}", + stage.as_str(), + journal_hint(*attempt_id) + ), + } +} + +fn print_unattempted<'a>(machines: impl IntoIterator, after: &Machine) { + for line in unattempted_lines(machines, after) { + println!("{line}"); + } +} + +fn unattempted_lines<'a>( + machines: impl IntoIterator, + after: &Machine, +) -> Vec { + machines + .into_iter() + .map(|machine| { + format!( + "Machine {} ({}): upgrade unattempted after {} ({})", + machine.name, machine.id, after.name, after.id + ) + }) + .collect() +} + +fn journal_hint(attempt_id: MachineUpgradeAttemptId) -> String { + format!("inspect locally with `journalctl -u ployz-upgrade-{attempt_id}.service`") +} + +fn uncertain( + machine: &Machine, + attempt_id: MachineUpgradeAttemptId, + error: impl std::fmt::Display, +) -> Error { + Error::usage(format!( + "Machine {} ({}) upgrade {attempt_id} outcome is uncertain: {error}; reconnect and run `ployz machine upgrade inspect {} --attempt {attempt_id}`; {}", + machine.name, + machine.id, + machine.id, + journal_hint(attempt_id) + )) +} + +fn uncertain_timeout(machine: &Machine, attempt_id: MachineUpgradeAttemptId) -> Error { + Error::usage(format!( + "Machine {} ({}) upgrade {attempt_id} outcome is uncertain after {} minutes; reconnect and run `ployz machine upgrade inspect {} --attempt {attempt_id}`; {}", + machine.name, + machine.id, + OBSERVATION_TIMEOUT.as_secs() / 60, + machine.id, + journal_hint(attempt_id) + )) +} + +#[cfg(test)] +mod tests { + use ployz_core::{ + AdvertisedEndpoint, MachineId, MachineName, MachineRuntime, RpcError, RpcErrorCode, + WireGuardPublicKey, + }; + use serde_json::Value; + + use super::*; + + struct FakeRequests { + request: Option>>, + seen: Vec<(MachineUpgradeAttemptId, String)>, + } + + impl UpgradeRequests for FakeRequests { + async fn request_upgrade( + &mut self, + request: RequestMachineUpgradeRequest, + target: &MachineTarget, + _wait: Duration, + ) -> Result> { + self.seen + .push((request.attempt_id, target.as_str().to_owned())); + self.request.take().expect("one request result") + } + + async fn inspect_upgrade( + &mut self, + _request: InspectMachineUpgradeRequest, + _target: &MachineTarget, + _wait: Duration, + ) -> Result> { + panic!("an unaccepted request is not inspected") + } + } + + #[tokio::test] + async fn definitive_busy_rejection_is_not_reported_as_uncertain() { + let machine = machine('a', 1); + let attempt_id = MachineUpgradeAttemptId::parse("1".repeat(32)).unwrap(); + let mut client = FakeRequests { + request: Some(Err(crate::setup_retry::Error::Permanent( + ConnectError::Remote(RpcError { + code: RpcErrorCode::Conflict, + message: "a Machine upgrade or mutation is active".into(), + details: Value::Null, + }), + ))), + seen: Vec::new(), + }; + + let error = run_one( + &mut client, + &machine, + MachineRelease::parse("beta").unwrap(), + attempt_id, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("upgrade or mutation is active")); + assert!(!error.to_string().contains("outcome is uncertain")); + assert_eq!(client.seen, [(attempt_id, machine.id.as_str().to_owned())]); + } + + #[tokio::test] + async fn lost_request_reply_is_uncertain_and_keeps_the_dispatched_attempt_id() { + let machine = machine('b', 2); + let attempt_id = MachineUpgradeAttemptId::parse("2".repeat(32)).unwrap(); + let mut client = FakeRequests { + request: Some(Err(crate::setup_retry::Error::Exhausted( + "request reply was lost".into(), + ))), + seen: Vec::new(), + }; + + let error = run_one( + &mut client, + &machine, + MachineRelease::parse("1.2.3").unwrap(), + attempt_id, + ) + .await + .unwrap_err() + .to_string(); + + assert!(error.contains("outcome is uncertain"), "{error}"); + assert!(error.contains(attempt_id.as_str()), "{error}"); + assert!(error.contains("machine upgrade inspect"), "{error}"); + assert_eq!(client.seen, [(attempt_id, machine.id.as_str().to_owned())]); + } + + #[test] + fn first_and_middle_failures_report_the_complete_ordered_suffix() { + let machines = [ + machine('a', 1), + machine('b', 2), + machine('c', 3), + machine('d', 4), + ]; + + let first = unattempted_lines(&machines[1..], &machines[0]); + assert_eq!( + first, + [ + "Machine b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb): upgrade unattempted after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)", + "Machine c (cccccccccccccccccccccccccccccccc): upgrade unattempted after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)", + "Machine d (dddddddddddddddddddddddddddddddd): upgrade unattempted after a (aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)", + ] + ); + + let middle = unattempted_lines(&machines[2..], &machines[1]); + assert_eq!( + middle, + [ + "Machine c (cccccccccccccccccccccccccccccccc): upgrade unattempted after b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)", + "Machine d (dddddddddddddddddddddddddddddddd): upgrade unattempted after b (bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb)", + ] + ); + } + + fn machine(id: char, subnet: u8) -> Machine { + Machine { + id: MachineId::parse(id.to_string().repeat(32)).unwrap(), + name: MachineName::parse(id.to_string()).unwrap(), + subnet: format!("10.210.{subnet}.0/24").parse().unwrap(), + public_key: WireGuardPublicKey([subnet; 32]), + public_ip: None, + advertised_endpoints: Vec::::new(), + runtime: MachineRuntime::default(), + } + } +} diff --git a/crates/ployz/src/handlers/mod.rs b/crates/ployz/src/handlers/mod.rs index 1528fda778..e8502809fd 100644 --- a/crates/ployz/src/handlers/mod.rs +++ b/crates/ployz/src/handlers/mod.rs @@ -249,6 +249,8 @@ fn handler_for(path: &str) -> Option { "machine rm" => machine::remove, "machine rtt" => machine::rtt, "machine update" => machine::update, + "machine upgrade" => machine::upgrade, + "machine upgrade inspect" => machine::inspect_upgrade, "proxy" => operator::proxy, "project ls" => project::list, "project rm" => project::remove, diff --git a/crates/ployz/src/provisioning.rs b/crates/ployz/src/provisioning.rs index 7f38ff37fd..3050d05b0d 100644 --- a/crates/ployz/src/provisioning.rs +++ b/crates/ployz/src/provisioning.rs @@ -1,46 +1,127 @@ use std::{ + env, + ffi::OsString, io::{self, IsTerminal, Write}, - path::PathBuf, + path::{Path, PathBuf}, process::{Command, Stdio}, }; -use base64::{Engine, engine::general_purpose::STANDARD}; use clap::ArgMatches; use ployz_core::StorageChoice; use thiserror::Error; +use uuid::Uuid; +use crate::context::{Connection, ConnectionError, SshDestination, Transport}; + +mod bootstrap; + +use bootstrap::Bootstrap; + +/// A failure while preparing a Machine through the shared daemon installer. #[derive(Debug, Error)] pub enum ProvisionError { - #[error("ssh+go provisioning is not implemented; use system ssh")] - SshGo, - #[error("remote machine destination is empty")] - EmptyDestination, + /// The remote setup command did not include a destination. #[error("remote machine destination is required")] MissingDestination, + /// The requested connection is not an SSH connection. + #[error("Machine provisioning requires an SSH destination, not {0}")] + RemoteTransport(String), + /// The SSH destination is malformed or uses a removed transport spelling. + #[error(transparent)] + Connection(#[from] ConnectionError), + /// The local OpenSSH client is unavailable. #[error("local ssh client not found; install an ssh client")] SshClientMissing(#[source] io::Error), + /// The initial remote identity command could not be run. #[error("run ssh whoami: {0}")] Whoami(#[source] io::Error), + /// The initial remote identity command exited unsuccessfully. #[error("ssh whoami failed: {0}")] WhoamiFailed(String), + /// The remote identity was not valid UTF-8. #[error("ssh whoami returned non-UTF-8 output")] WhoamiUtf8, + /// The remote identity command returned no user. #[error("ssh whoami returned an empty user")] EmptyUser, + /// The remote sudo preflight could not be run. #[error("check remote sudo: {0}")] Sudo(#[source] io::Error), + /// A non-root remote user could not authenticate with sudo. #[error("remote user {user} could not authenticate or obtain sudo privileges to install Ployz")] SudoRequired { user: String }, + /// The remote platform inspection command could not be run. + #[error("inspect remote Machine platform: {0}")] + Platform(#[source] io::Error), + /// The remote platform inspection command exited unsuccessfully. + #[error("remote Machine platform inspection failed: {0}")] + PlatformFailed(String), + /// The remote platform response was not valid UTF-8. + #[error("remote Machine platform inspection returned non-UTF-8 output")] + PlatformUtf8, + /// The target host cannot run the Linux Machine daemon. + #[error("Ployz Machine must be Linux")] + UnsupportedOs, + /// No daemon release archive exists for the target architecture. + #[error("unsupported Machine architecture: {0}")] + UnsupportedArchitecture(String), + /// A bootstrap filesystem or process operation failed. + #[error("{stage}: {source}")] + BootstrapIo { + stage: &'static str, + #[source] + source: io::Error, + }, + /// A bootstrap process exited unsuccessfully. + #[error("{stage} exited with {status}")] + BootstrapCommand { + stage: &'static str, + status: std::process::ExitStatus, + }, + /// A bootstrap checksum or version did not match the current CLI release. + #[error("bootstrap verification: {0}")] + BootstrapVerification(String), + /// A published bootstrap release could not be downloaded. + #[error("{stage}: {source}")] + BootstrapDownload { + stage: &'static str, + #[source] + source: reqwest::Error, + }, + /// The bootstrap or its local release files could not be copied to the Machine. + #[error("transfer bootstrap release: {0}")] + Transfer(#[source] io::Error), + /// Remote staging or transfer exited unsuccessfully. + #[error("bootstrap release transfer exited with {status}")] + TransferFailed { status: std::process::ExitStatus }, + /// The shared Machine installer could not be spawned. #[error("run Ployz installer: {0}")] Install(#[source] io::Error), + /// The shared Machine installer exited unsuccessfully. #[error("Ployz installer exited with {status}")] InstallFailed { status: std::process::ExitStatus }, + /// Remote bootstrap cleanup could not be run. + #[error("remove remote bootstrap: {0}")] + Cleanup(#[source] io::Error), + /// Remote bootstrap cleanup exited unsuccessfully. + #[error("remote bootstrap cleanup exited with {status}")] + CleanupFailed { status: std::process::ExitStatus }, + /// Setup failed and the subsequent remote cleanup also failed. + #[error("{primary}; cleanup: {cleanup}")] + CleanupAfter { + primary: Box, + cleanup: Box, + }, + /// Local Machine installation requires root privileges. #[error("run this command with sudo")] NotRoot, + /// Reading an interactive storage selection failed. #[error("read storage choice: {0}")] StorageInput(#[source] io::Error), + /// The selected storage value is invalid. #[error(transparent)] StorageChoice(#[from] ployz_core::ValueError), + /// ZFS was selected while installation was explicitly disabled. #[error("zfs storage preparation requires the installer; remove --no-install")] ZfsWithoutInstaller, } @@ -85,64 +166,195 @@ pub(crate) fn announce_storage(storage: StorageChoice) { } } -fn shell_quote(value: &str) -> String { - format!("'{}'", value.replace('\'', "'\"'\"'")) +enum Preparation<'user> { + Host { + storage: StorageChoice, + group_user: Option<&'user str>, + }, + SoftwareOnly, } -fn encoded_installer() -> String { - STANDARD.encode(include_bytes!("../../../scripts/install.sh")) +fn install_arguments( + version: &str, + preparation: Preparation<'_>, + release_dir: Option<&Path>, +) -> Vec { + let mut arguments = vec!["install".into(), "--version".into(), version.into()]; + match preparation { + Preparation::Host { + storage, + group_user, + } => { + arguments.extend(["--storage".into(), storage.as_str().into()]); + if let Some(user) = group_user { + arguments.extend(["--group-user".into(), user.into()]); + } + } + Preparation::SoftwareOnly => arguments.push("--software-only".into()), + } + if let Some(directory) = release_dir { + arguments.extend(["--release-dir".into(), directory.as_os_str().to_owned()]); + } + arguments } -fn pipefail(pipeline: &str) -> String { - format!("set -o pipefail; {pipeline}") +struct Remote { + destination: SshDestination, + key: PathBuf, + timeout: std::time::Duration, + control_path: Option, } -fn install_command( - script: &str, - version: &str, - group_user: Option<&str>, - via_sudo: bool, - storage: StorageChoice, -) -> String { - let script = shell_quote(script); - let version = shell_quote(version); - let group = group_user - .map(|user| format!("PLOYZ_GROUP_ADD_USER={} ", shell_quote(user))) - .unwrap_or_default(); - let sudo = if via_sudo { "sudo " } else { "" }; - format!( - "printf '%s' {script} | base64 -d | {sudo}{group}PLOYZ_STORAGE={} PLOYZ_VERSION={version} bash", - shell_quote(storage.as_str()) - ) -} +impl Remote { + fn from_matches(matches: &ArgMatches) -> Result { + let value = matches + .get_one::("destination") + .ok_or(ProvisionError::MissingDestination)?; + let connection = value.parse::()?; + let Transport::Ssh { destination, .. } = connection.transport() else { + return Err(ProvisionError::RemoteTransport(connection.to_string())); + }; + Ok(Self { + destination: destination.clone(), + key: ssh_key(matches), + timeout: crate::cli::ssh_timeout(matches), + control_path: crate::connect::control_path(), + }) + } -fn installer_status(result: io::Result) -> Result<(), ProvisionError> { - let status = result.map_err(ProvisionError::Install)?; - if status.success() { - Ok(()) - } else { - Err(ProvisionError::InstallFailed { status }) + fn ssh(&self) -> Command { + let mut command = Command::new("ssh"); + command.args(self.common_arguments()); + if let Some(port) = self.destination.port() { + command.args([OsString::from("-p"), port.to_string().into()]); + } + // Provisioning may wait for human authentication; only network setup is timed. + command.args(["-tt", self.destination.target()]); + command.stdin(Stdio::inherit()); + command + } + + fn scp(&self) -> Command { + let mut command = Command::new("scp"); + command.args(self.common_arguments()); + if let Some(port) = self.destination.port() { + command.args([OsString::from("-P"), port.to_string().into()]); + } + command } -} -fn ssh_parts(destination: &str) -> Result<(String, Option), ProvisionError> { - if destination.starts_with("ssh+go://") { - return Err(ProvisionError::SshGo); + fn common_arguments(&self) -> Vec { + let mut arguments: Vec = + crate::connect::ssh_control_args(self.control_path.as_deref()) + .into_iter() + .map(Into::into) + .collect(); + arguments.extend([ + "-o".into(), + format!("ConnectTimeout={}", self.timeout.as_secs().max(1)).into(), + "-o".into(), + "BatchMode=no".into(), + "-i".into(), + self.key.as_os_str().to_owned(), + ]); + arguments } - let destination = destination - .strip_prefix("ssh://") - .or_else(|| destination.strip_prefix("ssh+cli://")) - .unwrap_or(destination); - if destination.is_empty() { - return Err(ProvisionError::EmptyDestination); + + async fn platform(&self) -> Result { + let mut command = self.ssh(); + command.arg("uname -s; uname -m"); + let output = tokio::process::Command::from(command) + .output() + .await + .map_err(ProvisionError::Platform)?; + if !output.status.success() { + return Err(ProvisionError::PlatformFailed( + String::from_utf8_lossy(&output.stderr).trim().to_owned(), + )); + } + let output = String::from_utf8(output.stdout).map_err(|_| ProvisionError::PlatformUtf8)?; + let mut lines = output + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()); + if lines.next() != Some("Linux") { + return Err(ProvisionError::UnsupportedOs); + } + lines + .next() + .map(str::to_owned) + .ok_or_else(|| ProvisionError::PlatformFailed("uname returned no architecture".into())) } - if let Some((host, port)) = destination.rsplit_once(':') - && !port.is_empty() - && port.chars().all(|character| character.is_ascii_digit()) - { - return Ok((host.to_owned(), Some(port.to_owned()))); + + async fn stage( + &self, + bootstrap: &Bootstrap, + version: &str, + directory: &str, + ) -> Result<(), ProvisionError> { + let mut create = self.ssh(); + create.arg(format!("mkdir -m 700 -- {}", shell_quote(directory))); + transfer_status(tokio::process::Command::from(create).status().await)?; + + let mut transfer = self.scp(); + transfer.arg(bootstrap.daemon()); + if let Some(files) = bootstrap.release_files(version) { + transfer.args(files); + } + transfer.arg(format!("{}:{}/", self.destination.copy_target(), directory)); + transfer_status(tokio::process::Command::from(transfer).status().await) + } + + async fn verify(&self, daemon: &str) -> Result<(), ProvisionError> { + let mut command = self.ssh(); + command.arg(format!( + "chmod 700 {path} && {path} version", + path = shell_quote(daemon) + )); + bootstrap::verify_command_version("preflight remote bootstrap daemon", command).await + } + + async fn install( + &self, + daemon: &str, + arguments: &[OsString], + via_sudo: bool, + ) -> Result<(), ProvisionError> { + let mut command = self.ssh(); + command.arg(remote_command(daemon, arguments, via_sudo)); + installer_status(tokio::process::Command::from(command).status().await) + } + + async fn cleanup(&self, directory: &str) -> Result<(), ProvisionError> { + let mut cleanup = self.ssh(); + cleanup.arg(format!("rm -rf -- {}", shell_quote(directory))); + let status = tokio::process::Command::from(cleanup) + .status() + .await + .map_err(ProvisionError::Cleanup)?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::CleanupFailed { status }) + } + } + + async fn close_control_master(&self) { + let Some(control_path) = self.control_path.as_deref() else { + return; + }; + let mut command = Command::new("ssh"); + command.args(crate::connect::ssh_control_args(Some(control_path))); + if let Some(port) = self.destination.port() { + command.arg("-p").arg(port.to_string()); + } + command.args(["-O", "exit", self.destination.target()]); + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let _ = tokio::process::Command::from(command).status().await; } - Ok((destination.to_owned(), None)) } fn ssh_key(matches: &ArgMatches) -> PathBuf { @@ -152,7 +364,7 @@ fn ssh_key(matches: &ArgMatches) -> PathBuf { key.strip_prefix("~/").map_or_else( || PathBuf::from(key), |relative| { - std::env::var_os("HOME") + env::var_os("HOME") .map(PathBuf::from) .unwrap_or_default() .join(relative) @@ -160,35 +372,71 @@ fn ssh_key(matches: &ArgMatches) -> PathBuf { ) } -fn ssh_command(matches: &ArgMatches) -> Result<(Command, String), ProvisionError> { - let destination = matches - .get_one::("destination") - .ok_or(ProvisionError::MissingDestination)?; - let (destination, port) = ssh_parts(destination)?; - let mut command = Command::new("ssh"); - command.arg("-o").arg(format!( - "ConnectTimeout={}", - crate::cli::ssh_timeout(matches).as_secs() - )); - // Provisioning may wait for human authentication; only network setup is timed. - command.args(["-o", "BatchMode=no", "-tt"]); - command.args(crate::connect::ssh_control_args( - crate::connect::control_path().as_deref(), - )); - command.stdin(Stdio::inherit()); - command.arg("-i").arg(ssh_key(matches)); - if let Some(port) = port { - command.arg("-p").arg(port); - } - Ok((command, destination)) +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\"'\"'")) } -pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), ProvisionError> { - let (mut whoami, destination) = ssh_command(matches)?; - let output = whoami - .arg(&destination) - .arg("whoami") +fn remote_command(program: &str, arguments: &[OsString], via_sudo: bool) -> String { + let mut command = String::new(); + if via_sudo { + command.push_str("sudo "); + } + command.push_str(&shell_quote(program)); + for argument in arguments { + command.push(' '); + command.push_str(&shell_quote(&argument.to_string_lossy())); + } + command +} + +fn transfer_status(result: io::Result) -> Result<(), ProvisionError> { + let status = result.map_err(ProvisionError::Transfer)?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::TransferFailed { status }) + } +} + +fn installer_status(result: io::Result) -> Result<(), ProvisionError> { + let status = result.map_err(ProvisionError::Install)?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::InstallFailed { status }) + } +} + +fn finish_remote( + primary: Result<(), ProvisionError>, + cleanup: Result<(), ProvisionError>, +) -> Result<(), ProvisionError> { + match (primary, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(primary), Ok(())) => Err(primary), + (Ok(()), Err(cleanup)) => Err(cleanup), + (Err(primary), Err(cleanup)) => Err(ProvisionError::CleanupAfter { + primary: Box::new(primary), + cleanup: Box::new(cleanup), + }), + } +} + +/// Prepare a remote Linux Machine through its verified bootstrap daemon. +/// +/// # Errors +/// +/// Returns a [`ProvisionError`] when the SSH target or privilege preflight is +/// invalid, when platform inspection, bootstrap acquisition, checksum or version +/// verification fails, when staging or installation fails, or when the staged +/// bootstrap cannot be removed. +pub async fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), ProvisionError> { + let remote = Remote::from_matches(matches)?; + let mut whoami = remote.ssh(); + whoami.arg("whoami"); + let output = tokio::process::Command::from(whoami) .output() + .await .map_err(|error| { if error.kind() == io::ErrorKind::NotFound { ProvisionError::SshClientMissing(error) @@ -207,12 +455,13 @@ pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), Pro return Err(ProvisionError::EmptyUser); } - if user != "root" { - let (mut sudo, destination) = ssh_command(matches)?; - let status = sudo - .arg(destination) - .arg("sudo true") + let via_sudo = user != "root"; + if via_sudo { + let mut sudo = remote.ssh(); + sudo.arg("sudo true"); + let status = tokio::process::Command::from(sudo) .status() + .await .map_err(ProvisionError::Sudo)?; if !status.success() { return Err(ProvisionError::SudoRequired { @@ -221,50 +470,85 @@ pub fn provision(matches: &ArgMatches, storage: StorageChoice) -> Result<(), Pro } } - let encoded = encoded_installer(); + let architecture = remote.platform().await?; + let bootstrap = Bootstrap::acquire(&architecture).await?; let version = matches .get_one::("version") .expect("version has a default"); - let via_sudo = user != "root"; - let remote = format!( - "bash -c {}", - shell_quote(&pipefail(&install_command( - &encoded, - version, - via_sudo.then_some(user), - via_sudo, + let remote_directory = format!("/tmp/ployz-bootstrap-{}", Uuid::new_v4()); + let remote_daemon = format!("{remote_directory}/ployzd"); + let release_dir = bootstrap + .release_dir(version) + .map(|_| Path::new(&remote_directory)); + let arguments = install_arguments( + version, + Preparation::Host { storage, - ))) + group_user: via_sudo.then_some(user), + }, + release_dir, ); - let (mut install, destination) = ssh_command(matches)?; - installer_status(install.arg(destination).arg(remote).status()) + let primary = async { + remote.stage(&bootstrap, version, &remote_directory).await?; + remote.verify(&remote_daemon).await?; + remote.install(&remote_daemon, &arguments, via_sudo).await + } + .await; + let cleanup = remote.cleanup(&remote_directory).await; + // Host preparation may add the SSH user to the ployz group. A multiplexed + // session authenticated before installation retains its old group list. + remote.close_control_master().await; + finish_remote(primary, cleanup) +} + +/// Install and start local `ployzd` through a verified temporary daemon. +/// +/// # Errors +/// +/// Returns [`ProvisionError::NotRoot`] without local root privileges. It also +/// reports unsupported platforms, release acquisition and filesystem failures, +/// checksum or executable-version verification failures, and installer spawn or +/// non-zero exit failures. +pub async fn provision_local(version: &str, storage: StorageChoice) -> Result<(), ProvisionError> { + let group_user = env::var("SUDO_USER").ok().filter(|user| !user.is_empty()); + provision_local_with( + version, + Preparation::Host { + storage, + group_user: group_user.as_deref(), + }, + ) + .await } -/// Install and start local `ployzd` with the embedded Machine installer. +/// Synchronize the local daemon binary without preparing host software or storage. /// /// # Errors /// -/// Returns [`ProvisionError::NotRoot`] when this process is not root. -/// Returns [`ProvisionError::Install`] when the installer cannot be spawned, -/// or [`ProvisionError::InstallFailed`] when it exits non-zero. -pub fn provision_local(storage: StorageChoice) -> Result<(), ProvisionError> { +/// Returns the first privilege, platform, bootstrap acquisition, checksum, +/// executable-version, process, or installation failure. +pub async fn synchronize_local_daemon() -> Result<(), ProvisionError> { + provision_local_with(env!("CARGO_PKG_VERSION"), Preparation::SoftwareOnly).await +} + +async fn provision_local_with( + version: &str, + preparation: Preparation<'_>, +) -> Result<(), ProvisionError> { if !process_is_root() { return Err(ProvisionError::NotRoot); } - let group_user = std::env::var("SUDO_USER") - .ok() - .filter(|user| !user.is_empty()); + if env::consts::OS != "linux" { + return Err(ProvisionError::UnsupportedOs); + } + let bootstrap = Bootstrap::acquire(env::consts::ARCH).await?; + bootstrap.verify_local_version().await?; + let arguments = install_arguments(version, preparation, bootstrap.release_dir(version)); installer_status( - Command::new("bash") - .arg("-c") - .arg(pipefail(&install_command( - &encoded_installer(), - env!("CARGO_PKG_VERSION"), - group_user.as_deref(), - false, - storage, - ))) - .status(), + tokio::process::Command::new(bootstrap.daemon()) + .args(arguments) + .status() + .await, ) } @@ -280,10 +564,9 @@ pub(crate) fn process_is_root() -> bool { #[cfg(test)] mod tests { use super::*; - use ployz_core::StorageChoice; #[test] - fn provisioning_ssh_allows_interactive_authentication_with_a_network_timeout() { + fn provisioning_ssh_allows_interactive_authentication_with_shared_options() { for (extra, seconds) in [(vec![], "5"), (vec!["--ssh-timeout", "17"], "17")] { let mut args = vec!["ployz", "machine", "add", "root@host"]; args.extend(extra); @@ -293,34 +576,67 @@ mod tests { .unwrap() .subcommand_matches("add") .unwrap(); - let (command, _) = ssh_command(matches).unwrap(); - let args: Vec<_> = command.get_args().collect(); - assert!(args.contains(&std::ffi::OsStr::new("BatchMode=no"))); - assert!(args.contains(&std::ffi::OsStr::new("-tt"))); - for arg in crate::connect::ssh_control_args(crate::connect::control_path().as_deref()) { - assert!(args.contains(&std::ffi::OsStr::new(&arg))); + let remote = Remote::from_matches(matches).unwrap(); + let ssh = remote.ssh(); + let scp = remote.scp(); + for command in [&ssh, &scp] { + let args: Vec<_> = command.get_args().collect(); + assert!(args.contains(&std::ffi::OsStr::new("BatchMode=no"))); + for arg in + crate::connect::ssh_control_args(crate::connect::control_path().as_deref()) + { + assert!(args.contains(&std::ffi::OsStr::new(&arg))); + } + assert!( + command + .get_args() + .any(|arg| arg == format!("ConnectTimeout={seconds}").as_str()) + ); } - assert!( - command - .get_args() - .any(|arg| arg == format!("ConnectTimeout={seconds}").as_str()) - ); + assert!(ssh.get_args().any(|arg| arg == "-tt")); } } #[test] - fn embedded_installer_command_preserves_root_sudo_and_local_group() { + fn shared_installer_arguments_keep_bootstrap_and_target_distinct() { assert_eq!( - install_command("SCRIPT", "latest", None, false, StorageChoice::None), - "printf '%s' 'SCRIPT' | base64 -d | PLOYZ_STORAGE='none' PLOYZ_VERSION='latest' bash" + install_arguments( + "1.2.3", + Preparation::Host { + storage: StorageChoice::Zfs, + group_user: Some("deploy"), + }, + None, + ), + [ + "install", + "--version", + "1.2.3", + "--storage", + "zfs", + "--group-user", + "deploy", + ] + .map(OsString::from) + .to_vec() ); assert_eq!( - install_command("SCRIPT", "1.2.3", Some("deploy"), true, StorageChoice::Zfs,), - "printf '%s' 'SCRIPT' | base64 -d | sudo PLOYZ_GROUP_ADD_USER='deploy' PLOYZ_STORAGE='zfs' PLOYZ_VERSION='1.2.3' bash" + install_arguments("1.2.3", Preparation::SoftwareOnly, None), + ["install", "--version", "1.2.3", "--software-only"] + .map(OsString::from) + .to_vec() ); + } + + #[test] + fn cleanup_failures_preserve_primary_evidence() { + let primary = ProvisionError::BootstrapVerification("bad version".into()); + let cleanup = ProvisionError::Cleanup(io::Error::other("ssh failed")); + let error = finish_remote(Err(primary), Err(cleanup)).unwrap_err(); + assert_eq!( - install_command("SCRIPT", "1.2.3", Some("nick"), false, StorageChoice::None,), - "printf '%s' 'SCRIPT' | base64 -d | PLOYZ_GROUP_ADD_USER='nick' PLOYZ_STORAGE='none' PLOYZ_VERSION='1.2.3' bash" + error.to_string(), + "bootstrap verification: bad version; cleanup: remove remote bootstrap: ssh failed" ); } @@ -377,13 +693,13 @@ mod tests { resolve_storage(matches).unwrap() } - #[test] - fn local_provision_requires_root() { + #[tokio::test] + async fn local_provision_requires_root() { if process_is_root() { return; } assert!(matches!( - provision_local(StorageChoice::None), + provision_local(env!("CARGO_PKG_VERSION"), StorageChoice::None).await, Err(ProvisionError::NotRoot) )); } diff --git a/crates/ployz/src/provisioning/bootstrap.rs b/crates/ployz/src/provisioning/bootstrap.rs new file mode 100644 index 0000000000..ae7109087b --- /dev/null +++ b/crates/ployz/src/provisioning/bootstrap.rs @@ -0,0 +1,290 @@ +use std::{ + env, + fs::{self, File}, + io::{self, Read, Seek, SeekFrom}, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Command, Output, Stdio}, + time::Duration, +}; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; +use tokio::time::timeout; + +use super::ProvisionError; + +const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; +const VERSION_TIMEOUT: Duration = Duration::from_secs(10); + +/// A verified copy of this CLI release's daemon, owned by a private directory. +pub(super) struct Bootstrap { + directory: TempDir, + daemon: PathBuf, + archive: PathBuf, + checksums: PathBuf, + local_source: bool, +} + +impl Bootstrap { + /// Acquire, checksum, and extract the bootstrap for a Linux architecture. + pub(super) async fn acquire(architecture: &str) -> Result { + let archive_name = daemon_archive(architecture)?; + let directory = tempfile::Builder::new() + .prefix("ployz-bootstrap-") + .tempdir() + .map_err(|source| ProvisionError::BootstrapIo { + stage: "create bootstrap directory", + source, + })?; + let archive = directory.path().join(archive_name); + let checksums = directory.path().join("checksums.txt"); + let (archive_bytes, checksum_bytes, local_source) = + if let Some(source) = env::var_os("PLOYZ_RELEASE_DIR") { + let source = PathBuf::from(source); + let archive_bytes = + read_file(&source.join(archive_name), "copy bootstrap daemon archive")?; + let checksum_bytes = read_file( + &source.join("checksums.txt"), + "copy bootstrap release checksums", + )?; + (archive_bytes, checksum_bytes, true) + } else { + let base = format!( + "{RELEASE_REPOSITORY}/releases/download/v{}", + env!("CARGO_PKG_VERSION") + ); + let client = reqwest::Client::builder() + .https_only(true) + .connect_timeout(Duration::from_secs(10)) + .read_timeout(Duration::from_secs(30)) + .user_agent("ployz-bootstrap") + .build() + .map_err(|source| ProvisionError::BootstrapDownload { + stage: "build bootstrap download client", + source, + })?; + let archive_bytes = download( + &client, + &format!("{base}/{archive_name}"), + "download bootstrap daemon archive", + ) + .await?; + let checksum_bytes = download( + &client, + &format!("{base}/checksums.txt"), + "download bootstrap release checksums", + ) + .await?; + (archive_bytes, checksum_bytes, false) + }; + + verify_checksum(&archive_bytes, &checksum_bytes, archive_name)?; + write_file(&archive, &archive_bytes, "stage bootstrap daemon archive")?; + write_file( + &checksums, + &checksum_bytes, + "stage bootstrap release checksums", + )?; + let daemon = directory.path().join("ployzd"); + let mut extract = Command::new("tar"); + extract + .arg("-xzf") + .arg(&archive) + .arg("-C") + .arg(directory.path()) + .arg("ployzd"); + command_status("extract bootstrap daemon", extract).await?; + fs::set_permissions(&daemon, fs::Permissions::from_mode(0o700)).map_err(|source| { + ProvisionError::BootstrapIo { + stage: "mark bootstrap daemon executable", + source, + } + })?; + + Ok(Self { + directory, + daemon, + archive, + checksums, + local_source, + }) + } + + pub(super) fn daemon(&self) -> &Path { + &self.daemon + } + + /// Return the local release directory only when it describes the selected target. + pub(super) fn release_dir(&self, version: &str) -> Option<&Path> { + (self.local_source && version == env!("CARGO_PKG_VERSION")).then(|| self.directory.path()) + } + + pub(super) fn release_files(&self, version: &str) -> Option<[&Path; 2]> { + self.release_dir(version) + .map(|_| [self.archive.as_path(), self.checksums.as_path()]) + } + + pub(super) async fn verify_local_version(&self) -> Result<(), ProvisionError> { + let mut command = Command::new(&self.daemon); + command.arg("version"); + verify_command_version("preflight bootstrap daemon", command).await + } +} + +pub(super) async fn verify_command_version( + stage: &'static str, + command: Command, +) -> Result<(), ProvisionError> { + let output = command_output(command, VERSION_TIMEOUT) + .await + .map_err(|source| ProvisionError::BootstrapIo { stage, source })?; + if !output.status.success() { + return Err(ProvisionError::BootstrapCommand { + stage, + status: output.status, + }); + } + let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if observed == env!("CARGO_PKG_VERSION") { + Ok(()) + } else { + Err(ProvisionError::BootstrapVerification(format!( + "bootstrap daemon reported version {observed:?}, expected {}", + env!("CARGO_PKG_VERSION") + ))) + } +} + +async fn download( + client: &reqwest::Client, + url: &str, + stage: &'static str, +) -> Result, ProvisionError> { + client + .get(url) + .send() + .await + .map_err(|source| ProvisionError::BootstrapDownload { stage, source })? + .error_for_status() + .map_err(|source| ProvisionError::BootstrapDownload { stage, source })? + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|source| ProvisionError::BootstrapDownload { stage, source }) +} + +fn read_file(path: &Path, stage: &'static str) -> Result, ProvisionError> { + fs::read(path).map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + +fn verify_checksum( + archive: &[u8], + checksums: &[u8], + archive_name: &str, +) -> Result<(), ProvisionError> { + let checksums = std::str::from_utf8(checksums).map_err(|error| { + ProvisionError::BootstrapVerification(format!("checksums.txt is not UTF-8: {error}")) + })?; + let expected = checksums + .lines() + .find_map(|line| { + let mut fields = line.split_whitespace(); + let hash = fields.next()?; + let name = fields.next()?.trim_start_matches('*'); + (name == archive_name + && hash.len() == 64 + && hash.bytes().all(|byte| byte.is_ascii_hexdigit())) + .then(|| hash.to_ascii_lowercase()) + }) + .ok_or_else(|| { + ProvisionError::BootstrapVerification(format!( + "checksums.txt has no SHA-256 hash for {archive_name}" + )) + })?; + let actual = hex::encode(Sha256::digest(archive)); + if actual == expected { + Ok(()) + } else { + Err(ProvisionError::BootstrapVerification(format!( + "{archive_name} checksum was {actual}, expected {expected}" + ))) + } +} + +fn write_file(path: &Path, bytes: &[u8], stage: &'static str) -> Result<(), ProvisionError> { + fs::write(path, bytes).map_err(|source| ProvisionError::BootstrapIo { stage, source }) +} + +async fn command_status(stage: &'static str, command: Command) -> Result<(), ProvisionError> { + let status = tokio::process::Command::from(command) + .status() + .await + .map_err(|source| ProvisionError::BootstrapIo { stage, source })?; + if status.success() { + Ok(()) + } else { + Err(ProvisionError::BootstrapCommand { stage, status }) + } +} + +/// Capture bounded command output and explicitly kill and reap a timed-out child. +async fn command_output(mut command: Command, budget: Duration) -> io::Result { + let mut stdout = tempfile::tempfile()?; + let mut stderr = tempfile::tempfile()?; + command + .stdout(Stdio::from(stdout.try_clone()?)) + .stderr(Stdio::from(stderr.try_clone()?)); + let mut child = tokio::process::Command::from(command).spawn()?; + let status = match timeout(budget, child.wait()).await { + Ok(status) => status?, + Err(_) => { + if child.try_wait()?.is_none() { + child.kill().await?; + } + let _ = child.wait().await; + return Err(io::Error::new( + io::ErrorKind::TimedOut, + format!("timed out after {} seconds", budget.as_secs()), + )); + } + }; + Ok(Output { + status, + stdout: read_from_start(&mut stdout)?, + stderr: read_from_start(&mut stderr)?, + }) +} + +fn read_from_start(file: &mut File) -> io::Result> { + file.seek(SeekFrom::Start(0))?; + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes)?; + Ok(bytes) +} + +fn daemon_archive(architecture: &str) -> Result<&'static str, ProvisionError> { + match architecture { + "x86_64" => Ok("ployzd_linux_amd64.tar.gz"), + "aarch64" => Ok("ployzd_linux_arm64.tar.gz"), + architecture => Err(ProvisionError::UnsupportedArchitecture( + architecture.to_owned(), + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn bounded_output_kills_and_reaps_a_hung_child() { + let mut command = Command::new("sh"); + command.args(["-c", "exec sleep 30"]); + let error = command_output(command, Duration::from_millis(20)) + .await + .unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + } +} diff --git a/crates/ployz/tests/cli_shape.rs b/crates/ployz/tests/cli_shape.rs index 4d51d919ff..345023547c 100644 --- a/crates/ployz/tests/cli_shape.rs +++ b/crates/ployz/tests/cli_shape.rs @@ -61,6 +61,93 @@ fn native_completion_is_generated_for_every_supported_shell() { } } +#[test] +fn machine_upgrade_requires_explicit_targets_and_has_typed_inspection() { + let command = ployz::cli::command(); + let request = command + .clone() + .try_get_matches_from([ + "ployz", + "machine", + "upgrade", + "1.2.3-beta.4", + "--machine", + "edge-a", + "--machine", + "0123456789abcdef0123456789abcdef", + ]) + .unwrap(); + let upgrade = request + .subcommand_matches("machine") + .unwrap() + .subcommand_matches("upgrade") + .unwrap(); + assert_eq!( + upgrade + .get_one::("version") + .map(ployz_core::MachineRelease::as_str), + Some("1.2.3-beta.4") + ); + assert_eq!( + upgrade + .get_many::("machine") + .unwrap() + .map(String::as_str) + .collect::>(), + ["edge-a", "0123456789abcdef0123456789abcdef"] + ); + assert!( + command + .clone() + .try_get_matches_from(["ployz", "machine", "upgrade", "stable"]) + .is_err() + ); + assert!( + command + .clone() + .try_get_matches_from([ + "ployz", + "machine", + "upgrade", + "nightly", + "--machine", + "edge-a" + ]) + .is_err() + ); + + let inspect = command + .try_get_matches_from([ + "ployz", + "machine", + "upgrade", + "inspect", + "edge-a", + "--attempt", + "0123456789abcdef0123456789abcdef", + "-o", + "json", + ]) + .unwrap(); + let inspect = inspect + .subcommand_matches("machine") + .unwrap() + .subcommand_matches("upgrade") + .unwrap() + .subcommand_matches("inspect") + .unwrap(); + assert_eq!( + inspect + .get_one::("attempt") + .map(ToString::to_string), + Some("0123456789abcdef0123456789abcdef".into()) + ); + assert_eq!( + inspect.get_one::("output").map(String::as_str), + Some("json") + ); +} + #[cfg(unix)] #[test] fn completion_exits_on_sigpipe_when_the_reader_closes_after_one_line() { diff --git a/crates/ployz/tests/cloud_init/daemon_sync.rs b/crates/ployz/tests/cloud_init/daemon_sync.rs index 2c9d6f3ae0..0798ff7d7b 100644 --- a/crates/ployz/tests/cloud_init/daemon_sync.rs +++ b/crates/ployz/tests/cloud_init/daemon_sync.rs @@ -26,17 +26,17 @@ fn recording_installer( daemon: JoinDaemon, outcome: InstallOutcome, calls: Arc, -) -> impl Fn() -> Result<(), ployz::handlers::Error> { - move || { +) -> impl Fn(ployz_core::StorageChoice) -> std::future::Ready> { + move |_| { calls.fetch_add(1, Ordering::SeqCst); - match outcome { + std::future::ready(match outcome { InstallOutcome::UpdateDaemon => { daemon.set_daemon_version(env!("CARGO_PKG_VERSION")); Ok(()) } InstallOutcome::LeaveStale => Ok(()), InstallOutcome::Fail(message) => Err(ployz::handlers::Error::usage(message)), - } + }) } } @@ -62,7 +62,11 @@ fn enroll_matches(connect: &str, cloud_url: &str) -> ArgMatches { async fn run_enroll( matches: ArgMatches, - install: impl Fn() -> Result<(), ployz::handlers::Error> + Send + 'static, + install: impl Fn( + ployz_core::StorageChoice, + ) -> std::future::Ready> + + Send + + 'static, ) -> Result<(), ployz::handlers::Error> { tokio::task::spawn_blocking(move || { ployz::handlers::cloud_enroll_with_installer(&matches, &install) @@ -78,6 +82,18 @@ async fn enroll_locally( Result<(), ployz::handlers::Error>, Arc, Arc, +) { + enroll_locally_with_storage(daemon_version, outcome, "none").await +} + +async fn enroll_locally_with_storage( + daemon_version: &str, + outcome: InstallOutcome, + storage: &str, +) -> ( + Result<(), ployz::handlers::Error>, + Arc, + Arc, ) { let registration = registration(); let relay = RelayListen::start().await; @@ -85,7 +101,7 @@ async fn enroll_locally( CloudPairing::parse(&relay.url, PairingCredential::parse(PAIRING).unwrap()).unwrap(); let enroll = EnrollListen::start(json!({ "kind": "join", - "storage": "none", + "storage": storage, "pairing": pairing, "registration": registration, })) @@ -100,6 +116,23 @@ async fn enroll_locally( (result, calls, connections) } +#[tokio::test] +async fn zfs_preparation_reconnects_after_restarting_a_matching_daemon() { + let (result, calls, connections) = enroll_locally_with_storage( + env!("CARGO_PKG_VERSION"), + InstallOutcome::UpdateDaemon, + "zfs", + ) + .await; + + assert!(result.is_ok(), "{result:?}"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert!( + connections.load(Ordering::SeqCst) >= 2, + "enrollment must reconnect after storage preparation restarts the daemon" + ); +} + #[tokio::test] async fn matching_daemon_does_not_run_the_installer() { let (result, calls, _) = diff --git a/crates/ployz/tests/cloud_init/harness.rs b/crates/ployz/tests/cloud_init/harness.rs index 0cc7ac667a..f27c7c82be 100644 --- a/crates/ployz/tests/cloud_init/harness.rs +++ b/crates/ployz/tests/cloud_init/harness.rs @@ -503,6 +503,20 @@ impl MachineRpc for JoinDaemon { rpc_ok(CloudPairingSet {}) } + async fn request_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } + + async fn inspect_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } + async fn initialize( &self, request: Request, diff --git a/crates/ployz/tests/connect/support.rs b/crates/ployz/tests/connect/support.rs index 32f4152d0a..4f03fe89c7 100644 --- a/crates/ployz/tests/connect/support.rs +++ b/crates/ployz/tests/connect/support.rs @@ -606,6 +606,20 @@ impl MachineRpc for DiscoveryService { Ok(Response::new(RpcResponse::from(report).encode().unwrap())) } + async fn request_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + + async fn inspect_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn inspect_container( &self, _request: Request, diff --git a/crates/ployz/tests/deploy_client/support.rs b/crates/ployz/tests/deploy_client/support.rs index 0033ca5de4..eda18e4c4c 100644 --- a/crates/ployz/tests/deploy_client/support.rs +++ b/crates/ployz/tests/deploy_client/support.rs @@ -519,6 +519,18 @@ impl MachineRpc for DeployService { self.record_mutation(); unused() } + async fn request_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } + async fn inspect_machine_upgrade( + &self, + _request: Request, + ) -> Result, Status> { + unused() + } async fn inspect_container( &self, request: Request, diff --git a/crates/ployz/tests/setup_cli.rs b/crates/ployz/tests/setup_cli.rs new file mode 100644 index 0000000000..3e6c7276c3 --- /dev/null +++ b/crates/ployz/tests/setup_cli.rs @@ -0,0 +1,382 @@ +//! CLI setup adapters invoking the shared Machine installer. + +use std::{ + env, fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +use sha2::{Digest, Sha256}; + +fn write_executable(path: &Path, body: &str) { + fs::write(path, body).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +fn daemon_archive() -> &'static str { + match env::consts::ARCH { + "x86_64" => "ployzd_linux_amd64.tar.gz", + "aarch64" => "ployzd_linux_arm64.tar.gz", + architecture => panic!("unsupported test architecture {architecture}"), + } +} + +fn release_fixture(root: &Path, log: &Path) -> PathBuf { + let release = root.join("release"); + let payload = root.join("payload"); + fs::create_dir(&release).unwrap(); + fs::create_dir(&payload).unwrap(); + write_executable( + &payload.join("ployzd"), + &format!( + "#!/bin/sh\nif [ \"${{1:-}}\" = version ]; then printf '%s\\n' '{}'; else printf '%s\\n' \"$*\" >> '{}'; exit \"${{PLOYZ_FAKE_INSTALL_STATUS:-0}}\"; fi\n", + env!("CARGO_PKG_VERSION"), + log.display() + ), + ); + let archive = release.join(daemon_archive()); + assert!( + Command::new("tar") + .args(["-czf"]) + .arg(&archive) + .arg("-C") + .arg(&payload) + .arg("ployzd") + .status() + .unwrap() + .success() + ); + let hash = hex::encode(Sha256::digest(fs::read(&archive).unwrap())); + fs::write( + release.join("checksums.txt"), + format!("{hash} {}\n", daemon_archive()), + ) + .unwrap(); + release +} + +fn run(args: &[&str], root: &Path, release: &Path, bin: &Path) -> Output { + let inherited = env::var_os("PATH").unwrap_or_default(); + let path = + env::join_paths(std::iter::once(bin.to_path_buf()).chain(env::split_paths(&inherited))) + .unwrap(); + Command::new(env!("CARGO_BIN_EXE_ployz")) + .args(args) + .env("PATH", path) + .env("PLOYZ_RELEASE_DIR", release) + .env("PLOYZ_CONFIG", root.join("config.yaml")) + .env("PLOYZ_FAKE_INSTALL_STATUS", "17") + .env("LOG", root.join("commands.log")) + .env("PLOYZ_TEST_PID", root.join("preflight.pid")) + .env_remove("SUDO_USER") + .output() + .unwrap() +} + +#[test] +fn local_setup_runs_the_verified_bootstrap_installer() { + let fixture = tempfile::tempdir().unwrap(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &log); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("id"), + "#!/bin/sh\nif [ \"$1\" = -u ]; then echo 0; else exec /usr/bin/id \"$@\"; fi\n", + ); + + let output = run( + &[ + "machine", + "init", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-local", + ], + fixture.path(), + &release, + &bin, + ); + assert!(!output.status.success(), "fake daemon must not accept RPC"); + let log = fs::read_to_string(log).unwrap(); + assert!( + log.contains(&format!( + "install --version {} --storage none --release-dir", + env!("CARGO_PKG_VERSION") + )), + "{log}" + ); +} + +#[test] +fn corrupt_local_bootstrap_is_rejected_before_execution() { + let fixture = tempfile::tempdir().unwrap(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &log); + fs::write(release.join(daemon_archive()), b"corrupt").unwrap(); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("id"), + "#!/bin/sh\nif [ \"$1\" = -u ]; then echo 0; else exec /usr/bin/id \"$@\"; fi\n", + ); + + let output = run( + &[ + "machine", + "init", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-corrupt", + ], + fixture.path(), + &release, + &bin, + ); + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("bootstrap verification"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(!log.exists(), "corrupt bootstrap executed"); +} + +#[test] +fn ssh_setup_transfers_one_bootstrap_and_no_install_skips_it() { + let fixture = tempfile::tempdir().unwrap(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &log); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("ssh"), + &format!( + r#"#!/bin/sh +printf 'ssh %s\n' "$*" >> "$LOG" +case "$*" in + *' whoami') echo deploy ;; + *' sudo true') ;; + *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'mkdir -m 700 -- '*) ;; + *'chmod 700 '*' version') printf '%s\n' '{}' ;; + *" 'install' "*) ;; + *'rm -rf -- '*) ;; + *' -O exit '*) ;; + *' true') exit 23 ;; + *) echo "unexpected ssh invocation: $*" >&2; exit 24 ;; +esac +"#, + env::consts::ARCH, + env!("CARGO_PKG_VERSION") + ), + ); + write_executable( + &bin.join("scp"), + "#!/bin/sh\nprintf 'scp %s\\n' \"$*\" >> \"$LOG\"\n", + ); + + let common = [ + "machine", + "init", + "deploy@2001:db8::1", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-ssh", + ]; + let inherited = env::var_os("PATH").unwrap_or_default(); + let path = + env::join_paths(std::iter::once(bin.clone()).chain(env::split_paths(&inherited))).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_ployz")) + .args(common) + .env("PATH", &path) + .env("LOG", &log) + .env("PLOYZ_RELEASE_DIR", &release) + .env("PLOYZ_CONFIG", fixture.path().join("ssh-config.yaml")) + .output() + .unwrap(); + assert!(!output.status.success(), "fake daemon must not accept RPC"); + let setup_log = fs::read_to_string(&log).unwrap(); + assert!(setup_log.contains("scp "), "{setup_log}"); + assert!( + setup_log.contains("deploy@[2001:db8::1]:/tmp/ployz-bootstrap-"), + "{setup_log}" + ); + assert!(!setup_log.contains(" -P 1 "), "{setup_log}"); + assert!(setup_log.contains(daemon_archive()), "{setup_log}"); + assert!(setup_log.contains("checksums.txt"), "{setup_log}"); + assert!( + setup_log.contains(&format!( + "'install' '--version' '{}' '--storage' 'none' '--group-user' 'deploy'", + env!("CARGO_PKG_VERSION") + )), + "{setup_log}" + ); + assert!(setup_log.contains("'--release-dir' '/tmp/ployz-bootstrap-")); + assert!(setup_log.contains("rm -rf --"), "{setup_log}"); + let close = setup_log.find(" -O exit deploy@2001:db8::1").unwrap(); + let reconnect = setup_log.rfind(" true").unwrap(); + assert!(close < reconnect, "{setup_log}"); + assert!(!setup_log.contains("base64"), "{setup_log}"); + + fs::write(&log, "").unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_ployz")) + .args([ + "machine", + "init", + "deploy@2001:db8::1", + "--no-install", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-no-install", + ]) + .env("PATH", path) + .env("LOG", &log) + .env("PLOYZ_RELEASE_DIR", release) + .env( + "PLOYZ_CONFIG", + fixture.path().join("no-install-config.yaml"), + ) + .output() + .unwrap(); + assert!(!output.status.success(), "fake daemon must not accept RPC"); + let no_install_log = fs::read_to_string(log).unwrap(); + assert!(!no_install_log.contains("scp "), "{no_install_log}"); + assert!(!no_install_log.contains("'install'"), "{no_install_log}"); +} + +#[test] +fn remote_preflight_timeout_kills_the_child_and_cleans_the_stage() { + let fixture = tempfile::tempdir().unwrap(); + let log = fixture.path().join("commands.log"); + let release = release_fixture(fixture.path(), &fixture.path().join("installer.log")); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("ssh"), + &format!( + r#"#!/bin/sh +printf 'ssh %s\n' "$*" >> "$LOG" +case "$*" in + *' whoami') echo root ;; + *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'mkdir -m 700 -- '*) ;; + *'chmod 700 '*' version') echo $$ > "$PLOYZ_TEST_PID"; exec sleep 30 ;; + *'rm -rf -- '*) ;; + *) exit 24 ;; +esac +"#, + env::consts::ARCH + ), + ); + write_executable(&bin.join("scp"), "#!/bin/sh\nexit 0\n"); + + let output = run( + &[ + "machine", + "init", + "root@host", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-timeout", + ], + fixture.path(), + &release, + &bin, + ); + + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("timed out after 10 seconds"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let log = fs::read_to_string(log).unwrap(); + assert!(log.contains("rm -rf --"), "{log}"); + let pid = fs::read_to_string(fixture.path().join("preflight.pid")) + .unwrap() + .trim() + .to_owned(); + assert!( + !Command::new("kill") + .args(["-0", &pid]) + .status() + .unwrap() + .success(), + "timed-out SSH child {pid} survived" + ); +} + +#[test] +fn remote_cleanup_failure_is_returned_after_successful_installation() { + let fixture = tempfile::tempdir().unwrap(); + let release = release_fixture(fixture.path(), &fixture.path().join("installer.log")); + let bin = fixture.path().join("bin"); + fs::create_dir(&bin).unwrap(); + write_executable( + &bin.join("ssh"), + &format!( + r#"#!/bin/sh +case "$*" in + *' whoami') echo root ;; + *'uname -s; uname -m') printf 'Linux\n{}\n' ;; + *'mkdir -m 700 -- '*) ;; + *'chmod 700 '*' version') printf '%s\n' '{}' ;; + *" 'install' "*) ;; + *'rm -rf -- '*) exit 29 ;; + *) exit 24 ;; +esac +"#, + env::consts::ARCH, + env!("CARGO_PKG_VERSION") + ), + ); + write_executable(&bin.join("scp"), "#!/bin/sh\nexit 0\n"); + + let output = run( + &[ + "machine", + "init", + "root@host", + "--yes", + "--storage", + "none", + "--no-dns", + "--no-ingress", + "--context", + "setup-cleanup", + ], + fixture.path(), + &release, + &bin, + ); + + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr) + .contains("remote bootstrap cleanup exited with exit status: 29"), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/ployzd/Cargo.toml b/crates/ployzd/Cargo.toml index 6a2ed0b238..52bf466d5d 100644 --- a/crates/ployzd/Cargo.toml +++ b/crates/ployzd/Cargo.toml @@ -38,6 +38,7 @@ serde_json = "1.0.151" sha2 = "0.11.0" shell-words = "1.1.1" thiserror = "2.0.20" +tempfile = "3.27.0" tokio = { version = "1.53.1", features = ["io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = { version = "0.1.19", features = ["net"] } tokio-util = { version = "0.7.19", features = ["codec", "io"] } diff --git a/crates/ployzd/src/build.rs b/crates/ployzd/src/build.rs index dba2ce00dc..79d6a54fef 100644 --- a/crates/ployzd/src/build.rs +++ b/crates/ployzd/src/build.rs @@ -21,19 +21,31 @@ use tonic::Status; #[derive(Default)] struct AttemptState { evidence: Mutex, - retained: Mutex>, + retained: Mutex>, +} + +struct RetainedBuild { + // Field order keeps installation exclusion until Docker tag cleanup finishes. + _retention: ployz_build::ImageRetention, + _installation: crate::mutation::MutationGuard, +} + +struct ExecutionAdmission { + build: Admission, + machine: crate::machine::MutationAdmission, } pub(crate) fn start( machine_id: MachineId, requests: impl Stream> + Send + Unpin + 'static, runner: std::sync::Arc, + local: crate::machine::LocalMachine, ) -> RpcStream { let (events, receiver) = mpsc::channel(8); tokio::spawn(async move { let shutdown = runner.shutdown.clone(); let state = Arc::new(AttemptState::default()); - let outcome = attempt(machine_id, requests, &events, runner, state.clone()).await; + let outcome = attempt(machine_id, requests, &events, runner, local, state.clone()).await; let message = "Build terminal report exceeds the response size limit"; let fallback = match &outcome { Outcome::Unknown { stage, .. } => Outcome::Unknown { @@ -50,7 +62,8 @@ pub(crate) fn start( remote::encode(&Event::Finished(fallback)).expect("bounded terminal fallback") }); let _ = tokio::time::timeout(Duration::from_secs(5), events.send(Ok(payload))).await; - // Admission is already released. Only image retention follows this stream. + // Local mutation serialization is already released. Installation exclusion follows + // retained image cleanup because releasing temporary tags mutates Docker. if state .retained .lock() @@ -70,6 +83,7 @@ async fn attempt( mut requests: impl Stream> + Send + Unpin + 'static, events: &mpsc::Sender>, runner: Arc, + local: crate::machine::LocalMachine, state: Arc, ) -> Outcome { let start = tokio::time::timeout(Duration::from_secs(10), requests.next()).await; @@ -119,6 +133,28 @@ async fn attempt( Err(reason) => return failed(Stage::Queued, reason.to_string()).with_work(work), }; let policy = runner.policy.clone(); + let mutation = { + let admission = local.admit_mutation(); + tokio::pin!(admission); + tokio::select! { + biased; + () = runner.shutdown.cancelled() => return failed(Stage::Admission, "Build daemon stopped before admission").with_work(work), + () = events.closed() => return failed(Stage::Admission, "Build client disconnected before admission").with_work(work), + frame = requests.next() => { + let reason = match frame { + Some(Ok(payload)) if matches!(remote::decode::(&payload), Ok(Input::Cancel)) => "Build cancelled before admission", + None | Some(Err(_)) => "Build client disconnected before admission", + Some(Ok(_)) => "Build upload before admission is forbidden", + }; + return failed(Stage::Admission, reason).with_work(work); + } + () = tokio::time::sleep(policy.active_timeout) => return failed(Stage::Admission, "Build admission timed out waiting for another Machine mutation").with_work(work), + result = &mut admission => match result { + Ok(admission) => admission, + Err(error) => return failed(Stage::Admission, error.to_string()).with_work(work), + }, + } + }; let admission = match tokio::task::spawn_blocking({ let policy = policy.clone(); move || Admission::try_acquire_with(&policy) @@ -148,7 +184,16 @@ async fn attempt( let mut execution = tokio::task::spawn_blocking(move || { let _permit = permit; receive_and_execute( - machine_id, definition, source, admission, policy, &output, &observed, + machine_id, + definition, + source, + ExecutionAdmission { + build: admission, + machine: mutation, + }, + policy, + &output, + &observed, ) }); // Dropping the input pump closes the upload channel. A receiver blocked @@ -220,11 +265,15 @@ fn receive_and_execute( machine_id: MachineId, mut definition: Definition, mut source: mpsc::Receiver, - admission: Admission, + admission: ExecutionAdmission, policy: HostPolicy, events: &mpsc::Sender>, state: &AttemptState, ) -> Outcome { + let ExecutionAdmission { + build: admission, + machine: mutation, + } = admission; let cancellation = admission.cancellation(); let deadline = std::time::Instant::now() + admission.remaining(); let mut upload = match admission.upload() { @@ -294,7 +343,10 @@ fn receive_and_execute( match result { Ok(completed) => { if !definition.retained_tags.is_empty() { - *state.retained.lock().expect("Build retention lock") = Some(completed.retention); + *state.retained.lock().expect("Build retention lock") = Some(RetainedBuild { + _retention: completed.retention, + _installation: mutation.into_installation_guard(), + }); } match definition.output { Output::Load => Outcome::Images { diff --git a/crates/ployzd/src/build/tests.rs b/crates/ployzd/src/build/tests.rs index c2780363c3..991ec7f568 100644 --- a/crates/ployzd/src/build/tests.rs +++ b/crates/ployzd/src/build/tests.rs @@ -21,6 +21,7 @@ struct Fixture { root: PathBuf, policy: HostPolicy, cluster: tokio::task::JoinHandle<()>, + local: crate::machine::LocalMachine, machine: ployz_core::Machine, address: String, server: tokio::task::JoinHandle>, @@ -74,6 +75,7 @@ impl Fixture { update(&mut policy); let shutdown = tokio_util::sync::CancellationToken::new(); service.builds = Runner::new(policy.clone(), shutdown.clone()).unwrap(); + let local = service.local(); write_docker(&policy.docker, &root); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = format!("http://{}", listener.local_addr().unwrap()); @@ -86,6 +88,7 @@ impl Fixture { root, policy, cluster, + local, machine, address, server, @@ -199,6 +202,63 @@ async fn admitted_upload_queues_competitors_and_disconnect_releases_unused_owner assert!(matches!(event(&mut admitted).await, Event::Admitted { .. })); } +#[tokio::test] +async fn active_build_refuses_an_upgrade_request() { + let fixture = Fixture::new().await; + let (sender, mut response) = fixture.request(Output::Load).await; + assert!(matches!(event(&mut response).await, Event::Admitted { .. })); + + let error = fixture + .local + .request_upgrade(ployz_core::RequestMachineUpgradeRequest { + attempt_id: ployz_core::MachineUpgradeAttemptId::random(), + release: ployz_core::MachineRelease::parse("1.2.3").unwrap(), + }) + .await + .unwrap_err(); + assert!(matches!( + error, + crate::machine::LocalMachineError::Admission(crate::mutation::Error::Busy) + )); + + let mut ordinary = Box::pin(fixture.local.set_cloud_pairing(None)); + assert!( + tokio::time::timeout(Duration::from_millis(100), ordinary.as_mut()) + .await + .is_err(), + "ordinary mutation ran concurrently with active Build execution" + ); + drop(sender); + let _ = terminal(&mut response).await; + tokio::time::timeout(Duration::from_secs(2), ordinary) + .await + .expect("ordinary mutation stayed blocked after Build execution ended") + .unwrap(); +} + +#[tokio::test] +async fn durable_upgrade_marker_refuses_build_before_execution() { + let fixture = Fixture::new().await; + let marker = fixture.root.join("machine/.upgrade-active"); + fs::write(&marker, "active-upgrade").unwrap(); + + let (_sender, mut rejected) = fixture.request(Output::Load).await; + assert!(matches!( + terminal(&mut rejected).await, + Outcome::Failed { + stage: Stage::Admission, + .. + } + )); + assert!(!fixture.root.join("executed").exists()); + + fs::remove_file(marker).unwrap(); + let (sender, mut admitted) = fixture.request(Output::Load).await; + assert!(matches!(event(&mut admitted).await, Event::Admitted { .. })); + drop(sender); + let _ = terminal(&mut admitted).await; +} + #[tokio::test] async fn captured_build_crosses_owned_rpc_and_returns_only_remote_image_evidence() { let fixture = Fixture::new().await; @@ -248,6 +308,50 @@ async fn captured_build_crosses_owned_rpc_and_returns_only_remote_image_evidence assert!(Admission::try_acquire_with(&fixture.policy).is_ok()); } +#[tokio::test] +async fn retained_build_images_do_not_block_same_machine_mutation() { + let fixture = Fixture::new().await; + let client = ployz::connect::connect( + Path::new("/missing-test-config"), + Some(&fixture.address.replace("http://", "tcp://")), + None, + ) + .await + .unwrap(); + let retained = fixture + .capture() + .execute_remote_images( + &client, + fixture.machine.id, + tokio_util::sync::CancellationToken::new(), + |_| {}, + ) + .await + .unwrap(); + + tokio::time::timeout( + Duration::from_secs(2), + fixture.local.set_cloud_pairing(None), + ) + .await + .expect("retained Build stream held the local mutation mutex") + .unwrap(); + + let error = fixture + .local + .request_upgrade(ployz_core::RequestMachineUpgradeRequest { + attempt_id: ployz_core::MachineUpgradeAttemptId::random(), + release: ployz_core::MachineRelease::parse("1.2.3").unwrap(), + }) + .await + .unwrap_err(); + assert!(matches!( + error, + crate::machine::LocalMachineError::Admission(crate::mutation::Error::Busy) + )); + drop(retained); +} + #[tokio::test] async fn oversized_diagnostics_still_return_a_definite_terminal_failure() { let fixture = Fixture::new().await; @@ -401,6 +505,7 @@ async fn upload_timeout_stops_before_execution_and_releases_admission() { fixture.machine.id, ReceiverStream::new(receiver), Runner::new(policy, Default::default()).unwrap(), + fixture.local.clone(), ); assert!(matches!( remote::decode::(&responses.next().await.unwrap().unwrap()).unwrap(), diff --git a/crates/ployzd/src/daemon.rs b/crates/ployzd/src/daemon.rs index 4c985614e5..eaf9eda122 100644 --- a/crates/ployzd/src/daemon.rs +++ b/crates/ployzd/src/daemon.rs @@ -99,16 +99,28 @@ impl Daemon { /// /// If construction, binding, or required planes fail. pub async fn start(config: DaemonConfig) -> Result { + let run_dir = config + .socket + .parent() + .unwrap_or_else(|| Path::new("/run/ployz")) + .to_owned(); + crate::installer::upgrade::reconcile(&config.data_dir, &run_dir) + .await + .map_err(io::Error::other)?; let build_policy = ployz_build::HostPolicy::from_environment() .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; - Self::start_with_build_policy(config, build_policy).await + Self::start_with_build_policy(config, build_policy, run_dir).await } async fn start_with_build_policy( config: DaemonConfig, build_policy: ployz_build::HostPolicy, + run_dir: PathBuf, ) -> Result { - let store = Arc::new(Mutex::new(LocalMachineStore::open(&config.data_dir)?)); + let store = Arc::new(Mutex::new(LocalMachineStore::open_with_admission( + &config.data_dir, + run_dir, + )?)); let socket_lock = claim_socket(&config.socket)?; let cleanup = tokio::task::spawn_blocking({ let policy = build_policy.clone(); @@ -806,7 +818,8 @@ mod tests { ) .unwrap(); fs::set_permissions(&policy.docker, fs::Permissions::from_mode(0o700)).unwrap(); - let daemon = Daemon::start_with_build_policy(config, policy.clone()) + let run_dir = config.socket.parent().unwrap().to_owned(); + let daemon = Daemon::start_with_build_policy(config, policy.clone(), run_dir) .await .unwrap(); assert!( diff --git a/crates/ployzd/src/docker/test_support.rs b/crates/ployzd/src/docker/test_support.rs index b0f1e4487d..5b17fbe057 100644 --- a/crates/ployzd/src/docker/test_support.rs +++ b/crates/ployzd/src/docker/test_support.rs @@ -228,14 +228,16 @@ async fn fake_docker( serde_json::json!({"bound_bytes":bound,"used_bytes":0}), ); } - fake.volumes - .lock() - .unwrap() - .insert(name.clone(), observed.clone()); + let mut response = observed.clone(); + if driver == ployz_core::PROVISIONED_VOLUME_DRIVER { + // Docker's plugin Create response lacks Status; the following inspect supplies it. + response.as_object_mut().unwrap().remove("Status"); + } + fake.volumes.lock().unwrap().insert(name.clone(), observed); if fake.fail_after_create.lock().unwrap().contains(&name) { fake.fail_inspect_once.lock().unwrap().insert(name.clone()); } - (StatusCode::CREATED, observed) + (StatusCode::CREATED, response) } } else if method == Method::POST && (path.ends_with("/start") || path.ends_with("/stop")) { (StatusCode::NO_CONTENT, serde_json::Value::Null) diff --git a/crates/ployzd/src/docker/volume/tests.rs b/crates/ployzd/src/docker/volume/tests.rs index 285c06399a..f9751fca94 100644 --- a/crates/ployzd/src/docker/volume/tests.rs +++ b/crates/ployzd/src/docker/volume/tests.rs @@ -361,6 +361,14 @@ async fn existing_managed_volume_refuses_every_unsafe_shape_mismatch() { "Status":{"bound_bytes":1073741824,"used_bytes":0} }), ), + ( + provisioned_source("missing-metadata", 2_147_483_648), + serde_json::json!({ + "Name":"app_missing-metadata","Driver":"ployz","Mountpoint":"/var/lib/ployz-volumes/app_missing-metadata", + "Options":null,"Labels":null, + "Status":{"bound_bytes":2147483648_u64,"used_bytes":0} + }), + ), ]; for (source, observed) in cases { let name = source.docker_volume_name().unwrap().to_string(); diff --git a/crates/ployzd/src/installer/host.rs b/crates/ployzd/src/installer/host.rs new file mode 100644 index 0000000000..4f6e04681d --- /dev/null +++ b/crates/ployzd/src/installer/host.rs @@ -0,0 +1,374 @@ +//! Explicit Machine host preparation and systemd lifecycle management. + +use std::{ + fs, + io::Write, + os::unix::fs::MetadataExt, + path::Path, + process::{Command, Stdio}, +}; + +use tonic::transport::Endpoint; + +use crate::filesystem::atomic_write; +use ployz_core::{DescribeContractRequest, MachineRpcClient, MachineVersion, op}; + +use super::release::{fetch, installed_release}; +use super::{Error, InstallPaths, PLOYZ_USER, command_exists, run_apt, run_host, systemctl}; + +const DOCKER_DAEMON_CONFIG: &str = r#"{ + "features": { "containerd-snapshotter": true }, + "live-restore": true, + "log-driver": "json-file", + "log-opts": { "max-size": "10m", "max-file": "3" } +}"#; + +pub(super) fn install_prerequisites() -> Result<(), Error> { + if command_exists("curl") { + return Ok(()); + } + if command_exists("apt-get") { + run_apt( + "refresh packages for prerequisites", + ["update", "-qq"], + None, + )?; + run_apt( + "install prerequisites", + ["install", "-y", "-qq", "curl", "ca-certificates"], + None, + )?; + } else if command_exists("dnf") { + run_host( + "install prerequisites", + "dnf", + ["install", "-y", "curl", "ca-certificates"], + )?; + } else if command_exists("yum") { + run_host( + "install prerequisites", + "yum", + ["install", "-y", "curl", "ca-certificates"], + )?; + } else if command_exists("pacman") { + run_host( + "install prerequisites", + "pacman", + ["-Sy", "--noconfirm", "curl", "ca-certificates"], + )?; + } else if command_exists("zypper") { + run_host( + "install prerequisites", + "zypper", + ["--non-interactive", "install", "curl", "ca-certificates"], + )?; + } else { + return Err(Error::Command { + stage: "install prerequisites".into(), + message: "curl is required and no supported package manager was found".into(), + }); + } + Ok(()) +} + +pub(super) fn create_user_and_directories( + group_user: Option<&str>, + paths: &InstallPaths, +) -> Result<(), Error> { + let mut exists = Command::new("id"); + exists + .arg(PLOYZ_USER) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + if !exists + .status() + .map_err(|source| Error::Io { + stage: "inspect Ployz service account", + source, + })? + .success() + { + run_host( + "create Ployz service account", + "useradd", + [ + "--system", + "--home-dir", + "/nonexistent", + "--shell", + "/usr/sbin/nologin", + "--user-group", + PLOYZ_USER, + ], + )?; + } + if let Some(user) = group_user { + run_host( + "add operator to Ployz group", + "gpasswd", + ["--add", user, PLOYZ_USER], + )?; + } + let data = paths.data_dir.to_string_lossy(); + let run = paths.run_dir.to_string_lossy(); + run_host( + "create Ployz directories", + "install", + [ + "-d", "-m", "0750", "-o", PLOYZ_USER, "-g", PLOYZ_USER, &data, &run, + ], + )?; + Ok(()) +} + +pub(super) fn verify_software_prerequisites(paths: &InstallPaths) -> Result<(), Error> { + if !command_exists("dockerd") { + return Err(Error::Command { + stage: "software-only preflight".into(), + message: "Docker is not installed; run ployzd install without --software-only first" + .into(), + }); + } + let service_account = Command::new("id") + .arg(PLOYZ_USER) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|source| Error::Io { + stage: "software-only preflight", + source, + })?; + if !service_account.success() { + return Err(Error::Command { + stage: "software-only preflight".into(), + message: "the Ployz service account is missing; run ployzd install without --software-only first".into(), + }); + } + for path in [&paths.data_dir, &paths.run_dir] { + if !path.is_dir() { + return Err(Error::Command { + stage: "software-only preflight".into(), + message: format!( + "{} is missing; run ployzd install without --software-only first", + path.display() + ), + }); + } + } + Ok(()) +} + +pub(super) fn install_systemd(paths: &InstallPaths, install_only: bool) -> Result<(), Error> { + fs::create_dir_all(&paths.systemd_dir).map_err(|source| Error::Io { + stage: "create systemd unit directory", + source, + })?; + let bin = paths.bin_dir.display(); + write_file_atomically( + &paths.systemd_dir.join("ployz.service"), + &format!( + "[Unit]\nDescription=Ployz Machine daemon\nAfter=network-online.target docker.service\nWants=network-online.target\n\n[Service]\nType=notify\nExecStart={bin}/ployzd\n# Set PLOYZ_LOG=debug in /etc/default/ployz to raise verbosity.\nEnvironmentFile=-/etc/default/ployz\nTimeoutStartSec=20\nTimeoutStopSec=15\nRestart=always\nRestartPreventExitStatus=78\nRestartSec=2\nNoNewPrivileges=true\nProtectSystem=full\nProtectControlGroups=true\nProtectHome=read-only\nProtectKernelTunables=true\nPrivateTmp=true\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nRestrictNamespaces=true\n\n[Install]\nWantedBy=multi-user.target\n" + ), + "write systemd unit", + )?; + write_file_atomically( + &paths.systemd_dir.join("ployz-volume-plugin.socket"), + "[Unit]\nDescription=Ployz Docker Volume plugin socket\nBefore=docker.service\n\n[Socket]\nListenStream=/run/docker/plugins/ployz.sock\nSocketMode=0660\nDirectoryMode=0755\nAccept=no\nService=ployz-volume-plugin.service\n\n[Install]\nWantedBy=sockets.target\n", + "write systemd unit", + )?; + write_file_atomically( + &paths.systemd_dir.join("ployz-volume-plugin.service"), + &format!( + "[Unit]\nDescription=Ployz Docker Volume plugin\nBefore=docker.service\nAfter=zfs-import.target zfs-mount.service ployz-volume-plugin.socket\nRequires=ployz-volume-plugin.socket docker.service\n\n[Service]\nType=simple\nExecStart={bin}/ployzd volume-plugin\nSockets=ployz-volume-plugin.socket\nEnvironmentFile=-/etc/default/ployz\nRestart=on-failure\nRestartSec=2\nNoNewPrivileges=true\nRestrictAddressFamilies=AF_UNIX\nRestrictNamespaces=true\n" + ), + "write systemd unit", + )?; + if !install_only { + systemctl("reload systemd units", ["daemon-reload"])?; + systemctl("enable daemon", ["enable", "ployz.service"])?; + systemctl( + "enable volume plugin socket", + ["enable", "--now", "ployz-volume-plugin.socket"], + )?; + } + Ok(()) +} + +pub(super) fn write_file_atomically( + path: &Path, + content: &str, + stage: &'static str, +) -> Result<(), Error> { + atomic_write(path, content.as_bytes(), 0o644).map_err(|source| Error::Io { stage, source }) +} + +pub(super) async fn install_docker(paths: &InstallPaths) -> Result<(), Error> { + if command_exists("dockerd") { + let mut command = Command::new("docker"); + command.args(["info", "-f", "{{ .DriverStatus }}"]); + let snapshotter = command.output().ok().is_some_and(|output| { + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("io.containerd.snapshotter") + }); + if !snapshotter { + eprintln!( + "WARNING: Docker is retained unchanged; enable its containerd image store for best results" + ); + } + return Ok(()); + } + let script = fetch("https://get.docker.com", "download Docker installer").await?; + let mut command = Command::new("bash"); + command.args(["-o", "pipefail"]); + command.stdin(Stdio::piped()); + let mut child = command.spawn().map_err(|source| Error::Io { + stage: "start Docker installer", + source, + })?; + child + .stdin + .take() + .ok_or_else(|| { + Error::Verification("Docker installer standard input was unavailable".into()) + })? + .write_all(&script) + .map_err(|source| Error::Io { + stage: "send Docker installer", + source, + })?; + let status = child.wait().map_err(|source| Error::Io { + stage: "wait for Docker installer", + source, + })?; + if !status.success() { + return Err(Error::Command { + stage: "install Docker".into(), + message: format!("exited with {status}"), + }); + } + let parent = paths.docker_config.parent().ok_or_else(|| { + Error::Verification(format!( + "Docker config path {} has no parent", + paths.docker_config.display() + )) + })?; + fs::create_dir_all(parent).map_err(|source| Error::Io { + stage: "create Docker configuration directory", + source, + })?; + write_file_atomically( + &paths.docker_config, + DOCKER_DAEMON_CONFIG, + "write Docker configuration", + )?; + systemctl("restart Docker", ["restart", "docker"])?; + Ok(()) +} + +pub(super) async fn verify_running_daemon( + paths: &InstallPaths, + target: &MachineVersion, +) -> Result<(), Error> { + systemctl( + "check daemon readiness", + ["is-active", "--quiet", "ployz.service"], + )?; + let output = systemctl( + "inspect running daemon", + ["show", "--property=MainPID", "--value", "ployz.service"], + )?; + let pid = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if pid.is_empty() || pid == "0" || !pid.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(Error::Verification(format!( + "ployz.service is active but did not report a daemon process ID ({pid:?})" + ))); + } + let running = fs::metadata(format!("/proc/{pid}/exe")).map_err(|source| Error::Io { + stage: "inspect running daemon executable", + source, + })?; + let installed = fs::metadata(paths.bin_dir.join("ployzd")).map_err(|source| Error::Io { + stage: "inspect installed daemon executable", + source, + })?; + if (running.dev(), running.ino()) != (installed.dev(), installed.ino()) { + return Err(Error::Verification( + "ployz.service is active but does not run the activated daemon executable".into(), + )); + } + match installed_release(&paths.bin_dir.join("ployzd")).await? { + Some(observed) if &observed == target => {} + Some(observed) => Err(Error::Verification(format!( + "activated daemon reported {observed}, expected {target}" + )))?, + None => { + return Err(Error::Verification( + "activated daemon no longer reports a valid version".into(), + )); + } + } + verify_daemon_contract(&paths.run_dir.join("ployz.sock"), target).await +} + +async fn verify_daemon_contract(socket: &Path, target: &MachineVersion) -> Result<(), Error> { + let endpoint = + Endpoint::from_shared(format!("unix:{}", socket.display())).map_err(|error| { + Error::Verification(format!("invalid Machine API socket address: {error}")) + })?; + let channel = tokio::time::timeout(std::time::Duration::from_secs(10), endpoint.connect()) + .await + .map_err(|_| Error::Verification("Machine API readiness timed out after 10s".into()))? + .map_err(|error| Error::Verification(format!("Machine API is not ready: {error}")))?; + let payload = op::DescribeContract::into_request(DescribeContractRequest {}) + .encode() + .map_err(|error| Error::Verification(format!("encode readiness request: {error}")))?; + let response = tokio::time::timeout( + std::time::Duration::from_secs(10), + MachineRpcClient::new(channel).describe_contract(tonic::Request::new(payload)), + ) + .await + .map_err(|_| Error::Verification("Machine API readiness timed out after 10s".into()))? + .map_err(|error| Error::Verification(format!("Machine API is not ready: {error}")))? + .into_inner() + .decode_response() + .and_then(|response| response.decode::()) + .map_err(|error| Error::Verification(format!("decode readiness response: {error}")))?; + require_running_version(&response.daemon_version, target) +} + +fn require_running_version(observed: &str, target: &MachineVersion) -> Result<(), Error> { + let observed = MachineVersion::parse(observed).map_err(|_| { + Error::Verification(format!( + "running Machine API reported invalid daemon version {observed:?}" + )) + })?; + if &observed == target { + Ok(()) + } else { + Err(Error::Verification(format!( + "running Machine API reported {observed}, expected {target}" + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn running_machine_api_version_must_match_the_target() { + let target = MachineVersion::parse("1.2.3-beta.4").unwrap(); + assert!(require_running_version("1.2.3-beta.4", &target).is_ok()); + assert!(matches!( + require_running_version("1.2.3-beta.3", &target), + Err(Error::Verification(message)) + if message == "running Machine API reported 1.2.3-beta.3, expected 1.2.3-beta.4" + )); + assert!(matches!( + require_running_version("not-a-version", &target), + Err(Error::Verification(message)) + if message + == "running Machine API reported invalid daemon version \"not-a-version\"" + )); + } +} diff --git a/crates/ployzd/src/installer/mod.rs b/crates/ployzd/src/installer/mod.rs new file mode 100644 index 0000000000..52ab56acaf --- /dev/null +++ b/crates/ployzd/src/installer/mod.rs @@ -0,0 +1,765 @@ +//! Bounded local installation of a Ployz Machine release. + +mod host; +mod release; +mod storage; +pub mod upgrade; + +use std::{ + io, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +use ployz_core::{MachineUpgradeStage, StorageChoice}; +use thiserror::Error; + +use crate::mutation; + +use self::{ + host::{ + create_user_and_directories, install_docker, install_prerequisites, install_systemd, + verify_running_daemon, verify_software_prerequisites, + }, + release::{install_binaries, resolve_release}, + storage::prepare_storage, +}; + +pub use self::release::{ReleaseRequest, ReleaseSource}; + +const PLOYZ_USER: &str = "ployz"; +const DEFAULT_BIN_DIR: &str = "/usr/local/bin"; +const DEFAULT_SYSTEMD_DIR: &str = "/etc/systemd/system"; +const DEFAULT_RUN_DIR: &str = "/run/ployz"; +/// Unix socket installed systemd services use for the local Machine API. +pub const DEFAULT_SOCKET_PATH: &str = "/run/ployz/ployz.sock"; +/// Mutually exclusive host work for one Machine installation attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum InstallMode { + /// Write the release and units without starting systemd or preparing the host. + InstallationOnly, + /// Replace the daemon only after confirming the Machine is already prepared. + SoftwareOnly, + /// Prepare the host and optionally configure its requested storage. + PrepareHost { + /// Explicit storage preparation for a fresh Machine. + storage: StorageChoice, + /// Existing operator to add to the Ployz service group during host preparation. + group_user: Option, + }, +} + +/// Choices for one local Machine installation attempt. +#[derive(Clone, Debug)] +pub struct InstallRequest { + /// A fixed version or a channel resolved once before host mutation. + pub release: ReleaseRequest, + /// Trusted release source. Published releases never accept caller-provided URLs. + pub source: ReleaseSource, + /// Installation-only, software-only replacement, or full host preparation. + pub mode: InstallMode, +} + +/// What installation directly observed after activation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Readiness { + /// Files and unit definitions were installed, but systemd was deliberately not contacted. + InstallationOnly, + /// systemd reported the daemon active and its process is the verified target executable. + Running, +} + +/// The bounded outcome of one local installation attempt. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InstallOutcome { + /// Exact release selected before host mutation. + pub target: String, + /// Readiness observed before the command returned. + pub readiness: Readiness, +} + +/// Installation failures name the first stage that did not complete. +#[derive(Debug, Error)] +pub enum Error { + #[error("run this command with sudo")] + NotRoot, + #[error("another Ployz installation is active")] + Busy, + #[error("nightly is not a supported release channel")] + Nightly, + #[error("invalid Ployz release version '{value}'; expected X.Y.Z or X.Y.Z-beta.N")] + InvalidVersion { value: String }, + #[error("Ployz Machine must be Linux")] + UnsupportedOs, + #[error("unsupported architecture: {0}")] + UnsupportedArchitecture(String), + #[error("Ployz requires systemd")] + SystemdRequired, + #[error("{0}")] + NonstandardPaths(String), + #[error("release selection: {0}")] + ReleaseSelection(String), + #[error("artifact verification: {0}")] + Verification(String), + #[error("{stage}: {message}")] + Command { stage: String, message: String }, + #[error("{stage}: {source}")] + Io { + stage: &'static str, + #[source] + source: io::Error, + }, +} + +#[derive(Clone, Debug)] +pub(super) struct InstallPaths { + pub(super) bin_dir: PathBuf, + pub(super) systemd_dir: PathBuf, + pub(super) data_dir: PathBuf, + pub(super) run_dir: PathBuf, + pub(super) docker_config: PathBuf, + pub(super) modprobe_dir: PathBuf, +} + +impl InstallPaths { + fn system(data_dir: impl Into, run_dir: impl Into) -> Self { + Self { + bin_dir: PathBuf::from(DEFAULT_BIN_DIR), + systemd_dir: PathBuf::from(DEFAULT_SYSTEMD_DIR), + data_dir: data_dir.into(), + run_dir: run_dir.into(), + docker_config: PathBuf::from("/etc/docker/daemon.json"), + modprobe_dir: PathBuf::from("/etc/modprobe.d"), + } + } + + #[cfg(test)] + fn at(root: &Path) -> Self { + Self { + bin_dir: root.join("bin"), + systemd_dir: root.join("systemd"), + data_dir: root.join("data"), + run_dir: root.join("run"), + docker_config: root.join("docker/daemon.json"), + modprobe_dir: root.join("modprobe"), + } + } +} + +pub(crate) fn require_standard_machine_paths(data_dir: &Path, socket: &Path) -> Result<(), String> { + if data_dir == Path::new(crate::machine::DEFAULT_DATA_DIR) + && socket == Path::new(DEFAULT_SOCKET_PATH) + { + return Ok(()); + } + Err(format!( + "system installation requires --data-dir {} and --socket {}; received --data-dir {} and --socket {}", + crate::machine::DEFAULT_DATA_DIR, + DEFAULT_SOCKET_PATH, + data_dir.display(), + socket.display() + )) +} + +/// Install into global host locations for the standard Machine data and socket paths. +/// +/// # Errors +/// +/// Returns the observed first failed installation stage. +pub async fn install( + request: InstallRequest, + data_dir: impl Into, + socket: impl Into, +) -> Result { + let data_dir = data_dir.into(); + let socket = socket.into(); + require_standard_machine_paths(&data_dir, &socket).map_err(Error::NonstandardPaths)?; + install_at(request, InstallPaths::system(data_dir, DEFAULT_RUN_DIR)).await +} + +async fn install_at(request: InstallRequest, paths: InstallPaths) -> Result { + require_root()?; + let admission = mutation::MutationGate::new(&paths.run_dir, &paths.data_dir); + let lock = admission.try_installation().map_err(map_admission_error)?; + upgrade::reconcile_for_install(&admission, &paths.data_dir) + .await + .map_err(map_upgrade_reconciliation)?; + install_locked(request, paths, lock, |_| Ok(())).await +} + +async fn install_locked( + request: InstallRequest, + paths: InstallPaths, + _lock: mutation::InstallationGuard, + mut progress: impl FnMut(MachineUpgradeStage) -> Result<(), Error>, +) -> Result { + let installation_only = matches!(request.mode, InstallMode::InstallationOnly); + verify_system(installation_only)?; + let target = resolve_release(&request.release, &request.source).await?; + + progress(MachineUpgradeStage::Preparing)?; + match &request.mode { + InstallMode::InstallationOnly => {} + InstallMode::SoftwareOnly => verify_software_prerequisites(&paths)?, + InstallMode::PrepareHost { + storage, + group_user, + } => { + prepare_storage(*storage, &paths)?; + install_prerequisites()?; + let inherited_group = sudo_user(); + create_user_and_directories( + group_user.as_deref().or(inherited_group.as_deref()), + &paths, + )?; + } + } + + let mut restart_required = !paths.systemd_dir.join("ployz.service").is_file(); + restart_required |= install_binaries(&request.source, &paths, &target, &mut progress).await?; + install_systemd(&paths, installation_only)?; + if matches!(request.mode, InstallMode::PrepareHost { .. }) { + install_docker(&paths).await?; + } + + let readiness = if installation_only { + Readiness::InstallationOnly + } else { + if restart_required { + progress(MachineUpgradeStage::Restarting)?; + systemctl("restart daemon", ["restart", "ployz.service"])?; + systemctl( + "restart volume plugin", + ["try-restart", "ployz-volume-plugin.service"], + )?; + } + progress(MachineUpgradeStage::Readiness)?; + verify_running_daemon(&paths, &target).await?; + Readiness::Running + }; + Ok(InstallOutcome { + target: target.to_string(), + readiness, + }) +} + +fn map_admission_error(error: mutation::Error) -> Error { + match error { + mutation::Error::Busy => Error::Busy, + mutation::Error::Io(source) => Error::Io { + stage: "claim Machine installation admission", + source, + }, + } +} + +fn map_upgrade_reconciliation(error: upgrade::Error) -> Error { + if matches!(error, upgrade::Error::Busy) { + Error::Busy + } else { + Error::Command { + stage: "reconcile previous Machine upgrade".into(), + message: error.to_string(), + } + } +} + +fn require_root() -> Result<(), Error> { + let mut command = Command::new("id"); + command.arg("-u"); + let output = run_command("check installation privilege", &mut command)?; + if String::from_utf8_lossy(&output.stdout).trim() == "0" { + Ok(()) + } else { + Err(Error::NotRoot) + } +} + +fn verify_system(install_only: bool) -> Result<(), Error> { + if std::env::consts::OS != "linux" { + return Err(Error::UnsupportedOs); + } + daemon_archive()?; + if !command_exists("tar") { + return Err(Error::Command { + stage: "preflight daemon archive extraction".into(), + message: "tar is required to extract Ployz release archives".into(), + }); + } + if !install_only && !Path::new("/run/systemd/system").is_dir() { + return Err(Error::SystemdRequired); + } + Ok(()) +} + +pub(super) fn daemon_archive() -> Result<&'static str, Error> { + match std::env::consts::ARCH { + "x86_64" => Ok("ployzd_linux_amd64.tar.gz"), + "aarch64" => Ok("ployzd_linux_arm64.tar.gz"), + architecture => Err(Error::UnsupportedArchitecture(architecture.into())), + } +} + +fn sudo_user() -> Option { + std::env::var("SUDO_USER") + .ok() + .filter(|user| !user.is_empty()) +} + +pub(super) fn command_exists(name: &str) -> bool { + std::env::var_os("PATH").is_some_and(|path| { + std::env::split_paths(&path).any(|directory| { + let candidate = directory.join(name); + candidate.is_file() + && candidate + .metadata() + .is_ok_and(|metadata| metadata.permissions().mode() & 0o111 != 0) + }) + }) +} + +pub(super) fn run_command( + stage: impl Into, + command: &mut Command, +) -> Result { + let stage = stage.into(); + let output = command.output().map_err(|source| Error::Io { + stage: "start host command", + source, + })?; + if output.status.success() { + Ok(output) + } else { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + Err(Error::Command { + stage, + message: if stderr.is_empty() { + format!("exited with {}", output.status) + } else { + stderr + }, + }) + } +} + +pub(super) fn systemctl(stage: &str, args: [&str; N]) -> Result { + let mut command = Command::new("systemctl"); + command.args(args); + run_command(stage, &mut command) +} + +pub(super) fn run_host( + stage: &str, + program: &str, + args: [&str; N], +) -> Result { + let mut command = Command::new(program); + command.args(args); + run_command(stage, &mut command) +} + +pub(super) fn run_apt( + stage: &str, + args: [&str; N], + working_directory: Option<&Path>, +) -> Result { + let mut command = Command::new("apt-get"); + command + .args(["-o", "DPkg::Lock::Timeout=300"]) + .args(args) + .env("DEBIAN_FRONTEND", "noninteractive"); + if let Some(working_directory) = working_directory { + command.current_dir(working_directory); + } + run_command(stage, &mut command) +} + +#[cfg(test)] +mod tests { + use std::{ + env, + ffi::OsString, + fs, + io::Write, + os::unix::fs::{PermissionsExt, symlink}, + process::Command, + }; + + use ployz_core::MachineVersion; + use tempfile::TempDir; + + use super::*; + + #[tokio::test] + async fn system_install_rejects_nonstandard_machine_paths_before_mutation() { + let fixture = fixture("nonstandard-paths"); + let request = InstallRequest { + release: ReleaseRequest::Exact(MachineVersion::parse("1.2.3").unwrap()), + source: ReleaseSource::Local(fixture.path().join("release")), + mode: InstallMode::InstallationOnly, + }; + + for (data_dir, socket) in [ + (fixture.path(), Path::new(DEFAULT_SOCKET_PATH)), + (Path::new(crate::machine::DEFAULT_DATA_DIR), fixture.path()), + ] { + let error = install(request.clone(), data_dir, socket) + .await + .unwrap_err(); + assert!(matches!(error, Error::NonstandardPaths(_))); + } + assert!(fs::read_dir(fixture.path()).unwrap().next().is_none()); + } + + #[tokio::test] + async fn installation_interface_contract() { + if let Ok(case) = env::var("PLOYZ_INSTALLER_CONTRACT_CASE") { + run_installation_case(&case).await; + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + fs::write(root.join("child-completed"), format!("installation:{case}")).unwrap(); + return; + } + + for case in [ + "success", + "install-only", + "same-target", + "missing", + "missing-tar", + "bad-checksum", + "corrupt", + "rejected-executable", + "hung-executable", + "busy", + ] { + let fixture = fixture(case); + create_installation_fixture(fixture.path(), case); + run_contract_child("installation_interface_contract", fixture.path(), case); + } + } + + #[test] + fn software_only_prerequisite_contract() { + if env::var_os("PLOYZ_SOFTWARE_PREREQUISITE_CONTRACT").is_some() { + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + let paths = InstallPaths::at(&root); + assert!(matches!( + verify_software_prerequisites(&paths), + Err(Error::Command { stage, message }) + if stage == "software-only preflight" + && message + == "Docker is not installed; run ployzd install without --software-only first" + )); + fs::write(root.join("child-completed"), "software-prerequisite").unwrap(); + return; + } + + let fixture = fixture("software-prerequisite"); + fs::create_dir_all(fixture.path().join("commands")).unwrap(); + run_contract_child_with_environment( + "software_only_prerequisite_contract", + fixture.path(), + OsString::from("PLOYZ_SOFTWARE_PREREQUISITE_CONTRACT"), + OsString::from("1"), + "software-prerequisite", + [], + ); + } + + #[test] + fn zfs_candidate_download_contract() { + if env::var_os("PLOYZ_ZFS_CANDIDATE_CONTRACT").is_some() { + super::storage::install_zfs_packages("test-kernel").unwrap(); + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + fs::write(root.join("child-completed"), "zfs-candidate").unwrap(); + return; + } + + let fixture = fixture("zfs-candidate"); + let commands = fixture.path().join("commands"); + fs::create_dir_all(&commands).unwrap(); + let marker = fixture.path().join("installed"); + write_script( + &commands.join("apt-cache"), + "[ \"$2\" = linux-main-modules-zfs-test-kernel ]", + ); + write_script( + &commands.join("dpkg-query"), + "if [ -f \"$PLOYZ_ZFS_INSTALLED\" ]; then echo /lib/modules/test-kernel/kernel/zfs.ko; exit 0; fi\nexit 1", + ); + write_script( + &commands.join("apt-get"), + "case \"$*\" in\n *download*) : > module.deb ;;\n *install*) : > \"$PLOYZ_ZFS_INSTALLED\" ;;\nesac", + ); + write_script( + &commands.join("dpkg-deb"), + "echo /lib/modules/test-kernel/kernel/zfs.ko", + ); + run_contract_child_with_environment( + "zfs_candidate_download_contract", + fixture.path(), + OsString::from("PLOYZ_ZFS_CANDIDATE_CONTRACT"), + OsString::from("1"), + "zfs-candidate", + [( + OsString::from("PLOYZ_ZFS_INSTALLED"), + marker.into_os_string(), + )], + ); + assert!(fixture.path().join("installed").is_file()); + } + + async fn run_installation_case(case: &str) { + let root = PathBuf::from(env::var_os("PLOYZ_INSTALLER_CONTRACT_ROOT").unwrap()); + let paths = InstallPaths::at(&root); + let existing = match case { + "success" | "install-only" => None, + "same-target" => Some(write_existing_daemon(&paths, "1.2.3")), + _ => Some(write_existing_daemon(&paths, "1.2.2")), + }; + if case == "install-only" { + fs::remove_file(root.join("commands/dockerd")).unwrap(); + } + if case == "missing-tar" { + fs::remove_file(root.join("commands/tar")).unwrap(); + } + + let request = InstallRequest { + release: ReleaseRequest::Exact(MachineVersion::parse("1.2.3").unwrap()), + source: ReleaseSource::Local(root.join("release")), + mode: InstallMode::InstallationOnly, + }; + let result = if case == "busy" { + let admission = mutation::MutationGate::new(&paths.run_dir, &paths.data_dir); + let _held = admission.try_installation().unwrap(); + install_at(request, paths.clone()).await + } else { + install_at(request, paths.clone()).await + }; + + match case { + "success" | "install-only" => { + let outcome = result.unwrap(); + assert_eq!(outcome.target, "1.2.3"); + assert_eq!(outcome.readiness, Readiness::InstallationOnly); + assert!(paths.bin_dir.join("ployzd").is_file()); + assert!(paths.systemd_dir.join("ployz.service").is_file()); + if case == "install-only" { + assert!(!root.join("forbidden-invocation").exists()); + assert!(!paths.data_dir.exists()); + } + } + "same-target" => { + let outcome = result.unwrap(); + assert_eq!(outcome.target, "1.2.3"); + let existing = existing.as_ref().unwrap(); + assert_ne!(fs::read(paths.bin_dir.join("ployzd")).unwrap(), *existing); + assert_eq!( + fs::read(paths.bin_dir.join("ployzd.previous")).unwrap(), + *existing + ); + } + "busy" => assert!(matches!(result, Err(Error::Busy))), + "missing-tar" => assert!(matches!( + result, + Err(Error::Command { stage, message }) + if stage == "preflight daemon archive extraction" + && message == "tar is required to extract Ployz release archives" + )), + "hung-executable" => { + assert!(matches!( + result, + Err(Error::Command { stage, message }) + if stage == "preflight staged daemon" && message == "timed out after 100ms" + )); + } + "bad-checksum" => assert!(matches!( + result, + Err(Error::Verification(message)) + if message + == format!("checksums.txt has no hash for {}", daemon_archive().unwrap()) + )), + "corrupt" => assert!(matches!( + result, + Err(Error::Verification(message)) + if message.contains(daemon_archive().unwrap()) + && message.contains("checksum was") + && message.contains("expected") + )), + "missing" | "rejected-executable" => { + assert!(result.is_err(), "{case} artifact was accepted"); + } + other => panic!("unknown contract case {other}"), + } + if let Some(existing) = existing.filter(|_| case != "same-target") { + assert_eq!(fs::read(paths.bin_dir.join("ployzd")).unwrap(), existing); + } + } + + fn create_installation_fixture(root: &Path, case: &str) { + let commands = root.join("commands"); + let release = root.join("release"); + let payload = root.join("payload"); + fs::create_dir_all(&commands).unwrap(); + fs::create_dir_all(&release).unwrap(); + fs::create_dir_all(&payload).unwrap(); + write_script(&commands.join("id"), "if [ \"$1\" = -u ]; then echo 0; fi"); + write_script(&commands.join("dockerd"), "exit 0"); + symlink("/usr/bin/tar", commands.join("tar")).unwrap(); + symlink("/usr/bin/gzip", commands.join("gzip")).unwrap(); + symlink("/usr/bin/sleep", commands.join("sleep")).unwrap(); + + let daemon = match case { + "rejected-executable" => "case \"$1\" in version) echo 9.9.9 ;; esac", + "hung-executable" => "case \"$1\" in version) sleep 1; echo 1.2.3 ;; esac", + _ => "case \"$1\" in version) echo 1.2.3 ;; esac", + }; + write_script(&payload.join("ployzd"), daemon); + write_script(&payload.join("ployz-uninstall"), "exit 0"); + if case == "install-only" { + write_script( + &commands.join("id"), + "if [ \"$1\" = -u ]; then echo 0; else echo id >> \"$PLOYZ_INSTALLER_FORBIDDEN\"; exit 97; fi", + ); + for command in [ + "apt-get", + "dnf", + "yum", + "pacman", + "zypper", + "bash", + "docker", + "systemctl", + ] { + write_script( + &commands.join(command), + "echo \"$0\" >> \"$PLOYZ_INSTALLER_FORBIDDEN\"; exit 97", + ); + } + } + if case != "missing" { + let archive = release.join(daemon_archive().unwrap()); + let status = Command::new("tar") + .args(["-czf"]) + .arg(&archive) + .args(["-C"]) + .arg(&payload) + .args(["ployzd", "ployz-uninstall"]) + .status() + .unwrap(); + assert!(status.success()); + let checksum = String::from_utf8( + Command::new("sha256sum") + .arg(&archive) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .split_whitespace() + .next() + .unwrap() + .to_owned(); + fs::write( + release.join("checksums.txt"), + format!("{checksum} {}\n", daemon_archive().unwrap()), + ) + .unwrap(); + if case == "corrupt" { + fs::OpenOptions::new() + .append(true) + .open(archive) + .unwrap() + .write_all(b"corrupt") + .unwrap(); + } + } else { + fs::write( + release.join("checksums.txt"), + format!("{} {}\n", "0".repeat(64), daemon_archive().unwrap()), + ) + .unwrap(); + } + if case == "bad-checksum" { + fs::write(release.join("checksums.txt"), "bad checksum\n").unwrap(); + } + } + + fn write_existing_daemon(paths: &InstallPaths, version: &str) -> Vec { + fs::create_dir_all(&paths.bin_dir).unwrap(); + let existing = format!("#!/bin/sh\n[ \"$1\" = version ] && echo {version}\n").into_bytes(); + fs::write(paths.bin_dir.join("ployzd"), &existing).unwrap(); + fs::set_permissions( + paths.bin_dir.join("ployzd"), + fs::Permissions::from_mode(0o755), + ) + .unwrap(); + existing + } + + fn run_contract_child(test: &str, root: &Path, case: &str) { + let key = OsString::from("PLOYZ_INSTALLER_CONTRACT_CASE"); + let value = OsString::from(case); + let completion = format!("installation:{case}"); + if case == "install-only" { + run_contract_child_with_environment( + test, + root, + key, + value, + &completion, + [( + OsString::from("PLOYZ_INSTALLER_FORBIDDEN"), + root.join("forbidden-invocation").into_os_string(), + )], + ); + } else { + run_contract_child_with_environment(test, root, key, value, &completion, []); + } + } + + fn run_contract_child_with_environment( + test: &str, + root: &Path, + key: OsString, + value: OsString, + completion: &str, + extra: [(OsString, OsString); N], + ) { + let test = format!("installer::tests::{test}"); + let mut command = Command::new(env::current_exe().unwrap()); + command + .args(["--exact", &test, "--nocapture"]) + .env("PLOYZ_INSTALLER_CONTRACT_ROOT", root) + .env("PATH", root.join("commands")) + .env(key, value); + for (key, value) in extra { + command.env(key, value); + } + let output = command.output().unwrap(); + assert!( + output.status.success(), + "contract child {test} failed:\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + assert_eq!( + fs::read_to_string(root.join("child-completed")).unwrap(), + completion, + "contract child {test} did not complete its fixture", + ); + } + + fn write_script(path: &Path, body: &str) { + fs::write(path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + + fn fixture(name: &str) -> TempDir { + tempfile::Builder::new() + .prefix(&format!("ployzd-installer-{name}-")) + .tempdir() + .unwrap() + } +} diff --git a/crates/ployzd/src/installer/release.rs b/crates/ployzd/src/installer/release.rs new file mode 100644 index 0000000000..53f538e197 --- /dev/null +++ b/crates/ployzd/src/installer/release.rs @@ -0,0 +1,492 @@ +//! Release selection, acquisition, verification, and activation. + +use std::{ + ffi::OsStr, + fs::{self, File, OpenOptions}, + io::{self, Write}, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, + str::FromStr, + time::Duration, +}; + +#[cfg(not(test))] +use std::os::unix::fs::chown; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; +use tokio::{process::Command, time::timeout}; + +use super::{Error, InstallPaths, daemon_archive, run_command}; +use ployz_core::{MachineUpgradeStage, MachineVersion}; + +const RELEASE_REPOSITORY: &str = "https://github.com/getployz/ployz2"; +const CHANNEL_URL: &str = "https://ployz.sh"; +const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(not(test))] +const VERSION_COMMAND_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(test)] +const VERSION_COMMAND_TIMEOUT: Duration = Duration::from_millis(100); + +/// A trusted release target selected at the CLI boundary. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ReleaseRequest { + /// Resolve the stable channel once for this installation attempt. + Stable, + /// Resolve the beta channel once for this installation attempt. + Beta, + /// Install this exact published daemon version. + Exact(MachineVersion), +} + +impl FromStr for ReleaseRequest { + type Err = Error; + + fn from_str(value: &str) -> Result { + let value = value.strip_prefix('v').unwrap_or(value); + match value { + "" | "latest" | "stable" => Ok(Self::Stable), + "beta" => Ok(Self::Beta), + "nightly" => Err(Error::Nightly), + value => MachineVersion::parse(value.to_owned()) + .map(Self::Exact) + .map_err(|_| Error::InvalidVersion { + value: value.to_owned(), + }), + } + } +} + +/// A release source that is fixed by the local installer invocation. +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReleaseSource { + /// Ployz's fixed, trusted published release and channel endpoints. + Published, + /// An operator-provided local release directory used only by offline qualification. + Local(PathBuf), +} + +impl ReleaseSource { + fn is_local(&self) -> bool { + matches!(self, Self::Local(_)) + } + + async fn channel(&self, name: &str) -> Result { + let bytes = match self { + Self::Published => { + fetch(&format!("{CHANNEL_URL}/{name}"), "resolve release channel").await? + } + Self::Local(directory) => { + fs::read(directory.join(name)).map_err(|source| Error::Io { + stage: "read local release channel", + source, + })? + } + }; + String::from_utf8(bytes).map_err(|error| { + Error::ReleaseSelection(format!("{name} channel is not UTF-8: {error}")) + }) + } + + async fn release_file( + &self, + target: &MachineVersion, + file: &str, + stage: &'static str, + ) -> Result, Error> { + match self { + Self::Published => fetch(&release_url(target, file), stage).await, + Self::Local(directory) => fs::read(directory.join(file)).map_err(|source| Error::Io { + stage: "read local release artifact", + source, + }), + } + } +} + +pub(super) async fn resolve_release( + request: &ReleaseRequest, + source: &ReleaseSource, +) -> Result { + match request { + ReleaseRequest::Stable => { + let value = source.channel("stable").await?; + parse_channel_version(value.trim()) + } + ReleaseRequest::Beta => { + let value = source.channel("beta").await?; + parse_channel_version(value.trim()) + } + ReleaseRequest::Exact(version) => Ok(version.clone()), + } +} + +fn parse_channel_version(value: &str) -> Result { + MachineVersion::parse(value.to_owned()).map_err(|_| Error::InvalidVersion { + value: value.to_owned(), + }) +} + +pub(super) async fn fetch(url: &str, stage: &str) -> Result, Error> { + let client = reqwest::Client::builder() + .https_only(true) + .connect_timeout(DOWNLOAD_TIMEOUT) + .read_timeout(DOWNLOAD_TIMEOUT) + .user_agent("ployzd-installer") + .build() + .map_err(|error| Error::ReleaseSelection(format!("build download client: {error}")))?; + let response = client + .get(url) + .send() + .await + .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}")))? + .error_for_status() + .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}")))?; + response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| Error::ReleaseSelection(format!("{stage}: {error}"))) +} + +fn release_url(target: &MachineVersion, file: &str) -> String { + format!("{RELEASE_REPOSITORY}/releases/download/v{target}/{file}") +} + +pub(super) async fn install_binaries( + source: &ReleaseSource, + paths: &InstallPaths, + target: &MachineVersion, + progress: &mut impl FnMut(MachineUpgradeStage) -> Result<(), Error>, +) -> Result { + let installed = installed_release(&paths.bin_dir.join("ployzd")).await?; + let replace = replacement_required(source, installed.as_ref(), target); + if !replace { + println!( + "ployzd {} retained", + installed.expect("a skipped replacement has an installed release") + ); + return Ok(false); + } + + progress(MachineUpgradeStage::Acquiring)?; + let archive = daemon_archive()?; + let stage = staging_directory(&paths.bin_dir)?; + let archive_path = stage.path().join(archive); + let checksum = release_checksum(source, target, archive).await?; + let archive_bytes = source + .release_file(target, archive, "download daemon archive") + .await?; + progress(MachineUpgradeStage::Verifying)?; + verify_checksum(&archive_bytes, archive, &checksum)?; + write_private(&archive_path, &archive_bytes, "stage daemon archive")?; + extract_archive(&archive_path, stage.path())?; + let daemon = stage.path().join("ployzd"); + let uninstall = stage.path().join("ployz-uninstall"); + verify_executable(&daemon, target).await?; + verify_uninstall(&uninstall)?; + sync_staged_files(&daemon, &uninstall, stage.path())?; + progress(MachineUpgradeStage::Activating)?; + activate(&daemon, &uninstall, paths)?; + Ok(true) +} + +fn replacement_required( + source: &ReleaseSource, + installed: Option<&MachineVersion>, + target: &MachineVersion, +) -> bool { + source.is_local() || installed.is_none_or(|installed| installed != target) +} + +pub(super) async fn installed_release(path: &Path) -> Result, Error> { + if !path.is_file() { + return Ok(None); + } + let output = version_command(path, "inspect installed daemon").await?; + if !output.status.success() { + return Err(Error::Verification(format!( + "installed daemon version check exited with {}", + output.status + ))); + } + MachineVersion::parse(String::from_utf8_lossy(&output.stdout).trim().to_owned()) + .map(Some) + .map_err(|_| Error::Verification("installed daemon reported an invalid version".into())) +} + +async fn release_checksum( + source: &ReleaseSource, + target: &MachineVersion, + archive: &str, +) -> Result { + let checksums = source + .release_file(target, "checksums.txt", "download release checksums") + .await?; + checksum_for(&checksums, archive) + .ok_or_else(|| Error::Verification(format!("checksums.txt has no hash for {archive}"))) +} + +pub(super) fn checksum_for(checksums: &[u8], archive: &str) -> Option { + let checksums = std::str::from_utf8(checksums).ok()?; + checksums.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + let hash = fields.next()?; + let name = fields.next()?.trim_start_matches('*'); + (name == archive && valid_checksum(hash)).then(|| hash.to_ascii_lowercase()) + }) +} + +fn valid_checksum(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn verify_checksum(bytes: &[u8], archive: &str, expected: &str) -> Result<(), Error> { + let actual = hex::encode(Sha256::digest(bytes)); + if actual == expected { + Ok(()) + } else { + Err(Error::Verification(format!( + "{} checksum was {actual}, expected {expected}", + Path::new(archive) + .file_name() + .and_then(OsStr::to_str) + .unwrap_or("daemon archive") + ))) + } +} + +fn extract_archive(archive: &Path, destination: &Path) -> Result<(), Error> { + let mut command = std::process::Command::new("tar"); + command + // Release archives are produced by a different build user. The daemon must not inherit + // that user's numeric ownership when root installs it on a Machine. + .args(["--no-same-owner", "-xzf"]) + .arg(archive) + .arg("-C") + .arg(destination); + run_command("extract staged daemon archive", &mut command)?; + Ok(()) +} + +async fn verify_executable(path: &Path, target: &MachineVersion) -> Result<(), Error> { + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).map_err(|source| Error::Io { + stage: "mark staged daemon executable", + source, + })?; + let output = version_command(path, "preflight staged daemon").await?; + if !output.status.success() { + return Err(Error::Verification(format!( + "staged daemon version check exited with {}", + output.status + ))); + } + let observed = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if observed == target.as_str() { + Ok(()) + } else { + Err(Error::Verification(format!( + "staged daemon reported version {observed:?}, expected {target}" + ))) + } +} + +async fn version_command(path: &Path, stage: &str) -> Result { + let mut command = Command::new(path); + command.arg("version").kill_on_drop(true); + match timeout(VERSION_COMMAND_TIMEOUT, command.output()).await { + Ok(Ok(output)) => Ok(output), + Ok(Err(error)) => Err(Error::Command { + stage: stage.into(), + message: error.to_string(), + }), + Err(_) => Err(Error::Command { + stage: stage.into(), + message: format!("timed out after {}ms", VERSION_COMMAND_TIMEOUT.as_millis()), + }), + } +} + +fn verify_uninstall(path: &Path) -> Result<(), Error> { + let metadata = path.metadata().map_err(|source| Error::Io { + stage: "preflight staged uninstall command", + source, + })?; + if metadata.is_file() && metadata.len() != 0 { + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).map_err(|source| Error::Io { + stage: "mark staged uninstall command executable", + source, + }) + } else { + Err(Error::Verification( + "daemon archive does not contain a usable ployz-uninstall command".into(), + )) + } +} + +fn sync_staged_files(daemon: &Path, uninstall: &Path, staging: &Path) -> Result<(), Error> { + for path in [daemon, uninstall] { + File::open(path) + .and_then(|file| file.sync_all()) + .map_err(|source| Error::Io { + stage: "persist verified staged release", + source, + })?; + } + File::open(staging) + .and_then(|directory| directory.sync_all()) + .map_err(|source| Error::Io { + stage: "persist verified staging directory", + source, + }) +} + +pub(super) fn staging_directory(parent: &Path) -> Result { + fs::create_dir_all(parent).map_err(|source| Error::Io { + stage: "create daemon installation directory", + source, + })?; + tempfile::Builder::new() + .prefix(".ployz-stage-") + .permissions(fs::Permissions::from_mode(0o700)) + .tempdir_in(parent) + .map_err(|source| Error::Io { + stage: "create private daemon staging directory", + source, + }) +} + +pub(super) fn write_private(path: &Path, bytes: &[u8], stage: &'static str) -> Result<(), Error> { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(path) + .map_err(|source| Error::Io { stage, source })?; + file.write_all(bytes) + .and_then(|()| file.sync_all()) + .map_err(|source| Error::Io { stage, source }) +} + +fn activate(daemon: &Path, uninstall: &Path, paths: &InstallPaths) -> Result<(), Error> { + let installed = paths.bin_dir.join("ployzd"); + let previous = paths.bin_dir.join("ployzd.previous"); + let directory = File::open(&paths.bin_dir).map_err(|source| Error::Io { + stage: "open daemon installation directory", + source, + })?; + match fs::remove_file(&previous) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(source) => { + return Err(Error::Io { + stage: "remove previous daemon release", + source, + }); + } + } + if installed.exists() { + fs::hard_link(&installed, &previous).map_err(|source| Error::Io { + stage: "retain previous daemon release", + source, + })?; + root_ownership(&previous, "secure retained previous daemon release")?; + } + directory.sync_all().map_err(|source| Error::Io { + stage: "persist previous daemon release", + source, + })?; + fs::rename(daemon, &installed).map_err(|source| Error::Io { + stage: "activate verified daemon", + source, + })?; + root_ownership(&installed, "secure activated daemon")?; + directory.sync_all().map_err(|source| Error::Io { + stage: "persist daemon activation", + source, + })?; + let installed_uninstall = paths.bin_dir.join("ployz-uninstall"); + fs::rename(uninstall, &installed_uninstall).map_err(|source| Error::Io { + stage: "activate uninstall command", + source, + })?; + root_ownership(&installed_uninstall, "secure activated uninstall command")?; + directory.sync_all().map_err(|source| Error::Io { + stage: "persist uninstall command activation", + source, + }) +} + +fn root_ownership(path: &Path, stage: &'static str) -> Result<(), Error> { + #[cfg(not(test))] + chown(path, Some(0), Some(0)).map_err(|source| Error::Io { stage, source })?; + #[cfg(test)] + let _ = (path, stage); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::os::unix::fs::PermissionsExt; + + use super::*; + + #[test] + fn published_release_requests_accept_only_trusted_shapes() { + assert!(matches!("stable".parse(), Ok(ReleaseRequest::Stable))); + assert!(matches!("beta".parse(), Ok(ReleaseRequest::Beta))); + assert_eq!( + "v1.2.3-beta.4".parse::().unwrap(), + ReleaseRequest::Exact(MachineVersion::parse("1.2.3-beta.4").unwrap()) + ); + for invalid in ["1.2", "1.2.3-rc.1", "1.2.3-beta.x", "1.2.3+build.1"] { + assert!(invalid.parse::().is_err(), "{invalid}"); + } + } + + #[test] + fn checksum_parser_requires_the_requested_daemon_archive() { + let archive = "ployzd_linux_amd64.tar.gz"; + let checksum = "2d20468cbeb9745b56fdf16897f963773baeecf45c4ea2e00a46e8d82da6ee9f"; + assert_eq!( + checksum_for(format!("{checksum} {archive}\n").as_bytes(), archive), + Some(checksum.into()) + ); + assert_eq!(checksum_for(b"bad other.tar.gz\n", archive), None); + } + + #[test] + fn resolved_channels_replace_the_exact_target_and_skip_only_the_same_target() { + let target = MachineVersion::parse("1.2.3").unwrap(); + assert!(!replacement_required( + &ReleaseSource::Published, + Some(&target), + &target + )); + assert!(replacement_required( + &ReleaseSource::Published, + Some(&MachineVersion::parse("1.2.4").unwrap()), + &target + )); + assert!(replacement_required( + &ReleaseSource::Local(PathBuf::from("qualification")), + Some(&target), + &target + )); + } + + #[test] + fn staging_directory_is_private_and_removed_on_drop() { + let parent = tempfile::tempdir().unwrap(); + let path = { + let staging = staging_directory(parent.path()).unwrap(); + assert_eq!( + staging.path().metadata().unwrap().permissions().mode() & 0o777, + 0o700 + ); + staging.path().to_owned() + }; + assert!(!path.exists()); + } +} diff --git a/crates/ployzd/src/installer/storage.rs b/crates/ployzd/src/installer/storage.rs new file mode 100644 index 0000000000..8ec86822ff --- /dev/null +++ b/crates/ployzd/src/installer/storage.rs @@ -0,0 +1,469 @@ +//! Optional ZFS preparation for a fresh Machine. + +use std::{ + ffi::OsStr, + fs::{self, OpenOptions}, + io::Write, + os::unix::fs::MetadataExt, + path::Path, + process::Command, +}; + +use ployz_core::StorageChoice; + +use super::{Error, InstallPaths, command_exists, run_apt, run_command, run_host}; +use super::{ + host::write_file_atomically, + release::{staging_directory, write_private}, +}; + +const ZFS_SMOKE_BYTES: u64 = 128 * 1024 * 1024; +const POSIX_STAT_BLOCK_BYTES: u64 = 512; + +pub(super) fn prepare_storage(storage: StorageChoice, paths: &InstallPaths) -> Result<(), Error> { + match storage { + StorageChoice::None => Ok(()), + StorageChoice::Zfs => prepare_zfs(paths), + } +} + +fn prepare_zfs(paths: &InstallPaths) -> Result<(), Error> { + let os = operating_system_id()?; + if os != "ubuntu" { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "ZFS storage preparation is not supported on {os} yet; use a supported Ubuntu release" + ), + }); + } + let container = container_virtualization(); + if container == "openvz" || (Path::new("/proc/vz").is_dir() && !Path::new("/proc/bc").is_dir()) + { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: "OpenVZ does not allow this Machine to load the host ZFS kernel module".into(), + }); + } + if container == "lxc" && lxc_is_unprivileged()? { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: + "Unprivileged LXC does not allow this Machine to load the host ZFS kernel module" + .into(), + }); + } + if !command_exists("apt-get") { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: "Ubuntu apt-get is required for ZFS storage preparation".into(), + }); + } + let kernel = uname("-r", "read running kernel")?; + require_host_root_reserve(ZFS_SMOKE_BYTES)?; + let cap = zfs_arc_max()?; + persist_zfs_arc_max(paths, cap)?; + install_zfs_packages(&kernel)?; + run_host("load ZFS kernel module", "modprobe", ["zfs"])?; + set_and_verify_zfs_arc_max(cap)?; + validate_zfs()?; + println!("ZFS storage preparation validated; no Machine Pool was created"); + Ok(()) +} + +fn operating_system_id() -> Result { + let value = fs::read_to_string("/etc/os-release").map_err(|source| Error::Io { + stage: "identify Linux distribution for ZFS storage preparation", + source, + })?; + value + .lines() + .find_map(|line| line.strip_prefix("ID=")) + .map(|id| id.trim_matches('"').to_owned()) + .filter(|id| !id.is_empty()) + .ok_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: "Could not identify the Linux distribution for ZFS storage preparation".into(), + }) +} + +fn container_virtualization() -> String { + Command::new("systemd-detect-virt") + .arg("--container") + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned()) + .unwrap_or_default() +} + +fn lxc_is_unprivileged() -> Result { + let map = fs::read_to_string("/proc/self/uid_map").map_err(|source| Error::Io { + stage: "inspect LXC user namespace", + source, + })?; + let mut fields = map.split_whitespace(); + Ok(matches!( + (fields.next(), fields.next()), + (Some("0"), Some(outside)) if outside != "0" + )) +} + +fn uname(arg: &str, stage: &str) -> Result { + let output = run_host(stage, "uname", [arg])?; + let value = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if value.is_empty() { + Err(Error::Command { + stage: stage.into(), + message: "returned no value".into(), + }) + } else { + Ok(value) + } +} + +fn require_host_root_reserve(allocation: u64) -> Result<(), Error> { + let (size, available) = + crate::host_capacity::filesystem_space(Path::new("/")).map_err(|source| Error::Io { + stage: "inspect host-root capacity for ZFS storage preparation", + source, + })?; + let reserve = ployz_core::storage_host_reserve(size); + if available < reserve.saturating_add(allocation) { + return Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Host root has {available} bytes available; ZFS validation needs {allocation} bytes while preserving the {reserve}-byte host-root reserve" + ), + }); + } + Ok(()) +} + +fn zfs_arc_max() -> Result { + let memory = fs::read_to_string("/proc/meminfo").map_err(|source| Error::Io { + stage: "read total RAM for ZFS ARC limit", + source, + })?; + let kib = memory + .lines() + .find_map(|line| { + line.strip_prefix("MemTotal:") + .and_then(|value| value.split_whitespace().next()) + }) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: "Could not read total RAM for the ZFS ARC limit".into(), + })?; + Ok((kib.saturating_mul(1024) / 4).clamp(256 * 1024 * 1024, 1024 * 1024 * 1024)) +} + +fn persist_zfs_arc_max(paths: &InstallPaths, cap: u64) -> Result<(), Error> { + fs::create_dir_all(&paths.modprobe_dir).map_err(|source| Error::Io { + stage: "create ZFS module configuration directory", + source, + })?; + write_file_atomically( + &paths.modprobe_dir.join("ployz-zfs.conf"), + &format!("options zfs zfs_arc_max={cap}\n"), + "persist ZFS ARC limit", + ) +} + +fn package_has_zfs_module(files: &str, kernel: &str) -> bool { + let marker = format!("/lib/modules/{kernel}/"); + files.lines().any(|line| { + line.contains(&marker) && (line.ends_with("/zfs.ko") || line.contains("/zfs.ko.")) + }) +} + +fn installed_package_files(candidate: &str) -> Result, Error> { + let output = Command::new("dpkg-query") + .args(["-L", candidate]) + .output() + .map_err(|source| Error::Io { + stage: "inspect installed Ubuntu ZFS module package", + source, + })?; + Ok(output + .status + .success() + .then(|| String::from_utf8_lossy(&output.stdout).into_owned())) +} + +pub(super) fn install_zfs_packages(kernel: &str) -> Result<(), Error> { + run_apt("refresh Ubuntu packages for ZFS", ["update", "-qq"], None)?; + let mut package = None; + let mut download_error = None; + for candidate in [ + format!("linux-main-modules-zfs-{kernel}"), + format!("linux-modules-zfs-{kernel}"), + format!("linux-modules-{kernel}"), + format!("linux-modules-extra-{kernel}"), + ] { + let mut show = Command::new("apt-cache"); + show.args(["show", &candidate]); + if !show + .output() + .map_err(|source| Error::Io { + stage: "inspect Ubuntu ZFS module packages", + source, + })? + .status + .success() + { + continue; + } + if installed_package_files(&candidate)? + .is_some_and(|files| package_has_zfs_module(&files, kernel)) + { + package = Some(candidate); + break; + } + let scratch = staging_directory(Path::new("/var/tmp"))?; + if let Err(error) = run_apt( + "download Ubuntu ZFS module package", + ["download", &candidate], + Some(scratch.path()), + ) { + download_error = Some(error); + continue; + } + let archive = fs::read_dir(scratch.path()) + .map_err(|source| Error::Io { + stage: "inspect downloaded Ubuntu ZFS module package", + source, + })? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .find(|path| path.extension() == Some(OsStr::new("deb"))); + let Some(archive) = archive else { + return Err(Error::Verification(format!( + "downloaded Ubuntu ZFS module package {candidate} contained no Debian archive" + ))); + }; + let mut contents = Command::new("dpkg-deb"); + contents.args(["-c"]).arg(&archive); + let output = run_command( + "inspect downloaded Ubuntu ZFS module package", + &mut contents, + )?; + if package_has_zfs_module(&String::from_utf8_lossy(&output.stdout), kernel) { + package = Some(candidate); + break; + } + } + let package = package.ok_or_else(|| { + download_error.unwrap_or_else(|| Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Ubuntu has no packaged ZFS module for the running kernel {kernel}; install a supported Ubuntu kernel and retry" + ), + }) + })?; + run_apt( + "install ZFS packages", + [ + "install", + "-y", + "-qq", + "--no-install-recommends", + "zfsutils-linux", + &package, + ], + None, + )?; + let files = command_stdout( + "verify installed Ubuntu ZFS module package", + "dpkg-query", + ["-L", &package], + )?; + if package_has_zfs_module(&files, kernel) { + Ok(()) + } else { + Err(Error::Command { + stage: "prepare ZFS storage".into(), + message: format!( + "Installed package {package} does not supply the ZFS module for running kernel {kernel}" + ), + }) + } +} + +fn command_stdout( + stage: &str, + program: &str, + args: [&str; N], +) -> Result { + let output = run_host(stage, program, args)?; + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +fn set_and_verify_zfs_arc_max(cap: u64) -> Result<(), Error> { + let path = Path::new("/sys/module/zfs/parameters/zfs_arc_max"); + let mut file = OpenOptions::new() + .write(true) + .open(path) + .map_err(|source| Error::Io { + stage: "apply ZFS ARC limit", + source, + })?; + file.write_all(cap.to_string().as_bytes()) + .map_err(|source| Error::Io { + stage: "apply ZFS ARC limit", + source, + })?; + let observed = fs::read_to_string(path).map_err(|source| Error::Io { + stage: "verify ZFS ARC limit", + source, + })?; + if observed.trim() == cap.to_string() { + Ok(()) + } else { + Err(Error::Verification(format!( + "loaded ZFS module reports zfs_arc_max={} instead of the required {cap}", + observed.trim() + ))) + } +} + +fn validate_zfs() -> Result<(), Error> { + require_host_root_reserve(ZFS_SMOKE_BYTES)?; + let stage = staging_directory(Path::new("/var/tmp"))?; + let backing = stage.path().join("backing"); + write_private(&backing, b"", "create ZFS smoke backing file")?; + let mut allocate = Command::new("fallocate"); + allocate + .args(["-l", &ZFS_SMOKE_BYTES.to_string()]) + .arg(&backing); + run_command("preallocate ZFS smoke backing file", &mut allocate)?; + let metadata = backing.metadata().map_err(|source| Error::Io { + stage: "inspect ZFS smoke backing file", + source, + })?; + if metadata.blocks().saturating_mul(POSIX_STAT_BLOCK_BYTES) < ZFS_SMOKE_BYTES { + return Err(Error::Verification(format!( + "ZFS smoke backing file {} is sparse", + backing.display() + ))); + } + let root = fs::metadata("/").map_err(|source| Error::Io { + stage: "inspect host-root filesystem", + source, + })?; + if metadata.dev() != root.dev() { + return Err(Error::Verification(format!( + "ZFS smoke backing file {} is not on the host root filesystem", + backing.display() + ))); + } + validate_zfs_pool(stage) +} + +fn validate_zfs_pool(stage: tempfile::TempDir) -> Result<(), Error> { + let pool = format!( + "ployz-smoke-{}", + stage + .path() + .file_name() + .expect("temporary directory name") + .to_string_lossy() + ); + // ZFS may own this backing file as soon as creation is attempted. A persistent path + // cannot unlink it on error or unwind; only positively verified cleanup may delete it. + let directory = stage.keep(); + let backing = directory.join("backing"); + let mut pool_created = false; + let result = (|| { + let mut create = Command::new("zpool"); + create + .args(["create", "-f", "-m", "none", "-o", "cachefile=none", &pool]) + .arg(&backing); + run_command("create temporary ZFS smoke Pool", &mut create)?; + pool_created = true; + run_host( + "query temporary ZFS smoke Pool", + "zpool", + ["list", "-Hp", "-o", "name,size,alloc,free", &pool], + )?; + run_host( + "query temporary ZFS smoke dataset", + "zfs", + ["list", "-Hp", "-o", "name,mountpoint", &pool], + )?; + Ok(()) + })(); + let cleanup = cleanup_zfs_smoke(&pool, &backing, pool_created) + .and_then(|()| { + fs::remove_dir(&directory).map_err(|source| Error::Io { + stage: "remove temporary ZFS smoke directory", + source, + }) + }) + .map_err(|error| { + Error::Verification(format!( + "{error}; ZFS smoke backing directory retained at {} for inspection", + directory.display() + )) + }); + match (result, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) => Err(error), + (Ok(()), Err(error)) => Err(error), + (Err(error), Err(cleanup)) => { + Err(Error::Verification(format!("{cleanup} (after: {error})"))) + } + } +} + +#[cfg(test)] +mod tests; + +fn cleanup_zfs_smoke(pool: &str, backing: &Path, pool_created: bool) -> Result<(), Error> { + let existing_pool = if pool_created { + true + } else { + command_stdout( + "inspect temporary ZFS smoke Pool after failed creation", + "zpool", + ["list", "-H", "-o", "name"], + )? + .lines() + .any(|name| name == pool) + }; + if existing_pool { + let mut destroy = Command::new("zpool"); + destroy.args(["destroy", "-f", pool]); + run_command("destroy temporary ZFS smoke Pool", &mut destroy)?; + } + let pools = command_stdout( + "verify temporary ZFS smoke Pool cleanup", + "zpool", + ["list", "-H", "-o", "name"], + )?; + if pools.lines().any(|name| name == pool) { + return Err(Error::Verification(format!( + "Temporary ZFS smoke Pool {pool} remains after destroy" + ))); + } + let datasets = command_stdout( + "verify temporary ZFS smoke dataset cleanup", + "zfs", + ["list", "-H", "-o", "name"], + )?; + if datasets.lines().any(|name| { + name == pool + || name + .strip_prefix(pool) + .is_some_and(|suffix| suffix.starts_with('/')) + }) { + return Err(Error::Verification(format!( + "Temporary ZFS smoke dataset {pool} remains after destroy" + ))); + } + fs::remove_file(backing).map_err(|source| Error::Io { + stage: "remove temporary ZFS smoke backing file", + source, + }) +} diff --git a/crates/ployzd/src/installer/storage/tests.rs b/crates/ployzd/src/installer/storage/tests.rs new file mode 100644 index 0000000000..25447358e2 --- /dev/null +++ b/crates/ployzd/src/installer/storage/tests.rs @@ -0,0 +1,102 @@ +//! Smoke Pool lifecycle contracts with isolated ZFS command adapters. + +use super::*; +use std::{env, os::unix::fs::PermissionsExt}; + +#[test] +fn smoke_pool_cleanup_preserves_backing_until_absence_is_proven() { + const CASE: &str = "PLOYZ_SMOKE_CLEANUP_CASE"; + const ROOT: &str = "PLOYZ_SMOKE_CLEANUP_ROOT"; + if let Ok(case) = env::var(CASE) { + let root = std::path::PathBuf::from(env::var_os(ROOT).unwrap()); + let stage = tempfile::tempdir_in(&root).unwrap(); + let directory = stage.path().to_owned(); + let backing = directory.join("backing"); + fs::write(&backing, b"pool data").unwrap(); + let result = validate_zfs_pool(stage); + if case == "success" || case == "query-failed" { + assert!(!directory.exists()); + assert_eq!(result.is_ok(), case == "success"); + } else { + assert!( + result + .unwrap_err() + .to_string() + .contains(&directory.display().to_string()) + ); + assert_eq!(fs::read(&backing).unwrap(), b"pool data", "{case}"); + } + fs::write(root.join("completed"), case).unwrap(); + return; + } + + for case in [ + "success", + "query-failed", + "destroy-failed", + "pool-inspection-failed", + "pool-remains", + "dataset-inspection-failed", + "dataset-remains", + "create-unknown", + ] { + let root = tempfile::tempdir().unwrap(); + let commands = root.path().join("commands"); + fs::create_dir(&commands).unwrap(); + for (name, body) in [ + ( + "zpool", + r#" +case "$1" in + create) + for arg do previous=${last-}; last=$arg; done + echo "$previous" > "$PLOYZ_SMOKE_CLEANUP_ROOT/pool" + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != create-unknown ] ;; + destroy) [ "$PLOYZ_SMOKE_CLEANUP_CASE" != destroy-failed ] ;; + list) + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != create-unknown ] || exit 1 + if [ "$2" = -Hp ]; then + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != query-failed ] + else + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != pool-inspection-failed ] || exit 1 + if [ "$PLOYZ_SMOKE_CLEANUP_CASE" = pool-remains ]; then + /bin/cat "$PLOYZ_SMOKE_CLEANUP_ROOT/pool" + fi + fi ;; +esac +"#, + ), + ( + "zfs", + r#" +if [ "$2" = -H ]; then + [ "$PLOYZ_SMOKE_CLEANUP_CASE" != dataset-inspection-failed ] || exit 1 + if [ "$PLOYZ_SMOKE_CLEANUP_CASE" = dataset-remains ]; then + /bin/cat "$PLOYZ_SMOKE_CLEANUP_ROOT/pool" + fi +fi +"#, + ), + ] { + let path = commands.join(name); + fs::write(&path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + let output = Command::new(env::current_exe().unwrap()) + .args(["--exact", "installer::storage::tests::smoke_pool_cleanup_preserves_backing_until_absence_is_proven", "--nocapture"]) + .env(CASE, case) + .env(ROOT, root.path()) + .env("PATH", commands) + .output() + .unwrap(); + assert!( + output.status.success(), + "{case}: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::read_to_string(root.path().join("completed")).unwrap(), + case + ); + } +} diff --git a/crates/ployzd/src/installer/upgrade.rs b/crates/ployzd/src/installer/upgrade.rs new file mode 100644 index 0000000000..376560ddcd --- /dev/null +++ b/crates/ployzd/src/installer/upgrade.rs @@ -0,0 +1,777 @@ +//! Durable, bounded Machine upgrade attempts run by transient systemd services. + +use std::{ + env, fs, io, + path::{Path, PathBuf}, + time::Duration, +}; + +use ployz_core::{ + MachineRelease, MachineUpgradeAttempt, MachineUpgradeAttemptId, MachineUpgradeOutcome, + MachineUpgradeStage, RequestMachineUpgradeRequest, +}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::{process::Command, time::timeout}; + +use super::{ + Error as InstallError, InstallMode, InstallPaths, InstallRequest, ReleaseRequest, ReleaseSource, +}; +use crate::mutation; + +const RECEIPT_FILE: &str = "upgrade-attempt.json"; +const QUALIFICATION_RELEASE_DIR: &str = "PLOYZ_UPGRADE_RELEASE_DIR"; +const WORKER_RUNTIME: &str = "15min"; +const LAUNCH_TIMEOUT: Duration = Duration::from_secs(20); + +#[derive(Debug, Error)] +pub enum Error { + /// No receipt exists for the requested attempt. + #[error("no Machine upgrade attempt was found")] + NotFound, + /// A retry identity was reused with another release selector. + #[error("Machine upgrade attempt {0} was reused with a different release")] + AttemptConflict(MachineUpgradeAttemptId), + /// Another installation or local mutation currently owns the Machine. + #[error("a Machine upgrade or mutation is active")] + Busy, + /// The durable receipt could not be read. + #[error("read Machine upgrade receipt: {0}")] + Read(#[source] io::Error), + /// The durable receipt was not valid JSON for the current contract. + #[error("decode Machine upgrade receipt: {0}")] + Decode(#[source] serde_json::Error), + /// The durable receipt could not be atomically written. + #[error("write Machine upgrade receipt: {0}")] + Write(#[source] io::Error), + /// The current receipt could not be encoded. + #[error("encode Machine upgrade receipt: {0}")] + Encode(#[source] serde_json::Error), + /// The process-local qualification source was not a trusted absolute directory. + #[error("invalid qualification release directory: {0}")] + QualificationSource(String), + /// Global activation was requested from a daemon using unsupported Machine paths. + #[error("{0}")] + NonstandardPaths(String), + /// The requested release could not be parsed or resolved. + #[error("resolve Machine release: {0}")] + Resolve(#[source] InstallError), + /// The system manager rejected or did not confirm worker launch. + #[error("launch Machine upgrade worker: {0}")] + Launch(String), + /// The worker process or unit could not be queried. + #[error("inspect Machine upgrade worker: {0}")] + InspectWorker(#[source] io::Error), + /// `systemctl` returned evidence that did not prove either a running or stopped worker. + #[error("inspect Machine upgrade worker: {0}")] + WorkerEvidence(String), + /// The worker identity does not match the current nonterminal receipt. + #[error("Machine upgrade worker does not own active attempt {0}")] + NotActive(MachineUpgradeAttemptId), + /// The shared installer recorded a terminal failure. + #[error("Machine upgrade failed: {0}")] + Installation(#[source] InstallError), + /// Machine mutation ownership could not be claimed or inspected. + #[error(transparent)] + Admission(#[from] mutation::Error), +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +struct StoredAttempt { + requested: MachineRelease, + source: ReleaseSource, + attempt: MachineUpgradeAttempt, +} + +fn current_source() -> Result { + let Some(directory) = env::var_os(QUALIFICATION_RELEASE_DIR) else { + return Ok(ReleaseSource::Published); + }; + let directory = PathBuf::from(directory); + if !directory.is_absolute() { + return Err(Error::QualificationSource(format!( + "{QUALIFICATION_RELEASE_DIR} must be an absolute path" + ))); + } + Ok(ReleaseSource::Local(directory)) +} + +/// Return the durable result of retrying the same request without requiring installation +/// ownership. The receipt is atomically replaced, so an active worker can be observed safely. +pub(crate) fn existing_request( + request: &RequestMachineUpgradeRequest, + data_dir: &Path, +) -> Result, Error> { + let Some(stored) = read_optional(data_dir)? else { + return Ok(None); + }; + if stored.attempt.attempt_id != request.attempt_id { + return Ok(None); + } + if stored.requested != request.release { + return Err(Error::AttemptConflict(request.attempt_id)); + } + Ok(Some(stored.attempt)) +} + +/// Own acceptance and worker launch independently of the requesting client's lifetime. +/// Dropping the returned future only stops waiting; the task retains exclusive admission. +pub(crate) fn request( + request: RequestMachineUpgradeRequest, + data_dir: PathBuf, + run_dir: PathBuf, + guard: mutation::InstallationGuard, +) -> impl Future> { + let task = tokio::spawn(async move { + let _guard = guard; + request_locked(request, &data_dir, &run_dir).await + }); + async move { + task.await + .map_err(|error| Error::Launch(format!("upgrade acceptance task failed: {error}")))? + } +} + +async fn request_locked( + request: RequestMachineUpgradeRequest, + data_dir: &Path, + run_dir: &Path, +) -> Result { + let admission = mutation::MutationGate::new(run_dir, data_dir); + if let Some(mut stored) = read_optional(data_dir)? { + if stored.attempt.attempt_id == request.attempt_id { + if stored.requested != request.release { + return Err(Error::AttemptConflict(request.attempt_id)); + } + reconcile_locked(&admission, data_dir, &mut stored).await?; + return Ok(stored.attempt); + } + reconcile_locked(&admission, data_dir, &mut stored).await?; + if !stored.attempt.is_terminal() { + return Err(Error::Busy); + } + } + + if admission.active()? { + return Err(Error::Busy); + } + let source = current_source()?; + let release = request + .release + .as_str() + .parse::() + .map_err(Error::Resolve)?; + let target = super::release::resolve_release(&release, &source) + .await + .map_err(Error::Resolve)?; + let mut stored = StoredAttempt { + requested: request.release, + source, + attempt: MachineUpgradeAttempt { + attempt_id: request.attempt_id, + target, + outcome: MachineUpgradeOutcome::Accepted, + }, + }; + write(data_dir, &stored)?; + admission.mark_active(request.attempt_id.as_str())?; + + if let Err(error) = launch_worker(request.attempt_id, data_dir, run_dir).await { + stored.attempt.outcome = MachineUpgradeOutcome::Failed { + stage: MachineUpgradeStage::Launching, + error: error.to_string(), + }; + write(data_dir, &stored)?; + admission.clear_active(request.attempt_id.as_str())?; + } + Ok(stored.attempt) +} + +/// Read an attempt, turning a stopped worker without terminal evidence into interruption. +pub(crate) async fn inspect( + attempt_id: Option, + data_dir: &Path, + run_dir: &Path, +) -> Result { + let admission = mutation::MutationGate::new(run_dir, data_dir); + let mut stored = read(data_dir)?; + if attempt_id.is_some_and(|attempt_id| attempt_id != stored.attempt.attempt_id) { + return Err(Error::NotFound); + } + if stored.attempt.is_terminal() { + match admission.try_installation() { + Ok(_guard) => admission.clear_active(stored.attempt.attempt_id.as_str())?, + Err(mutation::Error::Busy) => {} + Err(error) => return Err(error.into()), + } + return Ok(stored.attempt); + } + if worker_state(stored.attempt.attempt_id).await? == WorkerState::Active { + return Ok(stored.attempt); + } + let _guard = match admission.try_installation() { + Ok(guard) => guard, + Err(mutation::Error::Busy) => return Ok(stored.attempt), + Err(error) => return Err(error.into()), + }; + stored = read(data_dir)?; + reconcile_locked(&admission, data_dir, &mut stored).await?; + Ok(stored.attempt) +} + +/// Run the accepted attempt from the transient systemd service. +/// +/// # Errors +/// +/// Returns an ownership, receipt, installation, or durable result error when the worker cannot +/// complete and record the accepted attempt. +pub async fn run_worker( + attempt_id: MachineUpgradeAttemptId, + data_dir: &Path, + run_dir: &Path, +) -> Result<(), Error> { + super::require_standard_machine_paths(data_dir, &run_dir.join("ployz.sock")) + .map_err(Error::NonstandardPaths)?; + let admission = mutation::MutationGate::new(run_dir, data_dir); + let guard = admission.lock_installation()?; + let mut stored = read(data_dir)?; + if stored.attempt.attempt_id != attempt_id || stored.attempt.is_terminal() { + return Err(Error::NotActive(attempt_id)); + } + let target = stored.attempt.target.clone(); + let mut stage = MachineUpgradeStage::Preparing; + stored.attempt.outcome = MachineUpgradeOutcome::Running { + stage: stage.clone(), + }; + write(data_dir, &stored)?; + + let result = super::install_locked( + InstallRequest { + release: ReleaseRequest::Exact(target.clone()), + source: stored.source.clone(), + mode: InstallMode::SoftwareOnly, + }, + InstallPaths::system(data_dir, run_dir), + guard, + |install_stage| { + stage = install_stage; + stored.attempt.outcome = MachineUpgradeOutcome::Running { + stage: stage.clone(), + }; + write(data_dir, &stored).map_err(|error| InstallError::Io { + stage: "record Machine upgrade progress", + source: io::Error::other(error), + }) + }, + ) + .await; + + match result { + Ok(_) => { + stored.attempt.outcome = MachineUpgradeOutcome::Succeeded { + version: target.clone(), + }; + write(data_dir, &stored)?; + admission.clear_active(attempt_id.as_str())?; + Ok(()) + } + Err(error) => { + stored.attempt.outcome = MachineUpgradeOutcome::Failed { + stage, + error: error.to_string(), + }; + write(data_dir, &stored)?; + admission.clear_active(attempt_id.as_str())?; + Err(Error::Installation(error)) + } + } +} + +/// Reconcile a retained receipt before serving requests after daemon restart. +pub(crate) async fn reconcile(data_dir: &Path, run_dir: &Path) -> Result<(), Error> { + match inspect(None, data_dir, run_dir).await { + Ok(_) | Err(Error::NotFound) => Ok(()), + Err(error) => Err(error), + } +} + +pub(super) async fn reconcile_for_install( + admission: &mutation::MutationGate, + data_dir: &Path, +) -> Result<(), Error> { + let Some(mut stored) = read_optional(data_dir)? else { + return if admission.active()? { + Err(Error::Busy) + } else { + Ok(()) + }; + }; + reconcile_locked(admission, data_dir, &mut stored).await?; + if stored.attempt.is_terminal() { + Ok(()) + } else { + Err(Error::Busy) + } +} + +async fn reconcile_locked( + admission: &mutation::MutationGate, + data_dir: &Path, + stored: &mut StoredAttempt, +) -> Result<(), Error> { + if stored.attempt.is_terminal() { + admission.clear_active(stored.attempt.attempt_id.as_str())?; + return Ok(()); + } + if worker_state(stored.attempt.attempt_id).await? == WorkerState::Active { + return Ok(()); + } + let stage = match &stored.attempt.outcome { + MachineUpgradeOutcome::Accepted => MachineUpgradeStage::Launching, + MachineUpgradeOutcome::Running { stage } => stage.clone(), + MachineUpgradeOutcome::Succeeded { .. } + | MachineUpgradeOutcome::Failed { .. } + | MachineUpgradeOutcome::Interrupted { .. } => return Ok(()), + }; + stored.attempt.outcome = MachineUpgradeOutcome::Interrupted { stage }; + write(data_dir, stored)?; + admission.clear_active(stored.attempt.attempt_id.as_str())?; + Ok(()) +} + +async fn launch_worker( + attempt_id: MachineUpgradeAttemptId, + data_dir: &Path, + run_dir: &Path, +) -> Result<(), Error> { + let executable = env::current_exe().map_err(Error::InspectWorker)?; + let unit = unit_name(attempt_id); + let writable = format!( + "ReadWritePaths=/usr/local/bin /etc/systemd/system {} {}", + data_dir.display(), + run_dir.display() + ); + let mut command = Command::new("systemd-run"); + command + .arg(format!("--unit={unit}")) + .args(["--collect", "--quiet"]) + .arg("--property=Type=exec") + .arg(format!("--property=RuntimeMaxSec={WORKER_RUNTIME}")) + .arg("--property=TimeoutStopSec=15s") + .arg("--property=KillMode=control-group") + .arg("--property=NoNewPrivileges=yes") + .arg("--property=ProtectSystem=full") + .arg("--property=ProtectHome=yes") + .arg("--property=ProtectControlGroups=yes") + .arg("--property=ProtectKernelTunables=yes") + .arg("--property=PrivateTmp=yes") + .arg("--property=RestrictNamespaces=yes") + .arg("--property=RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX") + .arg(format!("--property={writable}")) + .arg(executable) + .arg("--data-dir") + .arg(data_dir) + .arg("--socket") + .arg(run_dir.join("ployz.sock")) + .arg("upgrade-worker") + .arg("--attempt") + .arg(attempt_id.as_str()) + .kill_on_drop(true); + let output = timeout(LAUNCH_TIMEOUT, command.output()) + .await + .map_err(|_| { + Error::Launch(format!( + "systemd-run timed out after {}s", + LAUNCH_TIMEOUT.as_secs() + )) + })? + .map_err(Error::InspectWorker)?; + if output.status.success() { + Ok(()) + } else { + let detail = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + Err(Error::Launch(if detail.is_empty() { + format!("systemd-run exited with {}", output.status) + } else { + detail + })) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum WorkerState { + Active, + Stopped, +} + +async fn worker_state(attempt_id: MachineUpgradeAttemptId) -> Result { + let output = Command::new("systemctl") + .args([ + "show", + "--property=LoadState", + "--property=ActiveState", + &unit_name(attempt_id), + ]) + .output() + .await + .map_err(Error::InspectWorker)?; + if !output.status.success() { + let detail = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + return Err(Error::WorkerEvidence(if detail.is_empty() { + format!("systemctl show exited with {}", output.status) + } else { + format!("systemctl show exited with {}: {detail}", output.status) + })); + } + let properties = String::from_utf8(output.stdout) + .map_err(|_| Error::WorkerEvidence("systemctl show returned non-UTF-8 output".into()))?; + let mut load = None; + let mut active = None; + for line in properties.lines() { + if let Some(value) = line.strip_prefix("LoadState=") { + if load.replace(value).is_some() { + return Err(Error::WorkerEvidence( + "systemctl show repeated LoadState".into(), + )); + } + } else if let Some(value) = line.strip_prefix("ActiveState=") + && active.replace(value).is_some() + { + return Err(Error::WorkerEvidence( + "systemctl show repeated ActiveState".into(), + )); + } + } + if load == Some("not-found") { + return Ok(WorkerState::Stopped); + } + match active { + Some( + "active" | "activating" | "reloading" | "deactivating" | "maintenance" | "refreshing", + ) => Ok(WorkerState::Active), + Some("inactive" | "failed") => Ok(WorkerState::Stopped), + _ => Err(Error::WorkerEvidence(format!( + "systemctl show returned no recognized state: {}", + properties.trim().escape_debug() + ))), + } +} + +fn unit_name(attempt_id: MachineUpgradeAttemptId) -> String { + format!("ployz-upgrade-{attempt_id}.service") +} + +fn read(data_dir: &Path) -> Result { + read_optional(data_dir)?.ok_or(Error::NotFound) +} + +fn read_optional(data_dir: &Path) -> Result, Error> { + let bytes = match fs::read(data_dir.join(RECEIPT_FILE)) { + Ok(bytes) => bytes, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(Error::Read(error)), + }; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(Error::Decode) +} + +fn write(data_dir: &Path, stored: &StoredAttempt) -> Result<(), Error> { + fs::create_dir_all(data_dir).map_err(Error::Write)?; + let bytes = serde_json::to_vec(stored).map_err(Error::Encode)?; + crate::filesystem::atomic_write(&data_dir.join(RECEIPT_FILE), &bytes, 0o600) + .map_err(Error::Write) +} + +#[cfg(test)] +mod tests { + use super::*; + use ployz_core::MachineVersion; + + const CONTRACT_CASE: &str = "PLOYZ_UPGRADE_CONTRACT_CASE"; + const CONTRACT_ROOT: &str = "PLOYZ_UPGRADE_CONTRACT_ROOT"; + + #[tokio::test] + async fn worker_rejects_nonstandard_paths_before_reading_local_state() { + let root = tempfile::Builder::new() + .prefix("ployzd-upgrade-worker-paths-") + .tempdir() + .unwrap(); + let data_dir = root.path().join("data"); + let run_dir = root.path().join("run"); + + let error = run_worker(MachineUpgradeAttemptId::random(), &data_dir, &run_dir) + .await + .unwrap_err(); + + assert!(matches!(error, Error::NonstandardPaths(_))); + assert!(!data_dir.exists()); + assert!(!run_dir.exists()); + } + + #[test] + fn stored_attempt_round_trips_without_exposing_local_source_in_response() { + let attempt_id = MachineUpgradeAttemptId::random(); + let stored = StoredAttempt { + requested: MachineRelease::parse("beta").unwrap(), + source: ReleaseSource::Local("/root/qualification".into()), + attempt: MachineUpgradeAttempt { + attempt_id, + target: MachineVersion::parse("1.2.3-beta.4").unwrap(), + outcome: MachineUpgradeOutcome::Accepted, + }, + }; + let encoded = serde_json::to_string(&stored).unwrap(); + assert!(encoded.contains("/root/qualification")); + assert!( + !serde_json::to_string(&stored.attempt) + .unwrap() + .contains("qualification") + ); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + stored + ); + } + + #[test] + fn upgrade_attempt_contract() { + if let Ok(case) = env::var(CONTRACT_CASE) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(run_contract_case(&case)); + let root = PathBuf::from(env::var_os(CONTRACT_ROOT).unwrap()); + fs::write(root.join("child-completed"), case).unwrap(); + return; + } + + for case in [ + "retry-active", + "interrupted", + "inspection-unknown", + "launch-failed", + "cancelled-launch", + "cancelled-launch-failed", + ] { + let root = tempfile::Builder::new() + .prefix(&format!("ployzd-upgrade-{case}-")) + .tempdir() + .unwrap(); + let commands = root.path().join("commands"); + fs::create_dir(&commands).unwrap(); + write_script( + &commands.join("systemd-run"), + if case.starts_with("cancelled-launch") { + "echo started > \"$PLOYZ_UPGRADE_CONTRACT_ROOT/launch-started\"\nwhile [ ! -f \"$PLOYZ_UPGRADE_CONTRACT_ROOT/launch-release\" ]; do /bin/sleep 0.01; done\nprintf '%s\\n' \"$*\" >> \"$PLOYZ_UPGRADE_COMMAND_LOG\"\nif [ \"$PLOYZ_UPGRADE_CONTRACT_CASE\" = cancelled-launch-failed ]; then echo worker launch refused >&2; exit 1; fi" + } else if case == "launch-failed" { + "echo worker launch refused >&2; exit 1" + } else { + "printf '%s\\n' \"$*\" >> \"$PLOYZ_UPGRADE_COMMAND_LOG\"" + }, + ); + write_script( + &commands.join("systemctl"), + match case { + "interrupted" => "printf 'LoadState=loaded\\nActiveState=inactive\\n'", + "inspection-unknown" => "echo manager unavailable >&2; exit 1", + _ => "printf 'LoadState=loaded\\nActiveState=active\\n'", + }, + ); + fs::create_dir(root.path().join("release")).unwrap(); + let output = std::process::Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + "installer::upgrade::tests::upgrade_attempt_contract", + "--nocapture", + ]) + .env(CONTRACT_CASE, case) + .env(CONTRACT_ROOT, root.path()) + .env(QUALIFICATION_RELEASE_DIR, root.path().join("release")) + .env( + "PLOYZ_UPGRADE_COMMAND_LOG", + root.path().join("commands.log"), + ) + .env("PATH", &commands) + .output() + .unwrap(); + assert!( + output.status.success(), + "{case}: stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::read_to_string(root.path().join("child-completed")).unwrap(), + case + ); + } + } + + async fn run_contract_case(case: &str) { + let root = PathBuf::from(env::var_os(CONTRACT_ROOT).unwrap()); + let data = root.join("data"); + let run = root.join("run"); + let admission = mutation::MutationGate::new(&run, &data); + let attempt_id = MachineUpgradeAttemptId::parse("a".repeat(32)).unwrap(); + let request = RequestMachineUpgradeRequest { + attempt_id, + release: MachineRelease::parse("1.2.3").unwrap(), + }; + let guard = admission.try_installation().unwrap(); + if case.starts_with("cancelled-launch") { + let launch = tokio::spawn({ + let data = data.clone(); + let run = run.clone(); + let request = request.clone(); + async move { super::request(request, data, run, guard).await } + }); + timeout(Duration::from_secs(5), async { + while !root.join("launch-started").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("launch did not reach the cancellation window"); + launch.abort(); + assert!(launch.await.unwrap_err().is_cancelled()); + assert!(admission.active().unwrap()); + assert!(matches!( + admission.try_installation(), + Err(mutation::Error::Busy) + )); + fs::write(root.join("launch-release"), "continue").unwrap(); + timeout(Duration::from_secs(5), async { + while admission.try_installation().is_err() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("detached launch did not finish"); + let observed = existing_request(&request, &data).unwrap().unwrap(); + if case == "cancelled-launch-failed" { + assert!(matches!( + observed.outcome, + MachineUpgradeOutcome::Failed { + stage: MachineUpgradeStage::Launching, + .. + } + )); + assert!(!admission.active().unwrap()); + assert!(admission.try_mutation().is_ok()); + } else { + assert!(matches!(observed.outcome, MachineUpgradeOutcome::Accepted)); + assert!(admission.active().unwrap()); + } + assert_eq!( + fs::read_to_string(root.join("commands.log")) + .unwrap() + .lines() + .count(), + 1 + ); + return; + } + let accepted = super::request(request.clone(), data.clone(), run.clone(), guard) + .await + .unwrap(); + + match case { + "retry-active" => { + assert!(matches!(accepted.outcome, MachineUpgradeOutcome::Accepted)); + assert_eq!( + existing_request(&request, &data).unwrap(), + Some(accepted.clone()) + ); + assert!(matches!( + admission.try_mutation(), + Err(mutation::Error::Busy) + )); + let conflict = RequestMachineUpgradeRequest { + attempt_id, + release: MachineRelease::parse("1.2.4").unwrap(), + }; + assert!(matches!( + existing_request(&conflict, &data), + Err(Error::AttemptConflict(id)) if id == attempt_id + )); + + let guard = admission.try_installation().unwrap(); + assert_eq!( + super::request(request, data.clone(), run.clone(), guard) + .await + .unwrap(), + accepted + ); + let guard = admission.try_installation().unwrap(); + let other = RequestMachineUpgradeRequest { + attempt_id: MachineUpgradeAttemptId::parse("b".repeat(32)).unwrap(), + release: MachineRelease::parse("1.2.3").unwrap(), + }; + assert!(matches!( + super::request(other, data.clone(), run.clone(), guard).await, + Err(Error::Busy) + )); + let log = fs::read_to_string(root.join("commands.log")).unwrap(); + assert_eq!(log.lines().count(), 1, "{log}"); + for required in [ + "--property=Type=exec", + "--property=RuntimeMaxSec=15min", + "--property=NoNewPrivileges=yes", + "--property=ProtectSystem=full", + "upgrade-worker", + "--attempt", + attempt_id.as_str(), + ] { + assert!(log.contains(required), "missing {required}: {log}"); + } + } + "interrupted" => { + let observed = inspect(Some(attempt_id), &data, &run).await.unwrap(); + assert!(matches!( + observed.outcome, + MachineUpgradeOutcome::Interrupted { + stage: MachineUpgradeStage::Launching, + } + )); + assert!(!admission.active().unwrap()); + assert!(admission.try_mutation().is_ok()); + } + "inspection-unknown" => { + assert!(matches!( + inspect(Some(attempt_id), &data, &run).await, + Err(Error::WorkerEvidence(ref message)) + if message.contains("manager unavailable") + )); + assert!(admission.active().unwrap()); + assert!(matches!( + admission.try_mutation(), + Err(mutation::Error::Busy) + )); + } + "launch-failed" => { + assert!(matches!( + accepted.outcome, + MachineUpgradeOutcome::Failed { + stage: MachineUpgradeStage::Launching, + ref error, + } if error == "launch Machine upgrade worker: worker launch refused" + )); + assert!(!admission.active().unwrap()); + assert_eq!( + inspect(Some(attempt_id), &data, &run).await.unwrap(), + accepted + ); + } + other => panic!("unknown upgrade contract case {other}"), + } + } + + fn write_script(path: &Path, body: &str) { + use std::os::unix::fs::PermissionsExt; + + fs::write(path, format!("#!/bin/sh\nset -eu\n{body}\n")).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } +} diff --git a/crates/ployzd/src/lib.rs b/crates/ployzd/src/lib.rs index 17549da37e..2ccb11b8e3 100644 --- a/crates/ployzd/src/lib.rs +++ b/crates/ployzd/src/lib.rs @@ -24,11 +24,13 @@ pub(crate) mod filesystem; mod host_capacity; mod hosted_dns; pub(crate) mod ingress; +pub mod installer; pub mod logs; pub mod machine; pub mod machine_api; #[doc(hidden)] pub mod machine_pool; +pub mod mutation; pub mod network; pub mod relay; pub(crate) mod runtime_watch; diff --git a/crates/ployzd/src/machine/local_machine.rs b/crates/ployzd/src/machine/local_machine.rs index 6014870719..69066a2670 100644 --- a/crates/ployzd/src/machine/local_machine.rs +++ b/crates/ployzd/src/machine/local_machine.rs @@ -18,7 +18,7 @@ use ployz_core::{ synthesize_membership, }; use thiserror::Error; -use tokio::sync::watch; +use tokio::sync::{OwnedMutexGuard, watch}; use super::{FoundingCluster, LocalMachineRecord, LocalMachineStore, StoreError, local_runtime}; @@ -40,6 +40,7 @@ pub struct LocalMachine { } mod container; +mod upgrade; #[derive(Clone)] struct ClusterContext { @@ -120,6 +121,26 @@ pub enum Error { NotAllocator, #[error("this Machine is isolation-locked")] IsolationLocked, + #[error(transparent)] + Admission(#[from] crate::mutation::Error), + #[error(transparent)] + Upgrade(#[from] crate::installer::upgrade::Error), +} + +pub(crate) struct MutationAdmission { + _local: OwnedMutexGuard<()>, + _installation: crate::mutation::MutationGuard, +} + +impl MutationAdmission { + pub(crate) fn into_installation_guard(self) -> crate::mutation::MutationGuard { + let Self { + _local, + _installation, + } = self; + drop(_local); + _installation + } } impl LocalMachine { @@ -180,6 +201,46 @@ impl LocalMachine { self.store.lock().map_err(|_| Error::LockPoisoned) } + pub(crate) async fn admit_mutation(&self) -> Result { + let (local, installation) = { + let store = self.lock_store()?; + (store.admission_lock.clone(), store.mutation_gate.clone()) + }; + let local = local.lock_owned().await; + let installation = installation.try_mutation()?; + Ok(MutationAdmission { + _local: local, + _installation: installation, + }) + } + + async fn finish_mutation(&self, work: F) -> Result + where + T: Send + 'static, + F: std::future::Future> + Send + 'static, + { + let admission = self.admit_mutation().await?; + tokio::spawn(async move { + let _admission = admission; + work.await + }) + .await? + } + + #[cfg(test)] + pub(crate) async fn hold_mutation_for_test( + &self, + started: Arc, + release: Arc, + ) -> Result<(), Error> { + self.finish_mutation(async move { + started.notify_one(); + release.notified().await; + Ok(()) + }) + .await + } + pub(crate) fn replicated(&self) -> Result<&ReplicatedStore, Error> { self.cluster .as_ref() @@ -347,7 +408,7 @@ impl LocalMachine { /// /// Returns [`Error::LockPoisoned`] when the local record lock is poisoned /// and [`Error::Store`] when initialize is not legal in the current phase. - pub fn initialize(&self, request: InitializeRequest) -> Result { + fn initialize_admitted(&self, request: InitializeRequest) -> Result { let machine = self.lock_store()?.initialize( request.name, FoundingCluster { @@ -367,6 +428,18 @@ impl LocalMachine { Ok(Initialized { machine }) } + /// Initialize this Machine under local mutation admission. + /// + /// # Errors + /// + /// Returns when mutation admission is busy, the local task fails, or initialization is not + /// legal in the current persisted phase. + pub async fn initialize(&self, request: InitializeRequest) -> Result { + let local = self.clone(); + self.finish_mutation(async move { local.initialize_admitted(request) }) + .await + } + /// Assign a new Machine into the Cluster from this participating Machine /// when cluster KV names this Machine as Allocator. A replica row with the /// same public key and name is returned as-is, without publishing. Replay @@ -387,6 +460,12 @@ impl LocalMachine { /// another Machine or is missing, [`Error::Network`] when subnet allocation /// fails, and [`Error::Cluster`] when replicated I/O fails. pub async fn register(&self, request: RegisterRequest) -> Result { + let local = self.clone(); + self.finish_mutation(async move { local.register_admitted(request).await }) + .await + } + + async fn register_admitted(&self, request: RegisterRequest) -> Result { let me = { let record = self.record()?; if record.phase() != LocalMachinePhase::Participating { @@ -484,7 +563,6 @@ impl LocalMachine { advertised_endpoints: request.advertised_endpoints, runtime: request.runtime, }; - // TODO: cross-process registration stays unfenced and has no rollback. publication.publish(&assigned_machine).await?; assigned_machine } @@ -502,7 +580,7 @@ impl LocalMachine { /// /// Returns [`Error::LockPoisoned`] when the local record lock is poisoned /// and [`Error::Store`] when join is not legal in the current phase. - pub fn join(&self, request: JoinRequest) -> Result { + fn join_admitted(&self, request: JoinRequest) -> Result { let mut store = self.lock_store()?; store.join( request.registration.assigned_machine, @@ -525,6 +603,18 @@ impl LocalMachine { Ok(JoinAccepted {}) } + /// Persist a join assignment under local mutation admission and request restart. + /// + /// # Errors + /// + /// Returns when mutation admission is busy, the local task fails, or join is not legal in + /// the current persisted phase. + pub async fn join(&self, request: JoinRequest) -> Result { + let local = self.clone(); + self.finish_mutation(async move { local.join_admitted(request) }) + .await + } + /// Persist Cloud Pairing so this Machine can hold Relay Register, or clear /// it so Register is dropped. /// @@ -533,7 +623,7 @@ impl LocalMachine { /// Returns [`Error::NotParticipating`] when this Machine is not /// participating, [`Error::LockPoisoned`] when the local record lock is /// poisoned, and [`Error::Store`] when the record cannot be written. - pub fn set_cloud_pairing(&self, pairing: Option) -> Result<(), Error> { + fn set_cloud_pairing_admitted(&self, pairing: Option) -> Result<(), Error> { let mut store = self.lock_store()?; if store.record().phase() != LocalMachinePhase::Participating { return Err(Error::NotParticipating); @@ -542,6 +632,18 @@ impl LocalMachine { Ok(()) } + /// Persist or clear Cloud Pairing under local mutation admission. + /// + /// # Errors + /// + /// Returns when mutation admission is busy, the local task fails, this Machine is not + /// participating, or the local record cannot be persisted. + pub async fn set_cloud_pairing(&self, pairing: Option) -> Result<(), Error> { + let local = self.clone(); + self.finish_mutation(async move { local.set_cloud_pairing_admitted(pairing) }) + .await + } + /// Membership Observation of Machines visible from this participating Machine. /// /// # Errors @@ -583,6 +685,15 @@ impl LocalMachine { /// [`Error::Store`] when the update is not legal, and [`Error::Cluster`] /// when listing visible Machines fails. pub async fn update(&self, request: UpdateMachineRequest) -> Result { + let local = self.clone(); + self.finish_mutation(async move { local.update_admitted(request).await }) + .await + } + + async fn update_admitted( + &self, + request: UpdateMachineRequest, + ) -> Result { if request.update.is_empty() { return Err(Error::EmptyUpdate); } @@ -607,6 +718,15 @@ impl LocalMachine { pub async fn remove_peer( &self, request: RemoveMachineRequest, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { local.remove_peer_admitted(request).await }) + .await + } + + async fn remove_peer_admitted( + &self, + request: RemoveMachineRequest, ) -> Result { self.replicated()? .remove_machine(&request.machine_id) @@ -634,14 +754,9 @@ impl LocalMachine { &self, request: RemoveLocalMachineRequest, ) -> Result { - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let local = self.clone(); - tokio::spawn(async move { - let _guard = guard; - local.remove_local_admitted(request).await - }) - .await? + self.finish_mutation(async move { local.remove_local_admitted(request).await }) + .await } async fn remove_local_admitted( @@ -690,14 +805,9 @@ impl LocalMachine { /// [`Error::Store`] when reset is not legal in the current phase, and /// [`Error::OperationTask`] if the admitted operation task fails. pub async fn reset(&self) -> Result { - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let local = self.clone(); - tokio::spawn(async move { - let _guard = guard; - local.reset_admitted().await - }) - .await? + self.finish_mutation(async move { local.reset_admitted().await }) + .await } async fn reset_admitted(&self) -> Result { diff --git a/crates/ployzd/src/machine/local_machine/container.rs b/crates/ployzd/src/machine/local_machine/container.rs index d7d33c340b..0912d5ffd7 100644 --- a/crates/ployzd/src/machine/local_machine/container.rs +++ b/crates/ployzd/src/machine/local_machine/container.rs @@ -1,15 +1,17 @@ //! Machine-local container admission and creation. -use std::path::Path; +use std::{path::Path, sync::Arc}; use ployz_core::{ - ContainerCreated, ContainerKind, LocalMachinePhase, MachineStorageObservation, ProjectName, - ResolvedServiceSpec, + ContainerChanged, ContainerCreated, ContainerId, ContainerKind, CreateVolumeReport, + CreateVolumeRequest, DockerVolumeName, ImageIngestOpened, ImageIngestReason, ImagePulled, + LocalMachinePhase, MachineStorageObservation, ProjectName, PullImageFromMachineRequest, + ResolvedServiceSpec, VolumeRemoved, }; use super::super::ingress::admit_ingress_service; use super::{Error, LocalMachine}; -use crate::docker::{ContainerRequest, GlobalSlotConvergence, GlobalSlotRequest}; +use crate::docker::{ContainerRequest, GlobalSlotConvergence, GlobalSlotRequest, ImageIngest}; use crate::machine::{STORAGE_OBSERVATION_TIMEOUT, local_storage}; impl LocalMachine { @@ -23,12 +25,9 @@ impl LocalMachine { &self, specs: Vec, ) -> Result { - use ployz_core::{RawVolumeSource, RpcError}; - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; + use ployz_core::RawVolumeSource; let local = self.clone(); - tokio::spawn(async move { - let _guard = guard; + self.finish_mutation(async move { let containers = local.containers.as_ref().ok_or(Error::DockerUnavailable)?; let record = local.record()?; if !matches!( @@ -71,15 +70,10 @@ impl LocalMachine { crate::storage::plugin("Storage.Prepare", &requested).await?; containers .ensure_provisioned_volumes(&machine.id, &specs) - .await - .map_err(|error| { - let mut error = RpcError::from(&error); - error.details = serde_json::json!({ "prepared_volumes": names }); - error - })?; + .await?; Ok(ployz_core::PreparedVolumes { names }) }) - .await? + .await } /// Create a container after storage admission and deferred Machine-local validation. @@ -95,14 +89,180 @@ impl LocalMachine { spec: &ResolvedServiceSpec, ) -> Result { // Once admitted, caller cancellation must not let reset overtake a Docker request. - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let (local, project, spec) = (self.clone(), project.clone(), spec.clone()); - tokio::spawn(async move { - let _guard = guard; - local.create_container_admitted(kind, &project, &spec).await + self.finish_mutation( + async move { local.create_container_admitted(kind, &project, &spec).await }, + ) + .await + } + + /// Start one local container while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot start the + /// container. + pub(crate) async fn start_container( + &self, + container_id: ContainerId, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .start(&container_id) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await + } + + /// Stop one local container while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot stop the + /// container. + pub(crate) async fn stop_container( + &self, + container_id: ContainerId, + signal: Option, + grace_period_seconds: Option, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .stop(&container_id, signal.as_deref(), grace_period_seconds) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await + } + + /// Remove one local container while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot remove the + /// container. + pub(crate) async fn remove_container( + &self, + container_id: ContainerId, + remove_volumes: bool, + force: bool, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .remove(&container_id, remove_volumes, force) + .await?; + Ok(ContainerChanged { container_id }) + }) + .await + } + + /// Create one local Docker Volume while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when local state or Docker is unavailable, admission is busy, or Docker cannot + /// create the Volume. + pub(crate) async fn create_volume( + &self, + request: CreateVolumeRequest, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + let machine_id = local.record()?.id(); + Ok(local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .create_volume(&machine_id, request) + .await?) + }) + .await + } + + /// Remove one local Docker Volume while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or Docker cannot remove the + /// Volume. + pub(crate) async fn remove_volume( + &self, + name: DockerVolumeName, + force: bool, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local + .containers + .as_ref() + .ok_or(Error::DockerUnavailable)? + .remove_volume(&name, force) + .await?; + Ok(VolumeRemoved {}) }) - .await? + .await + } + + /// Start or reconcile image ingest while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when this Machine is not participating, admission is busy, or the ingest helper + /// cannot start. + pub(crate) async fn ensure_image_ingest( + &self, + ingest: Arc, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + let record = local.record()?; + let address = record + .machine() + .filter(|_| record.phase() == LocalMachinePhase::Participating) + .map(|machine| machine.management_address()) + .ok_or_else(|| { + Error::StoragePreparation( + ImageIngestReason::NotParticipating + .rpc_error("Machine is not participating"), + ) + })?; + ingest + .open(address) + .await + .map_err(Error::StoragePreparation) + }) + .await + } + + /// Pull one image from peer ingest while holding Machine mutation admission. + /// + /// # Errors + /// + /// Returns when Docker is unavailable, admission is busy, or the image transfer fails. + pub(crate) async fn pull_image_from_machine( + &self, + request: PullImageFromMachineRequest, + ) -> Result { + let local = self.clone(); + self.finish_mutation(async move { + local.containers.as_ref().ok_or(Error::DockerUnavailable)?; + crate::docker::pull_from_ingest(&request.image, request.source).await?; + Ok(ImagePulled {}) + }) + .await } async fn create_container_admitted( @@ -145,14 +305,11 @@ impl LocalMachine { project: &ProjectName, spec: &ResolvedServiceSpec, ) -> Result { - let admission = self.lock_store()?.admission_lock.clone(); - let guard = admission.lock_owned().await; let (local, project, spec) = (self.clone(), project.clone(), spec.clone()); - tokio::spawn(async move { - let _guard = guard; - local.converge_global_slot_admitted(&project, &spec).await - }) - .await? + self.finish_mutation( + async move { local.converge_global_slot_admitted(&project, &spec).await }, + ) + .await } async fn converge_global_slot_admitted( diff --git a/crates/ployzd/src/machine/local_machine/upgrade.rs b/crates/ployzd/src/machine/local_machine/upgrade.rs new file mode 100644 index 0000000000..14541d2c9b --- /dev/null +++ b/crates/ployzd/src/machine/local_machine/upgrade.rs @@ -0,0 +1,66 @@ +//! Local ownership of Machine upgrade replay, admission, and durable inspection. + +use ployz_core::{ + InspectMachineUpgradeRequest, MachineUpgradeAttempt, RequestMachineUpgradeRequest, +}; + +use super::{Error, LocalMachine}; + +impl LocalMachine { + /// Resolve, durably accept, and launch one retry-safe Machine upgrade. + /// + /// A replay is read before claiming exclusive mutation ownership so a lost reply can be + /// recovered while the independent worker owns the installation lock. + /// + /// # Errors + /// + /// Returns [`Error::Upgrade`] for a conflicting retry identity, active mutation, release + /// failure, receipt failure, or worker launch uncertainty. Returns [`Error::LockPoisoned`] + /// when the Local Machine store cannot be read. + pub(crate) async fn request_upgrade( + &self, + request: RequestMachineUpgradeRequest, + ) -> Result { + let (data_dir, run_dir, local, gate) = { + let store = self.lock_store()?; + ( + store.data_dir.clone(), + store.run_dir.clone(), + store.admission_lock.clone(), + store.mutation_gate.clone(), + ) + }; + if let Some(attempt) = crate::installer::upgrade::existing_request(&request, &data_dir)? { + return Ok(attempt); + } + let _local = local + .try_lock_owned() + .map_err(|_| crate::mutation::Error::Busy)?; + let installation = gate.try_installation()?; + crate::installer::require_standard_machine_paths(&data_dir, &run_dir.join("ployz.sock")) + .map_err(crate::installer::upgrade::Error::NonstandardPaths)?; + crate::installer::upgrade::request(request, data_dir, run_dir, installation) + .await + .map_err(Into::into) + } + + /// Read one durable Machine upgrade attempt and reconcile positively stopped work. + /// + /// # Errors + /// + /// Returns [`Error::Upgrade`] when no matching receipt exists or worker and ownership + /// evidence cannot be read. Returns [`Error::LockPoisoned`] when the Local Machine store + /// cannot be read. + pub(crate) async fn inspect_upgrade( + &self, + request: InspectMachineUpgradeRequest, + ) -> Result { + let (data_dir, run_dir) = { + let store = self.lock_store()?; + (store.data_dir.clone(), store.run_dir.clone()) + }; + crate::installer::upgrade::inspect(request.attempt_id, &data_dir, &run_dir) + .await + .map_err(Into::into) + } +} diff --git a/crates/ployzd/src/machine/mod.rs b/crates/ployzd/src/machine/mod.rs index 5e5e018fa9..81e6aec8ee 100644 --- a/crates/ployzd/src/machine/mod.rs +++ b/crates/ployzd/src/machine/mod.rs @@ -2,13 +2,11 @@ use std::{ collections::BTreeMap, - fs::{self, File, OpenOptions}, - io::{self, Read, Seek, SeekFrom, Write}, - net::IpAddr, - os::unix::fs::{OpenOptionsExt, PermissionsExt}, - path::{Component, Path, PathBuf}, + fs::{self, OpenOptions}, + io::{self, Read, Seek, SeekFrom}, + os::unix::fs::OpenOptionsExt, + path::Path, process::{Command, Stdio}, - sync::Arc, thread, time::{Duration, Instant}, }; @@ -16,32 +14,31 @@ use std::{ use ipnet::Ipv4Net; use ployz_core::{ CloudPairing, LocalMachinePhase, Machine, MachineId, MachineRuntime, MachineStorageObservation, - MachineUpdate, MachineUpdateError, SelectedEndpoint, apply_machine_update, + SelectedEndpoint, }; use serde::{Deserialize, Serialize}; -use thiserror::Error; use crate::machine_pool; use crate::network::WireGuardPrivateKey; -use crate::network::allocate_machine_subnet; mod record_wire; use record_wire::LocalMachineRecordWire; mod ingress; mod local_machine; +mod store; + +#[cfg(test)] +use store::PENDING_RESET_FILE_NAME; +pub use store::{LocalMachineStore, StoreError}; -pub(crate) use local_machine::RuntimeWatchTelemetry; pub use local_machine::{Error as LocalMachineError, LocalMachine}; +pub(crate) use local_machine::{MutationAdmission, RuntimeWatchTelemetry}; #[cfg(test)] mod register_tests; pub const DEFAULT_DATA_DIR: &str = "/var/lib/ployz"; -const STATE_FILE_NAME: &str = "machine.json"; -const TEMPORARY_FILE_NAME: &str = ".machine.json.tmp"; -const PENDING_RESET_FILE_NAME: &str = ".machine.reset.pending"; -const LOCK_FILE_NAME: &str = ".ployzd.lock"; const DOCKER_VERSION_TIMEOUT: Duration = Duration::from_secs(2); const STORAGE_OBSERVATION_TIMEOUT: Duration = Duration::from_secs(2); @@ -355,312 +352,6 @@ impl LocalMachineRecord { } } -pub struct LocalMachineStore { - data_dir: PathBuf, - record: LocalMachineRecord, - _lock: File, - // Lock order: admission, then publication/ingress/Docker, then short store locks. - // Reset waits for admitted operations, including Docker streams with no total deadline. - // ponytail: serialize local creates; use shared admission reads if throughput requires it. - admission_lock: Arc>, -} - -pub(crate) struct PreparedReset { - data_dir: PathBuf, - original: LocalMachineRecord, - resetting: LocalMachineRecord, -} - -impl PreparedReset { - pub(crate) fn commit(self, store: &mut LocalMachineStore) -> Result<(), StoreError> { - let mut current = store.record.clone(); - current - .selected_endpoints - .clone_from(&self.original.selected_endpoints); - if store.data_dir != self.data_dir || current != self.original { - return Err(StoreError::ResetPreparationLost(store.data_dir.clone())); - } - fs::rename( - self.data_dir.join(PENDING_RESET_FILE_NAME), - self.data_dir.join(STATE_FILE_NAME), - )?; - store.record = self.resetting.clone(); - if let Err(error) = File::open(&self.data_dir).and_then(|directory| directory.sync_all()) { - eprintln!("failed to sync committed local Machine reset: {error}"); - } - Ok(()) - } -} - -impl Drop for PreparedReset { - fn drop(&mut self) { - let _ = fs::remove_file(self.data_dir.join(PENDING_RESET_FILE_NAME)); - } -} - -impl Drop for LocalMachineStore { - fn drop(&mut self) { - // A concurrently forked child can briefly inherit the flock before exec closes the fd. - let _ = fs2::FileExt::unlock(&self._lock); - } -} - -impl LocalMachineStore { - pub fn open(data_dir: impl AsRef) -> Result { - let data_dir = data_dir.as_ref().to_owned(); - validate_data_dir(&data_dir)?; - claim_data_dir(&data_dir)?; - fs::create_dir_all(&data_dir)?; - fs::set_permissions(&data_dir, fs::Permissions::from_mode(0o711))?; - let lock = OpenOptions::new() - .create(true) - .truncate(false) - .write(true) - .mode(0o600) - .open(data_dir.join(LOCK_FILE_NAME))?; - fs2::FileExt::try_lock_exclusive(&lock).map_err(|error| { - if error.kind() == io::ErrorKind::WouldBlock { - StoreError::AlreadyRunning(data_dir.clone()) - } else { - StoreError::Io(error) - } - })?; - let path = data_dir.join(STATE_FILE_NAME); - let record = match fs::read(&path) { - Ok(data) => serde_json::from_slice(&data)?, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - match fs::remove_file(data_dir.join(TEMPORARY_FILE_NAME)) { - Ok(()) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - let record = LocalMachineRecord { - body: LocalMachineBody::Uninitialized { - id: MachineId::random(), - }, - wireguard_private_key: WireGuardPrivateKey::generate(), - wireguard_mtu: None, - cloud_pairing: None, - selected_endpoints: BTreeMap::new(), - }; - save(&data_dir, &record)?; - record - } - Err(error) => return Err(error.into()), - }; - - let mut store = Self { - data_dir, - record, - _lock: lock, - admission_lock: Arc::new(tokio::sync::Mutex::new(())), - }; - if store.record.phase() == LocalMachinePhase::Resetting { - let data_dir = store.data_dir.clone(); - store.complete_reset()?; - drop(store); - return Self::open(data_dir); - } - store.refresh_runtime()?; - Ok(store) - } - - #[must_use] - pub fn record(&self) -> &LocalMachineRecord { - &self.record - } - - fn refresh_runtime(&mut self) -> Result<(), StoreError> { - let runtime = local_runtime(); - let stale = match &self.record.body { - LocalMachineBody::Joining { machine, .. } - | LocalMachineBody::Participating { machine, .. } => machine.runtime != runtime, - LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { - return Ok(()); - } - }; - if !stale { - return Ok(()); - } - let mut refreshed = self.record.clone(); - match &mut refreshed.body { - LocalMachineBody::Joining { machine, .. } - | LocalMachineBody::Participating { machine, .. } => machine.runtime = runtime, - LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { - return Ok(()); - } - } - save(&self.data_dir, &refreshed)?; - self.record = refreshed; - Ok(()) - } - - pub fn begin_reset(&mut self) -> Result<(), StoreError> { - let prepared = self.prepare_reset()?; - prepared.commit(self) - } - - pub(crate) fn prepare_reset(&self) -> Result { - if self.record.phase() == LocalMachinePhase::Resetting { - return Err(StoreError::AlreadyResetting); - } - let resetting = self.record.clone().into_resetting(); - write_record(&self.data_dir.join(PENDING_RESET_FILE_NAME), &resetting)?; - File::open(&self.data_dir)?.sync_all()?; - Ok(PreparedReset { - data_dir: self.data_dir.clone(), - original: self.record.clone(), - resetting, - }) - } - - pub fn initialize( - &mut self, - name: ployz_core::MachineName, - founding_cluster: FoundingCluster, - public_ip: Option, - advertised_endpoints: Vec, - wireguard_mtu: Option, - cloud_pairing: Option, - ) -> Result { - self.require_uninitialized()?; - if advertised_endpoints.is_empty() { - return Err(StoreError::MissingEndpoints); - } - let public_key = self.record.wireguard_private_key.public_key(); - let machine = Machine { - id: self.record.id(), - name, - subnet: allocate_machine_subnet(founding_cluster.network, []) - .map_err(|error| StoreError::InvalidNetwork(error.to_string()))?, - public_key, - public_ip, - advertised_endpoints, - runtime: local_runtime(), - }; - let mut initialized = self.record.clone(); - initialized.body = LocalMachineBody::Participating { - machine: machine.clone(), - origin: ParticipationOrigin::Founder { - cluster: founding_cluster, - }, - }; - initialized.wireguard_mtu = wireguard_mtu; - initialized.cloud_pairing = cloud_pairing; - save(&self.data_dir, &initialized)?; - self.record = initialized; - Ok(machine) - } - - pub fn join( - &mut self, - mut assigned_machine: Machine, - visible_peers: Vec, - target_versions: BTreeMap, - wireguard_mtu: Option, - cloud_pairing: Option, - ) -> Result<(), StoreError> { - self.require_uninitialized()?; - if visible_peers.is_empty() { - return Err(StoreError::MissingPeers); - } - if self.record.wireguard_private_key.public_key() != assigned_machine.public_key { - return Err(StoreError::KeyMismatch); - } - if assigned_machine.advertised_endpoints.is_empty() { - return Err(StoreError::MissingEndpoints); - } - assigned_machine.runtime = local_runtime(); - let mut joining = self.record.clone(); - joining.body = LocalMachineBody::Joining { - machine: assigned_machine, - bootstrap: visible_peers, - min_store_version: target_versions, - }; - joining.wireguard_mtu = wireguard_mtu; - joining.cloud_pairing = cloud_pairing; - save(&self.data_dir, &joining)?; - self.record = joining; - Ok(()) - } - - pub fn update( - &mut self, - update: MachineUpdate, - visible: &[Machine], - ) -> Result { - let mut record = self.record.clone(); - let LocalMachineBody::Participating { machine, .. } = &mut record.body else { - return Err(StoreError::NotParticipating); - }; - let updated = apply_machine_update(machine, visible, update)?; - *machine = updated.clone(); - save(&self.data_dir, &record)?; - self.record = record; - Ok(updated) - } - - fn require_uninitialized(&self) -> Result<(), StoreError> { - if matches!(self.record.body, LocalMachineBody::Uninitialized { .. }) { - Ok(()) - } else { - Err(StoreError::AlreadyInitialized) - } - } - - pub fn complete_catch_up(&mut self) -> Result<(), StoreError> { - let mut participating = self.record.clone(); - let LocalMachineBody::Joining { - machine, bootstrap, .. - } = participating.body - else { - return Err(StoreError::NotJoining); - }; - participating.body = LocalMachineBody::Participating { - machine, - origin: ParticipationOrigin::Join { bootstrap }, - }; - save(&self.data_dir, &participating)?; - self.record = participating; - Ok(()) - } - - pub fn persist_selected_endpoint( - &mut self, - machine_id: MachineId, - endpoint: SelectedEndpoint, - ) -> Result<(), StoreError> { - let mut updated = self.record.clone(); - updated.selected_endpoints.insert(machine_id, endpoint); - save(&self.data_dir, &updated)?; - self.record = updated; - Ok(()) - } - - pub fn persist_cloud_pairing( - &mut self, - pairing: Option, - ) -> Result<(), StoreError> { - let mut updated = self.record.clone(); - updated.cloud_pairing = pairing; - save(&self.data_dir, &updated)?; - self.record = updated; - Ok(()) - } - - pub fn complete_reset(&self) -> Result<(), StoreError> { - if self.record.phase() != LocalMachinePhase::Resetting { - return Err(StoreError::NotResetting); - } - let persisted: LocalMachineRecord = - serde_json::from_slice(&fs::read(self.data_dir.join(STATE_FILE_NAME))?)?; - if persisted != self.record { - return Err(StoreError::OwnershipLost(self.data_dir.clone())); - } - fs::remove_dir_all(&self.data_dir).map_err(StoreError::Io) - } -} - #[must_use] pub fn local_runtime() -> MachineRuntime { MachineRuntime { @@ -771,113 +462,10 @@ fn read_trimmed(path: &str) -> String { .unwrap_or_default() } -fn validate_data_dir(path: &Path) -> Result<(), StoreError> { - if path.file_name().is_none() - || path - .components() - .any(|component| component == Component::ParentDir) - { - return Err(StoreError::UnsafeDataDirectory(path.to_owned())); - } - Ok(()) -} - -fn claim_data_dir(data_dir: &Path) -> Result<(), StoreError> { - match fs::read_dir(data_dir) { - Ok(entries) => { - let names = entries - .map(|entry| Ok(entry?.file_name())) - .collect::>>()?; - let owns_directory = names.iter().any(|name| name == STATE_FILE_NAME) - || names - .iter() - .all(|name| name == TEMPORARY_FILE_NAME || name == LOCK_FILE_NAME); - if owns_directory { - Ok(()) - } else { - Err(StoreError::UnownedDataDirectory(data_dir.to_owned())) - } - } - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - } -} - -fn save(data_dir: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { - fs::create_dir_all(data_dir)?; - fs::set_permissions(data_dir, fs::Permissions::from_mode(0o711))?; - - let path = data_dir.join(STATE_FILE_NAME); - let temporary = data_dir.join(TEMPORARY_FILE_NAME); - write_record(&temporary, record)?; - fs::rename(temporary, path)?; - File::open(data_dir)?.sync_all()?; - Ok(()) -} - -fn write_record(path: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { - let mut file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .mode(0o600) - .open(path)?; - file.set_permissions(fs::Permissions::from_mode(0o600))?; - serde_json::to_writer_pretty(&mut file, record)?; - file.write_all(b"\n")?; - file.sync_all()?; - Ok(()) -} - -fn display_data_directory(path: &Path) -> String { - let mut text = String::new(); - for chunk in path.as_os_str().as_encoded_bytes().utf8_chunks() { - text.extend(chunk.valid().escape_debug()); - text.extend(chunk.invalid().escape_ascii().map(char::from)); - } - text -} - -#[derive(Debug, Error)] -pub enum StoreError { - #[error("local Machine record I/O failed: {0}")] - Io(#[from] io::Error), - #[error("local Machine record JSON is invalid: {0}")] - Json(#[from] serde_json::Error), - #[error("machine is not participating")] - NotParticipating, - #[error("machine is already resetting")] - AlreadyResetting, - #[error("machine is not resetting")] - NotResetting, - #[error("machine is not joining")] - NotJoining, - #[error("machine is already initialized")] - AlreadyInitialized, - #[error("at least one advertised endpoint is required")] - MissingEndpoints, - #[error("at least one visible peer is required to join")] - MissingPeers, - #[error("assigned public key does not match this Machine")] - KeyMismatch, - #[error("invalid Cluster network: {0}")] - InvalidNetwork(String), - #[error(transparent)] - MachineUpdate(#[from] MachineUpdateError), - #[error("another daemon already owns data directory {}", display_data_directory(.0))] - AlreadyRunning(PathBuf), - #[error("refusing to clear broad data directory {}", display_data_directory(.0))] - UnsafeDataDirectory(PathBuf), - #[error("refusing to claim nonempty data directory {}", display_data_directory(.0))] - UnownedDataDirectory(PathBuf), - #[error("local Machine record changed before clearing data directory {}", display_data_directory(.0))] - OwnershipLost(PathBuf), - #[error("local Machine record changed before prepared reset was committed in {}", display_data_directory(.0))] - ResetPreparationLost(PathBuf), -} - #[cfg(test)] mod tests { + use std::os::unix::fs::PermissionsExt; + use super::*; #[tokio::test] diff --git a/crates/ployzd/src/machine/register_tests/mod.rs b/crates/ployzd/src/machine/register_tests/mod.rs index c500150c05..f88ca34736 100644 --- a/crates/ployzd/src/machine/register_tests/mod.rs +++ b/crates/ployzd/src/machine/register_tests/mod.rs @@ -143,6 +143,7 @@ async fn register_rpc_exact_replay_returns_the_original_joinable_assignment() { wireguard_mtu: None, cloud_pairing: None, }) + .await .unwrap(); assert_eq!(target.record().unwrap().phase(), LocalMachinePhase::Joining); @@ -184,6 +185,7 @@ async fn register_does_not_reconstruct_membership_while_joining() { wireguard_mtu: None, cloud_pairing: None, }) + .await .unwrap(); assert_eq!(joiner.record().unwrap().phase(), LocalMachinePhase::Joining); diff --git a/crates/ployzd/src/machine/store.rs b/crates/ployzd/src/machine/store.rs new file mode 100644 index 0000000000..bc673cefe2 --- /dev/null +++ b/crates/ployzd/src/machine/store.rs @@ -0,0 +1,451 @@ +//! Durable Local Machine record ownership and persistence. + +use std::{ + collections::BTreeMap, + fs::{self, File, OpenOptions}, + io::{self, Write}, + net::IpAddr, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Component, Path, PathBuf}, + sync::Arc, +}; + +use ployz_core::{ + CloudPairing, LocalMachinePhase, Machine, MachineId, MachineUpdate, MachineUpdateError, + SelectedEndpoint, apply_machine_update, +}; +use thiserror::Error; + +use super::{ + FoundingCluster, LocalMachineBody, LocalMachineRecord, ParticipationOrigin, local_runtime, +}; +use crate::network::{WireGuardPrivateKey, allocate_machine_subnet}; + +const STATE_FILE_NAME: &str = "machine.json"; +const TEMPORARY_FILE_NAME: &str = ".machine.json.tmp"; +pub(super) const PENDING_RESET_FILE_NAME: &str = ".machine.reset.pending"; +const LOCK_FILE_NAME: &str = ".ployzd.lock"; + +pub struct LocalMachineStore { + pub(super) data_dir: PathBuf, + pub(super) run_dir: PathBuf, + record: LocalMachineRecord, + _lock: File, + // Lock order: admission, then publication/ingress/Docker, then short store locks. + // Reset waits for admitted operations, including Docker streams with no total deadline. + // ponytail: serialize local creates; use shared admission reads if throughput requires it. + pub(super) admission_lock: Arc>, + pub(super) mutation_gate: crate::mutation::MutationGate, +} + +pub(crate) struct PreparedReset { + data_dir: PathBuf, + original: LocalMachineRecord, + resetting: LocalMachineRecord, +} + +impl PreparedReset { + pub(crate) fn commit(self, store: &mut LocalMachineStore) -> Result<(), StoreError> { + let mut current = store.record.clone(); + current + .selected_endpoints + .clone_from(&self.original.selected_endpoints); + if store.data_dir != self.data_dir || current != self.original { + return Err(StoreError::ResetPreparationLost(store.data_dir.clone())); + } + fs::rename( + self.data_dir.join(PENDING_RESET_FILE_NAME), + self.data_dir.join(STATE_FILE_NAME), + )?; + store.record = self.resetting.clone(); + if let Err(error) = File::open(&self.data_dir).and_then(|directory| directory.sync_all()) { + eprintln!("failed to sync committed local Machine reset: {error}"); + } + Ok(()) + } +} + +impl Drop for PreparedReset { + fn drop(&mut self) { + let _ = fs::remove_file(self.data_dir.join(PENDING_RESET_FILE_NAME)); + } +} + +impl Drop for LocalMachineStore { + fn drop(&mut self) { + // A concurrently forked child can briefly inherit the flock before exec closes the fd. + let _ = fs2::FileExt::unlock(&self._lock); + } +} + +impl LocalMachineStore { + pub fn open(data_dir: impl AsRef) -> Result { + let data_dir = data_dir.as_ref(); + Self::open_with_admission(data_dir, data_dir.join(".run")) + } + + pub(crate) fn open_with_admission( + data_dir: impl AsRef, + run_dir: impl AsRef, + ) -> Result { + let data_dir = data_dir.as_ref().to_owned(); + let run_dir = run_dir.as_ref().to_owned(); + validate_data_dir(&data_dir)?; + claim_data_dir(&data_dir)?; + fs::create_dir_all(&data_dir)?; + fs::set_permissions(&data_dir, fs::Permissions::from_mode(0o711))?; + let lock = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .mode(0o600) + .open(data_dir.join(LOCK_FILE_NAME))?; + fs2::FileExt::try_lock_exclusive(&lock).map_err(|error| { + if error.kind() == io::ErrorKind::WouldBlock { + StoreError::AlreadyRunning(data_dir.clone()) + } else { + StoreError::Io(error) + } + })?; + let path = data_dir.join(STATE_FILE_NAME); + let record = match fs::read(&path) { + Ok(data) => serde_json::from_slice(&data)?, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + match fs::remove_file(data_dir.join(TEMPORARY_FILE_NAME)) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + let record = LocalMachineRecord { + body: LocalMachineBody::Uninitialized { + id: MachineId::random(), + }, + wireguard_private_key: WireGuardPrivateKey::generate(), + wireguard_mtu: None, + cloud_pairing: None, + selected_endpoints: BTreeMap::new(), + }; + save(&data_dir, &record)?; + record + } + Err(error) => return Err(error.into()), + }; + + let mut store = Self { + data_dir: data_dir.clone(), + record, + _lock: lock, + admission_lock: Arc::new(tokio::sync::Mutex::new(())), + run_dir: run_dir.clone(), + mutation_gate: crate::mutation::MutationGate::new(&run_dir, &data_dir), + }; + if store.record.phase() == LocalMachinePhase::Resetting { + let data_dir = store.data_dir.clone(); + store.complete_reset()?; + drop(store); + return Self::open_with_admission(data_dir, run_dir); + } + store.refresh_runtime()?; + Ok(store) + } + + #[must_use] + pub fn record(&self) -> &LocalMachineRecord { + &self.record + } + + fn refresh_runtime(&mut self) -> Result<(), StoreError> { + let runtime = local_runtime(); + let stale = match &self.record.body { + LocalMachineBody::Joining { machine, .. } + | LocalMachineBody::Participating { machine, .. } => machine.runtime != runtime, + LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { + return Ok(()); + } + }; + if !stale { + return Ok(()); + } + let mut refreshed = self.record.clone(); + match &mut refreshed.body { + LocalMachineBody::Joining { machine, .. } + | LocalMachineBody::Participating { machine, .. } => machine.runtime = runtime, + LocalMachineBody::Uninitialized { .. } | LocalMachineBody::Resetting { .. } => { + return Ok(()); + } + } + save(&self.data_dir, &refreshed)?; + self.record = refreshed; + Ok(()) + } + + pub fn begin_reset(&mut self) -> Result<(), StoreError> { + let prepared = self.prepare_reset()?; + prepared.commit(self) + } + + pub(crate) fn prepare_reset(&self) -> Result { + if self.record.phase() == LocalMachinePhase::Resetting { + return Err(StoreError::AlreadyResetting); + } + let resetting = self.record.clone().into_resetting(); + write_record(&self.data_dir.join(PENDING_RESET_FILE_NAME), &resetting)?; + File::open(&self.data_dir)?.sync_all()?; + Ok(PreparedReset { + data_dir: self.data_dir.clone(), + original: self.record.clone(), + resetting, + }) + } + + pub fn initialize( + &mut self, + name: ployz_core::MachineName, + founding_cluster: FoundingCluster, + public_ip: Option, + advertised_endpoints: Vec, + wireguard_mtu: Option, + cloud_pairing: Option, + ) -> Result { + self.require_uninitialized()?; + if advertised_endpoints.is_empty() { + return Err(StoreError::MissingEndpoints); + } + let public_key = self.record.wireguard_private_key.public_key(); + let machine = Machine { + id: self.record.id(), + name, + subnet: allocate_machine_subnet(founding_cluster.network, []) + .map_err(|error| StoreError::InvalidNetwork(error.to_string()))?, + public_key, + public_ip, + advertised_endpoints, + runtime: local_runtime(), + }; + let mut initialized = self.record.clone(); + initialized.body = LocalMachineBody::Participating { + machine: machine.clone(), + origin: ParticipationOrigin::Founder { + cluster: founding_cluster, + }, + }; + initialized.wireguard_mtu = wireguard_mtu; + initialized.cloud_pairing = cloud_pairing; + save(&self.data_dir, &initialized)?; + self.record = initialized; + Ok(machine) + } + + pub fn join( + &mut self, + mut assigned_machine: Machine, + visible_peers: Vec, + target_versions: BTreeMap, + wireguard_mtu: Option, + cloud_pairing: Option, + ) -> Result<(), StoreError> { + self.require_uninitialized()?; + if visible_peers.is_empty() { + return Err(StoreError::MissingPeers); + } + if self.record.wireguard_private_key.public_key() != assigned_machine.public_key { + return Err(StoreError::KeyMismatch); + } + if assigned_machine.advertised_endpoints.is_empty() { + return Err(StoreError::MissingEndpoints); + } + assigned_machine.runtime = local_runtime(); + let mut joining = self.record.clone(); + joining.body = LocalMachineBody::Joining { + machine: assigned_machine, + bootstrap: visible_peers, + min_store_version: target_versions, + }; + joining.wireguard_mtu = wireguard_mtu; + joining.cloud_pairing = cloud_pairing; + save(&self.data_dir, &joining)?; + self.record = joining; + Ok(()) + } + + pub fn update( + &mut self, + update: MachineUpdate, + visible: &[Machine], + ) -> Result { + let mut record = self.record.clone(); + let LocalMachineBody::Participating { machine, .. } = &mut record.body else { + return Err(StoreError::NotParticipating); + }; + let updated = apply_machine_update(machine, visible, update)?; + *machine = updated.clone(); + save(&self.data_dir, &record)?; + self.record = record; + Ok(updated) + } + + fn require_uninitialized(&self) -> Result<(), StoreError> { + if matches!(self.record.body, LocalMachineBody::Uninitialized { .. }) { + Ok(()) + } else { + Err(StoreError::AlreadyInitialized) + } + } + + pub fn complete_catch_up(&mut self) -> Result<(), StoreError> { + let mut participating = self.record.clone(); + let LocalMachineBody::Joining { + machine, bootstrap, .. + } = participating.body + else { + return Err(StoreError::NotJoining); + }; + participating.body = LocalMachineBody::Participating { + machine, + origin: ParticipationOrigin::Join { bootstrap }, + }; + save(&self.data_dir, &participating)?; + self.record = participating; + Ok(()) + } + + pub fn persist_selected_endpoint( + &mut self, + machine_id: MachineId, + endpoint: SelectedEndpoint, + ) -> Result<(), StoreError> { + let mut updated = self.record.clone(); + updated.selected_endpoints.insert(machine_id, endpoint); + save(&self.data_dir, &updated)?; + self.record = updated; + Ok(()) + } + + pub fn persist_cloud_pairing( + &mut self, + pairing: Option, + ) -> Result<(), StoreError> { + let mut updated = self.record.clone(); + updated.cloud_pairing = pairing; + save(&self.data_dir, &updated)?; + self.record = updated; + Ok(()) + } + + pub fn complete_reset(&self) -> Result<(), StoreError> { + if self.record.phase() != LocalMachinePhase::Resetting { + return Err(StoreError::NotResetting); + } + let persisted: LocalMachineRecord = + serde_json::from_slice(&fs::read(self.data_dir.join(STATE_FILE_NAME))?)?; + if persisted != self.record { + return Err(StoreError::OwnershipLost(self.data_dir.clone())); + } + fs::remove_dir_all(&self.data_dir).map_err(StoreError::Io) + } +} + +fn validate_data_dir(path: &Path) -> Result<(), StoreError> { + if path.file_name().is_none() + || path + .components() + .any(|component| component == Component::ParentDir) + { + return Err(StoreError::UnsafeDataDirectory(path.to_owned())); + } + Ok(()) +} + +fn claim_data_dir(data_dir: &Path) -> Result<(), StoreError> { + match fs::read_dir(data_dir) { + Ok(entries) => { + let names = entries + .map(|entry| Ok(entry?.file_name())) + .collect::>>()?; + let owns_directory = names.iter().any(|name| name == STATE_FILE_NAME) + || names + .iter() + .all(|name| name == TEMPORARY_FILE_NAME || name == LOCK_FILE_NAME); + if owns_directory { + Ok(()) + } else { + Err(StoreError::UnownedDataDirectory(data_dir.to_owned())) + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn save(data_dir: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { + fs::create_dir_all(data_dir)?; + fs::set_permissions(data_dir, fs::Permissions::from_mode(0o711))?; + + let path = data_dir.join(STATE_FILE_NAME); + let temporary = data_dir.join(TEMPORARY_FILE_NAME); + write_record(&temporary, record)?; + fs::rename(temporary, path)?; + File::open(data_dir)?.sync_all()?; + Ok(()) +} + +fn write_record(path: &Path, record: &LocalMachineRecord) -> Result<(), StoreError> { + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path)?; + file.set_permissions(fs::Permissions::from_mode(0o600))?; + serde_json::to_writer_pretty(&mut file, record)?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) +} + +fn display_data_directory(path: &Path) -> String { + let mut text = String::new(); + for chunk in path.as_os_str().as_encoded_bytes().utf8_chunks() { + text.extend(chunk.valid().escape_debug()); + text.extend(chunk.invalid().escape_ascii().map(char::from)); + } + text +} + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("local Machine record I/O failed: {0}")] + Io(#[from] io::Error), + #[error("local Machine record JSON is invalid: {0}")] + Json(#[from] serde_json::Error), + #[error("machine is not participating")] + NotParticipating, + #[error("machine is already resetting")] + AlreadyResetting, + #[error("machine is not resetting")] + NotResetting, + #[error("machine is not joining")] + NotJoining, + #[error("machine is already initialized")] + AlreadyInitialized, + #[error("at least one advertised endpoint is required")] + MissingEndpoints, + #[error("at least one visible peer is required to join")] + MissingPeers, + #[error("assigned public key does not match this Machine")] + KeyMismatch, + #[error("invalid Cluster network: {0}")] + InvalidNetwork(String), + #[error(transparent)] + MachineUpdate(#[from] MachineUpdateError), + #[error("another daemon already owns data directory {}", display_data_directory(.0))] + AlreadyRunning(PathBuf), + #[error("refusing to clear broad data directory {}", display_data_directory(.0))] + UnsafeDataDirectory(PathBuf), + #[error("refusing to claim nonempty data directory {}", display_data_directory(.0))] + UnownedDataDirectory(PathBuf), + #[error("local Machine record changed before clearing data directory {}", display_data_directory(.0))] + OwnershipLost(PathBuf), + #[error("local Machine record changed before prepared reset was committed in {}", display_data_directory(.0))] + ResetPreparationLost(PathBuf), +} diff --git a/crates/ployzd/src/machine_api/local.rs b/crates/ployzd/src/machine_api/local.rs index d10b1db758..257fdaf739 100644 --- a/crates/ployzd/src/machine_api/local.rs +++ b/crates/ployzd/src/machine_api/local.rs @@ -8,11 +8,10 @@ use std::{ }; use ployz_core::{ - CapabilityAdvertisement, CloudPairing, CloudPairingSet, ContainerChanged, ContainerList, - ContainerObservationMap, ContractDescription, Domain, DomainRecords, ImageIngestReason, - ImagePulled, IngressProxyConfig, LocalMachinePhase, LogMetadata, LogOrigin, MachineId, - MachineLogService, MachineRpc, MachineRpcClient, OpaquePayload, PROTOCOL_MAJOR, Rpc, RpcError, - RpcErrorCode, RpcRequestBody, RpcResponse, VolumeRemoved, op, + CapabilityAdvertisement, CloudPairing, CloudPairingSet, ContainerList, ContainerObservationMap, + ContractDescription, Domain, DomainRecords, IngressProxyConfig, LocalMachinePhase, LogMetadata, + LogOrigin, MachineId, MachineLogService, MachineRpc, MachineRpcClient, OpaquePayload, + PROTOCOL_MAJOR, Rpc, RpcError, RpcErrorCode, RpcRequestBody, RpcResponse, op, }; use serde_json::Value; use tokio::{sync::watch, time::Instant}; @@ -306,7 +305,11 @@ impl MachineRpc for MachineService { &self, request: Request, ) -> Result, Status> { - finish(self.local.initialize(expect::(request)?)) + finish( + self.local + .initialize(expect::(request)?) + .await, + ) } async fn register( @@ -335,7 +338,7 @@ impl MachineRpc for MachineService { &self, request: Request, ) -> Result, Status> { - finish(self.local.join(expect::(request)?)) + finish(self.local.join(expect::(request)?).await) } async fn set_cloud_pairing( @@ -343,7 +346,11 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - if let Err(error) = self.local.set_cloud_pairing(request.cloud_pairing.clone()) { + if let Err(error) = self + .local + .set_cloud_pairing(request.cloud_pairing.clone()) + .await + { return local_error(error); } if let Some(sender) = &self.cloud_pairing { @@ -451,16 +458,7 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers, - Err(error) => return respond(error), - }; - match containers.start(&request.container_id).await { - Ok(()) => respond(ContainerChanged { - container_id: request.container_id, - }), - Err(error) => respond(RpcError::from(&error)), - } + finish(self.local.start_container(request.container_id).await) } async fn stop_container( @@ -468,23 +466,15 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers, - Err(error) => return respond(error), - }; - match containers - .stop( - &request.container_id, - request.signal.as_deref(), - request.grace_period_seconds, - ) - .await - { - Ok(()) => respond(ContainerChanged { - container_id: request.container_id, - }), - Err(error) => respond(RpcError::from(&error)), - } + finish( + self.local + .stop_container( + request.container_id, + request.signal, + request.grace_period_seconds, + ) + .await, + ) } async fn remove_container( @@ -492,19 +482,11 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers, - Err(error) => return respond(error), - }; - match containers - .remove(&request.container_id, request.remove_volumes, request.force) - .await - { - Ok(()) => respond(ContainerChanged { - container_id: request.container_id, - }), - Err(error) => respond(RpcError::from(&error)), - } + finish( + self.local + .remove_container(request.container_id, request.remove_volumes, request.force) + .await, + ) } async fn create_volume( @@ -512,15 +494,7 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let machine_id = self.local_record()?.id(); - let containers = match self.containers() { - Ok(containers) => containers, - Err(error) => return respond(error), - }; - match containers.create_volume(&machine_id, request).await { - Ok(volume) => respond(volume), - Err(error) => respond(RpcError::from(&error)), - } + finish(self.local.create_volume(request).await) } async fn inspect_storage( @@ -579,14 +553,7 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { let request = expect::(request)?; - let containers = match self.containers() { - Ok(containers) => containers, - Err(error) => return respond(error), - }; - match containers.remove_volume(&request.name, request.force).await { - Ok(()) => respond(VolumeRemoved {}), - Err(error) => respond(RpcError::from(&error)), - } + finish(self.local.remove_volume(request.name, request.force).await) } async fn build( @@ -599,6 +566,7 @@ impl MachineRpc for MachineService { self.local_record()?.id(), request.into_inner(), self.builds.clone(), + self.local.clone(), ))) } @@ -697,6 +665,28 @@ impl MachineRpc for MachineService { ) } + async fn request_machine_upgrade( + &self, + request: Request, + ) -> Result, Status> { + finish( + self.local + .request_upgrade(expect::(request)?) + .await, + ) + } + + async fn inspect_machine_upgrade( + &self, + request: Request, + ) -> Result, Status> { + finish( + self.local + .inspect_upgrade(expect::(request)?) + .await, + ) + } + async fn remove_local_machine( &self, request: Request, @@ -748,19 +738,11 @@ impl MachineRpc for MachineService { request: Request, ) -> Result, Status> { expect::(request)?; - let record = self.local_record()?; - let Some(machine) = record - .machine() - .filter(|_| record.phase() == LocalMachinePhase::Participating) - else { - return respond( - ImageIngestReason::NotParticipating.rpc_error("Machine is not participating"), - ); - }; - match self.ingest.open(machine.management_address()).await { - Ok(opened) => respond(opened), - Err(error) => respond(error), - } + finish( + self.local + .ensure_image_ingest(Arc::clone(&self.ingest)) + .await, + ) } async fn pull_image_from_machine( @@ -775,13 +757,7 @@ impl MachineRpc for MachineService { details: Value::Null, }); } - if self.containers().is_err() { - return respond(unavailable("Docker is not available")); - } - match crate::docker::pull_from_ingest(&request.image, request.source).await { - Ok(()) => respond(ImagePulled {}), - Err(error) => respond(RpcError::from(&error)), - } + finish(self.local.pull_image_from_machine(request).await) } async fn get_ingress_proxy_config( @@ -972,6 +948,42 @@ fn local_error(error: LocalMachineError) -> Result, Stat LocalMachineError::AllocatorNotQuiet | LocalMachineError::NotAllocator | LocalMachineError::IsolationLocked => respond(unavailable(&error.to_string())), + LocalMachineError::Admission(crate::mutation::Error::Busy) => respond(RpcError { + code: RpcErrorCode::Conflict, + message: "a Ployz installation or upgrade is active".into(), + details: Value::Null, + }), + LocalMachineError::Admission(crate::mutation::Error::Io(error)) => { + Err(Status::internal(error.to_string())) + } + LocalMachineError::Upgrade(error) => respond(upgrade_error(error)), + } +} + +fn upgrade_error(error: crate::installer::upgrade::Error) -> RpcError { + use crate::{installer::upgrade::Error, mutation}; + let code = match &error { + Error::NotFound => RpcErrorCode::NotFound, + Error::AttemptConflict(_) | Error::Busy | Error::Admission(mutation::Error::Busy) => { + RpcErrorCode::Conflict + } + Error::Resolve(_) | Error::NonstandardPaths(_) => RpcErrorCode::InvalidArgument, + Error::Read(_) + | Error::Decode(_) + | Error::Write(_) + | Error::Encode(_) + | Error::QualificationSource(_) + | Error::Launch(_) + | Error::InspectWorker(_) + | Error::WorkerEvidence(_) + | Error::NotActive(_) + | Error::Installation(_) + | Error::Admission(mutation::Error::Io(_)) => RpcErrorCode::Internal, + }; + RpcError { + code, + message: error.to_string(), + details: Value::Null, } } diff --git a/crates/ployzd/src/machine_api/local_tests.rs b/crates/ployzd/src/machine_api/local_tests.rs index aa2c49b336..d50eefc9fb 100644 --- a/crates/ployzd/src/machine_api/local_tests.rs +++ b/crates/ployzd/src/machine_api/local_tests.rs @@ -6,7 +6,8 @@ use crate::machine::{LocalMachineError, LocalMachineStore, StoreError}; use ployz_core::{ ContainerAddress, ContainerId, ContainerKind, ContainerObservation, ContainerRuntimeObservation, GET_CONTAINER_OBSERVATIONS_CAPABILITY, - GetContainerObservationsRequest, HealthObservation, MachineId, MachineRpc, ProjectName, + GetContainerObservationsRequest, HealthObservation, MachineId, MachineRelease, MachineRpc, + MachineUpgradeAttemptId, ProjectName, RequestMachineUpgradeRequest, ResetRequest, ResolvedServiceSpec, RpcErrorCode, RpcResponseBody, RuntimeWatchRequest, ServiceId, ServiceName, op, }; @@ -101,6 +102,99 @@ fn allocator_not_quiet_is_retryable_unavailable() { assert_eq!(error.message, "Allocator is not quiet"); } +#[tokio::test] +async fn upgrade_and_machine_mutations_refuse_each_other_at_the_rpc_boundary() { + let data_dir = std::env::temp_dir().join(format!( + "ployzd-upgrade-rpc-admission-{}", + MachineId::random() + )); + let store = Arc::new(Mutex::new(LocalMachineStore::open(&data_dir).unwrap())); + let service = MachineService::with_cluster(store, watch::channel(false).0, None); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let mutation = tokio::spawn({ + let local = service.local(); + let started = Arc::clone(&started); + let release = Arc::clone(&release); + async move { local.hold_mutation_for_test(started, release).await } + }); + started.notified().await; + + let response = service + .request_machine_upgrade(Request::new( + op::RequestMachineUpgrade::into_request(RequestMachineUpgradeRequest { + attempt_id: MachineUpgradeAttemptId::parse("a".repeat(32)).unwrap(), + release: MachineRelease::parse("1.2.3").unwrap(), + }) + .encode() + .unwrap(), + )) + .await + .unwrap() + .into_inner() + .decode_response() + .unwrap(); + assert!(matches!( + response.body, + RpcResponseBody::Error(error) if error.code == RpcErrorCode::Conflict + )); + release.notify_one(); + mutation.await.unwrap().unwrap(); + + std::fs::write(data_dir.join(".upgrade-active"), "attempt").unwrap(); + let response = service + .reset(Request::new( + op::Reset::into_request(ResetRequest {}).encode().unwrap(), + )) + .await + .unwrap() + .into_inner() + .decode_response() + .unwrap(); + assert!(matches!( + response.body, + RpcResponseBody::Error(error) if error.code == RpcErrorCode::Conflict + )); + + drop(service); + std::fs::remove_dir_all(data_dir).unwrap(); +} + +#[tokio::test] +async fn upgrade_rpc_rejects_nonstandard_machine_paths_before_acceptance() { + let data_dir = + std::env::temp_dir().join(format!("ployzd-upgrade-rpc-paths-{}", MachineId::random())); + let store = Arc::new(Mutex::new(LocalMachineStore::open(&data_dir).unwrap())); + let service = MachineService::with_cluster(store, watch::channel(false).0, None); + + let response = service + .request_machine_upgrade(Request::new( + op::RequestMachineUpgrade::into_request(RequestMachineUpgradeRequest { + attempt_id: MachineUpgradeAttemptId::parse("b".repeat(32)).unwrap(), + release: MachineRelease::parse("1.2.3").unwrap(), + }) + .encode() + .unwrap(), + )) + .await + .unwrap() + .into_inner() + .decode_response() + .unwrap(); + + assert!(matches!( + response.body, + RpcResponseBody::Error(error) + if error.code == RpcErrorCode::InvalidArgument + && error.message.contains("system installation requires --data-dir") + )); + assert!(!data_dir.join("upgrade-attempt.json").exists()); + assert!(!data_dir.join(".upgrade-active").exists()); + + drop(service); + std::fs::remove_dir_all(data_dir).unwrap(); +} + #[test] fn not_allocator_does_not_allocate() { let RpcResponseBody::Error(error) = local_error(LocalMachineError::NotAllocator) diff --git a/crates/ployzd/src/main.rs b/crates/ployzd/src/main.rs index f470533146..8dc2a493d3 100644 --- a/crates/ployzd/src/main.rs +++ b/crates/ployzd/src/main.rs @@ -12,16 +12,18 @@ use std::{ }; use clap::{Parser, Subcommand}; -use ployz_core::DOCKER_NETWORK_CONFLICT_EXIT_STATUS; +use ployz_core::{DOCKER_NETWORK_CONFLICT_EXIT_STATUS, MachineUpgradeAttemptId, StorageChoice}; use ployzd::{ daemon::{ContainerMode, Daemon, DaemonConfig, Error, wait_until_socket_accepts}, diag, + installer::{ + DEFAULT_SOCKET_PATH, InstallMode, InstallRequest, Readiness, ReleaseRequest, ReleaseSource, + }, machine::DEFAULT_DATA_DIR, network::NetworkError, }; use tokio::io::{AsyncWriteExt, copy, stdin, stdout}; -const DEFAULT_SOCKET_PATH: &str = "/run/ployz/ployz.sock"; const DIAL_STDIO_SOCKET_TIMEOUT: Duration = Duration::from_secs(20); #[derive(Parser)] @@ -53,6 +55,33 @@ enum Command { DialStdio, /// Serve the Docker Volume plugin on its systemd socket. VolumePlugin, + /// Execute one accepted Machine upgrade from its transient systemd service. + #[command(hide = true)] + UpgradeWorker { + #[arg(long)] + attempt: MachineUpgradeAttemptId, + }, + /// Install or replace this Machine's daemon release. + Install { + /// Release channel (stable or beta) or exact published version. + #[arg(long, default_value = "stable")] + version: String, + /// Prepare ZFS storage, or leave this Machine stateless. + #[arg(long, default_value = "none")] + storage: StorageChoice, + /// Replace daemon software only; do not install Docker, OS packages, or prepare storage. + #[arg(long)] + software_only: bool, + /// Write files and units but do not contact or start systemd. + #[arg(long)] + install_only: bool, + /// Add this existing operator to the Ployz service group during host preparation. + #[arg(long, value_name = "USER")] + group_user: Option, + /// Read a verified release from this local directory. Used by offline qualification. + #[arg(long, hide = true, value_name = "DIR")] + release_dir: Option, + }, } fn main() -> ExitCode { @@ -97,11 +126,58 @@ async fn run(args: Args) -> Result<(), Error> { if matches!(args.command, Some(Command::DialStdio)) { return dial_stdio(&args.socket).await.map_err(Error::from); } + let run_dir = args + .socket + .parent() + .unwrap_or_else(|| Path::new("/run/ployz")) + .to_owned(); + if let Some(Command::UpgradeWorker { attempt }) = args.command { + return ployzd::installer::upgrade::run_worker(attempt, &args.data_dir, &run_dir) + .await + .map_err(io::Error::other) + .map_err(Error::from); + } + if let Some(Command::Install { + version, + storage, + software_only, + install_only, + group_user, + release_dir, + }) = args.command + { + let request = install_request( + version, + storage, + software_only, + install_only, + group_user, + release_dir, + ) + .map_err(Error::from)?; + let outcome = ployzd::installer::install(request, &args.data_dir, &args.socket) + .await + .map_err(io::Error::other)?; + match outcome.readiness { + Readiness::InstallationOnly => { + println!( + "Ployz {} installed; systemd was not started", + outcome.target + ); + } + Readiness::Running => { + println!("Ployz {} is running and ready", outcome.target); + } + } + return Ok(()); + } diag::init(args.log_level.as_deref()) .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; if matches!(args.command, Some(Command::VolumePlugin)) { let listener = volume_plugin::inherited_listener()?; - return volume_plugin::run(listener).await.map_err(Error::from); + return volume_plugin::run(listener, &args.data_dir, &run_dir) + .await + .map_err(Error::from); } let daemon = Daemon::start(DaemonConfig { data_dir: args.data_dir, @@ -115,6 +191,58 @@ async fn run(args: Args) -> Result<(), Error> { daemon.wait().await } +fn install_request( + version: String, + storage: StorageChoice, + software_only: bool, + install_only: bool, + group_user: Option, + release_dir: Option, +) -> io::Result { + let release = version + .parse::() + .map_err(io::Error::other)?; + if install_only && storage != StorageChoice::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--install-only cannot be combined with --storage", + )); + } + if install_only && group_user.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--install-only cannot be combined with --group-user", + )); + } + let mode = if install_only { + InstallMode::InstallationOnly + } else if software_only { + if storage != StorageChoice::None { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--software-only cannot be combined with --storage", + )); + } + if group_user.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "--software-only cannot be combined with --group-user", + )); + } + InstallMode::SoftwareOnly + } else { + InstallMode::PrepareHost { + storage, + group_user, + } + }; + Ok(InstallRequest { + release, + source: release_dir.map_or(ReleaseSource::Published, ReleaseSource::Local), + mode, + }) +} + async fn dial_stdio(path: &Path) -> io::Result<()> { let stream = wait_until_socket_accepts(path, DIAL_STDIO_SOCKET_TIMEOUT).await?; let (mut socket_read, mut socket_write) = stream.into_split(); @@ -151,4 +279,80 @@ mod tests { ExitCode::FAILURE ); } + + #[test] + fn install_cli_rejects_host_options_for_software_only_replacement() { + let storage = install_request("stable".into(), StorageChoice::Zfs, true, false, None, None) + .unwrap_err(); + assert_eq!(storage.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + storage.to_string(), + "--software-only cannot be combined with --storage" + ); + + let group = install_request( + "stable".into(), + StorageChoice::None, + true, + false, + Some("operator".into()), + None, + ) + .unwrap_err(); + assert_eq!(group.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + group.to_string(), + "--software-only cannot be combined with --group-user" + ); + } + + #[test] + fn install_cli_rejects_host_options_for_installation_only() { + let storage = install_request("stable".into(), StorageChoice::Zfs, false, true, None, None) + .unwrap_err(); + assert_eq!(storage.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + storage.to_string(), + "--install-only cannot be combined with --storage" + ); + + let group = install_request( + "stable".into(), + StorageChoice::None, + false, + true, + Some("operator".into()), + None, + ) + .unwrap_err(); + assert_eq!(group.kind(), io::ErrorKind::InvalidInput); + assert_eq!( + group.to_string(), + "--install-only cannot be combined with --group-user" + ); + } + + #[test] + fn install_cli_builds_one_explicit_mode() { + let replacement = + install_request("1.2.3".into(), StorageChoice::None, true, true, None, None).unwrap(); + assert!(matches!(replacement.mode, InstallMode::InstallationOnly)); + + let host = install_request( + "1.2.3".into(), + StorageChoice::Zfs, + false, + false, + Some("operator".into()), + None, + ) + .unwrap(); + assert!(matches!( + host.mode, + InstallMode::PrepareHost { + storage: StorageChoice::Zfs, + group_user: Some(_) + } + )); + } } diff --git a/crates/ployzd/src/mutation.rs b/crates/ployzd/src/mutation.rs new file mode 100644 index 0000000000..0f858b5d93 --- /dev/null +++ b/crates/ployzd/src/mutation.rs @@ -0,0 +1,205 @@ +//! Cross-process exclusion between Machine installation and local mutation. + +use std::{ + fs::{self, File, OpenOptions, TryLockError}, + io, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::PathBuf, +}; + +use thiserror::Error; + +const LOCK_FILE: &str = ".install.lock"; +const ACTIVE_FILE: &str = ".upgrade-active"; + +/// Shared Machine mutation policy backed by an OS lock and durable upgrade evidence. +#[derive(Clone, Debug)] +pub struct MutationGate { + lock_dir: PathBuf, + data_dir: PathBuf, +} + +/// A refusal or local I/O failure while claiming Machine mutation ownership. +#[derive(Debug, Error)] +pub enum Error { + /// Another mutation, installation, or durable upgrade attempt owns the Machine. + #[error("a Ployz installation or upgrade is active")] + Busy, + /// Local ownership evidence could not be inspected or changed. + #[error("Machine mutation admission: {0}")] + Io(#[from] io::Error), +} + +/// Shared ownership that excludes Machine installation while a mutation finishes. +pub struct MutationGuard(File); + +/// Exclusive ownership that excludes every other Machine installation or mutation. +pub struct InstallationGuard(File); + +impl MutationGate { + /// Bind one gate to its runtime lock and durable data directories. + #[must_use] + pub fn new(lock_dir: impl Into, data_dir: impl Into) -> Self { + Self { + lock_dir: lock_dir.into(), + data_dir: data_dir.into(), + } + } + + /// Claim shared mutation ownership when no installation evidence exists. + /// + /// # Errors + /// + /// Returns [`Error::Busy`] when an installation lock or any durable active-marker entry + /// exists. Returns [`Error::Io`] when either ownership source cannot be inspected. + pub fn try_mutation(&self) -> Result { + let file = self.open_lock()?; + file.try_lock_shared().map_err(lock_error)?; + match fs::symlink_metadata(self.active_path()) { + Ok(_) => { + let _ = file.unlock(); + Err(Error::Busy) + } + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(MutationGuard(file)), + Err(error) => { + let _ = file.unlock(); + Err(Error::Io(error)) + } + } + } + + /// Try to claim exclusive installation ownership without waiting. + /// + /// # Errors + /// + /// Returns [`Error::Busy`] when another owner holds the lock and [`Error::Io`] when the + /// lock directory or file cannot be prepared. + pub fn try_installation(&self) -> Result { + let file = self.open_lock()?; + file.try_lock().map_err(lock_error)?; + Ok(InstallationGuard(file)) + } + + /// Wait for exclusive installation ownership. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when the lock directory or file cannot be prepared or locked. + pub fn lock_installation(&self) -> Result { + let file = self.open_lock()?; + file.lock()?; + Ok(InstallationGuard(file)) + } + + /// Whether any filesystem entry exists at the durable active-marker path. + /// + /// # Errors + /// + /// Returns [`Error::Io`] when absence cannot be established from filesystem metadata. + pub fn active(&self) -> Result { + match fs::symlink_metadata(self.active_path()) { + Ok(_) => Ok(true), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(Error::Io(error)), + } + } + + pub(crate) fn mark_active(&self, attempt_id: &str) -> Result<(), Error> { + fs::create_dir_all(&self.data_dir)?; + crate::filesystem::atomic_write(&self.active_path(), attempt_id.as_bytes(), 0o600)?; + Ok(()) + } + + pub(crate) fn clear_active(&self, attempt_id: &str) -> Result<(), Error> { + let path = self.active_path(); + match fs::read_to_string(&path) { + Ok(active) if active == attempt_id => fs::remove_file(path)?, + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + } + File::open(&self.data_dir)?.sync_all()?; + Ok(()) + } + + fn open_lock(&self) -> Result { + fs::create_dir_all(&self.lock_dir)?; + fs::set_permissions(&self.lock_dir, fs::Permissions::from_mode(0o750))?; + OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .mode(0o600) + .open(self.lock_dir.join(LOCK_FILE)) + .map_err(Into::into) + } + + fn active_path(&self) -> PathBuf { + self.data_dir.join(ACTIVE_FILE) + } +} + +fn lock_error(error: TryLockError) -> Error { + match error { + TryLockError::WouldBlock => Error::Busy, + TryLockError::Error(error) => Error::Io(error), + } +} + +impl Drop for MutationGuard { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +impl Drop for InstallationGuard { + fn drop(&mut self) { + let _ = self.0.unlock(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installation_and_active_handoff_exclude_mutations_across_instances() { + let root = crate::test_dir::TestDir::new("mutation-gate"); + let first = MutationGate::new(root.0.join("run"), root.0.join("data")); + let second = MutationGate::new(root.0.join("run"), root.0.join("data")); + + let install = first.try_installation().unwrap(); + assert!(matches!(second.try_mutation(), Err(Error::Busy))); + first.mark_active("attempt").unwrap(); + drop(install); + assert!(matches!(second.try_mutation(), Err(Error::Busy))); + + let install = second.try_installation().unwrap(); + second.clear_active("attempt").unwrap(); + drop(install); + assert!(second.try_mutation().is_ok()); + } + + #[test] + fn every_existing_marker_type_fails_closed() { + let root = crate::test_dir::TestDir::new("mutation-marker-type"); + let gate = MutationGate::new(root.0.join("run"), root.0.join("data")); + fs::create_dir_all(gate.active_path()).unwrap(); + + assert!(matches!(gate.try_mutation(), Err(Error::Busy))); + assert!(gate.active().unwrap()); + } + + #[test] + fn marker_metadata_errors_are_not_absence() { + let root = crate::test_dir::TestDir::new("mutation-marker-error"); + fs::create_dir_all(&root.0).unwrap(); + let data = root.0.join("data-is-a-file"); + fs::write(&data, "not a directory").unwrap(); + let gate = MutationGate::new(root.0.join("run"), data); + + assert!(matches!(gate.try_mutation(), Err(Error::Io(_)))); + assert!(matches!(gate.active(), Err(Error::Io(_)))); + } +} diff --git a/crates/ployzd/src/volume_plugin.rs b/crates/ployzd/src/volume_plugin.rs index 9cd7d04741..c941cd24bc 100644 --- a/crates/ployzd/src/volume_plugin.rs +++ b/crates/ployzd/src/volume_plugin.rs @@ -1,8 +1,7 @@ //! Docker v1 Volume plugin backed by bounded ZFS datasets. use std::{ - collections::BTreeMap, fmt, io, os::unix::net::UnixListener as StdUnixListener, path::PathBuf, - str::FromStr, sync::Arc, + collections::BTreeMap, fmt, io, os::unix::net::UnixListener as StdUnixListener, str::FromStr, }; use axum::{ @@ -13,18 +12,19 @@ use axum::{ response::Response, routing::post, }; -use ployzd::machine_pool::MachinePool; use serde::{Deserialize, Serialize}; -use tokio::{net::UnixListener, process::Command, sync::Mutex}; +use tokio::net::UnixListener; mod capacity; mod pool; mod removal; +mod storage; -use pool::PoolStorage; +use storage::{ + CapacityAdmission, DATASET_ROOT, Dataset, MOUNT_ROOT, VolumeStorage, checked_command, + parse_size, +}; -const DATASET_ROOT: &str = "ployz"; -const MOUNT_ROOT: &str = "/var/lib/ployz-volumes"; type Result = std::result::Result; #[derive(Debug, thiserror::Error)] @@ -77,347 +77,6 @@ impl DockerVolumeName { } } -#[derive(Clone)] -struct VolumeStorage { - pool: PoolStorage, - zfs: PathBuf, - mutation: Arc>, -} - -enum CapacityAdmission { - Required, - Ensured, -} - -impl VolumeStorage { - fn new() -> Self { - Self { - pool: PoolStorage::new("zpool"), - zfs: "zfs".into(), - mutation: Arc::new(Mutex::new(())), - } - } - - #[cfg(test)] - fn with_programs(zpool: impl Into, zfs: impl Into) -> Self { - let zpool = zpool.into(); - let backing = zpool.with_file_name("machine-pool"); - Self { - pool: PoolStorage::new(zpool).with_backing(backing), - zfs: zfs.into(), - mutation: Arc::new(Mutex::new(())), - } - } - - async fn create_volume( - &self, - pool: &MachinePool, - name: &DockerVolumeName, - requested: u64, - origin: CapacityAdmission, - ) -> Result<()> { - let datasets = self.datasets(pool).await?; - let root = format!("{}/{DATASET_ROOT}", pool.name()); - let volume = format!("{root}/{name}"); - - if let Some(existing) = Self::dataset(&datasets, pool, name)? { - existing.require_mountpoint(&name.mountpoint())?; - existing.require_writable()?; - return if existing.refquota == requested { - Ok(()) - } else { - Err(format!( - "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", - existing.refquota - ) - .into()) - }; - } - - if matches!(origin, CapacityAdmission::Required) { - let commitment = datasets - .iter() - .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) - .map(|dataset| dataset.refquota) - .try_fold(requested, u64::checked_add) - .ok_or_else(|| { - VolumeError::from("Provisioned Volume commitments overflowed u64") - })?; - let managed_used = datasets - .iter() - .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) - .map(|dataset| dataset.active_used_bytes) - .try_fold(0u64, u64::checked_add) - .ok_or("Dataset occupancy overflows u64")?; - self.pool - .ensure_capacity( - pool, - commitment, - pool.used_bytes().saturating_sub(managed_used), - ) - .await?; - } - - if !datasets.iter().any(|dataset| dataset.name == root) { - self.zfs(&[ - "create", - "-o", - "canmount=off", - "-o", - &format!("mountpoint={MOUNT_ROOT}"), - &root, - ]) - .await?; - } - self.zfs(&["create", "-o", &format!("refquota={requested}"), &volume]) - .await?; - Ok(()) - } - - async fn mountpoint(&self, name: &DockerVolumeName) -> Result { - let _guard = self.mutation.lock().await; - let pool = self.one_pool().await?; - let datasets = self.datasets(&pool).await?; - let dataset = Self::dataset(&datasets, &pool, name)? - .ok_or_else(|| format!("Provisioned Volume {name} does not exist"))?; - dataset.require_provisioned(name)?; - dataset.require_writable()?; - if dataset.mounted { - return Ok(dataset.mountpoint.clone()); - } - self.zfs(&["mount", &dataset.name]).await?; - let datasets = self.datasets(&pool).await?; - let dataset = Self::dataset(&datasets, &pool, name)? - .ok_or_else(|| format!("Provisioned Volume {name} disappeared while mounting"))?; - dataset.require_provisioned(name)?; - dataset.require_writable()?; - if !dataset.mounted { - return Err(format!("Provisioned Volume {name} did not mount").into()); - } - Ok(dataset.mountpoint.clone()) - } - - async fn one_pool(&self) -> Result { - self.pool - .one_usable() - .await? - .ok_or_else(|| "no usable existing Machine Pool".into()) - } - - async fn datasets(&self, pool: &MachinePool) -> Result> { - let output = self - .zfs(&[ - "list", - "-Hp", - "-o", - "name,refquota,used,usedbydataset,mountpoint,mounted,readonly", - "-r", - pool.name(), - ]) - .await?; - output - .lines() - .map(|line| Dataset::parse(line, pool.name())) - .collect() - } - - fn dataset<'datasets>( - datasets: &'datasets [Dataset], - pool: &MachinePool, - name: &DockerVolumeName, - ) -> Result> { - let root_name = format!("{}/{DATASET_ROOT}", pool.name()); - let root = datasets.iter().find(|dataset| dataset.name == root_name); - if let Some(root) = root { - root.require_mountpoint(MOUNT_ROOT)?; - root.require_writable()?; - } - - let requested = format!("{}/{DATASET_ROOT}/{name}", pool.name()); - let descendant_prefix = format!("{requested}/"); - if let Some(dataset) = datasets - .iter() - .find(|dataset| dataset.name.starts_with(&descendant_prefix)) - { - return Err(format!( - "ZFS dataset {} is a descendant of Provisioned Volume {name}; remove it before retrying", - dataset.name - ) - .into()); - } - let dataset = datasets.iter().find(|dataset| dataset.name == requested); - if dataset.is_some() && root.is_none() { - return Err(format!("ZFS did not report managed root dataset {root_name}").into()); - } - Ok(dataset) - } - - async fn zfs(&self, args: &[&str]) -> Result { - checked_command(&self.zfs, args).await - } -} - -struct Dataset { - name: String, - refquota: u64, - used_bytes: u64, - active_used_bytes: u64, - mountpoint: String, - mounted: bool, - readonly: bool, -} - -impl Dataset { - fn parse(line: &str, pool: &str) -> Result { - let mut fields = line.split('\t'); - let invalid = || format!("invalid ZFS dataset output for Pool {pool}: {line}"); - let ( - Some(name), - Some(refquota), - Some(used_bytes), - Some(active_used_bytes), - Some(mountpoint), - Some(mounted), - Some(readonly), - None, - ) = ( - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - fields.next(), - ) - else { - return Err(invalid().into()); - }; - let used_bytes = used_bytes.parse::().map_err(|_| invalid())?; - let active_used_bytes = active_used_bytes.parse::().map_err(|_| invalid())?; - if active_used_bytes > used_bytes { - return Err(invalid().into()); - } - Ok(Self { - name: name.to_owned(), - refquota: parse_zfs_bytes(refquota)?, - used_bytes, - active_used_bytes, - mountpoint: mountpoint.to_owned(), - mounted: mounted == "yes", - readonly: match readonly { - "off" => false, - "on" => true, - _ => return Err(invalid().into()), - }, - }) - } - - fn require_mountpoint(&self, expected: &str) -> Result<()> { - if self.mountpoint == expected { - return Ok(()); - } - Err(format!( - "ZFS dataset {} has incompatible mountpoint {}; set it to {expected} before retrying", - self.name, self.mountpoint - ) - .into()) - } - - fn require_writable(&self) -> Result<()> { - if !self.readonly { - return Ok(()); - } - Err(format!( - "ZFS dataset {} is read-only; make it writable before retrying", - self.name - ) - .into()) - } - - fn require_provisioned(&self, name: &DockerVolumeName) -> Result<()> { - if self.refquota == 0 { - return Err(format!( - "ZFS dataset {} has no Provisioned Volume bound; refusing to use it", - self.name - ) - .into()); - } - self.require_mountpoint(&name.mountpoint()) - } -} - -fn parse_zfs_bytes(value: &str) -> Result { - match value { - "none" | "-" => Ok(0), - _ => value - .parse() - .map_err(|_| VolumeError::from(format!("invalid byte count from ZFS: {value}"))), - } -} - -fn parse_size(options: &BTreeMap) -> Result { - if options.len() != 1 || !options.contains_key("size") { - return Err("Volume option size is required and is the only supported option".into()); - } - let value = options - .get("size") - .expect("the only accepted option is size"); - let (amount, suffix) = value.split_at(value.len().saturating_sub(1)); - let multiplier = match suffix { - "b" => 1, - "k" => 1024_u64, - "m" => 1024_u64.pow(2), - "g" => 1024_u64.pow(3), - "t" => 1024_u64.pow(4), - _ => { - return Err(format!( - "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" - ) - .into()); - } - }; - let amount = amount.parse::().map_err(|_| { - format!( - "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" - ) - })?; - if amount == 0 { - return Err("Volume size must be greater than zero".into()); - } - amount - .checked_mul(multiplier) - .ok_or_else(|| format!("Volume size {value:?} overflows bytes").into()) -} - -async fn checked_command(program: &PathBuf, args: &[&str]) -> Result { - let mut attempt = 0; - let output = loop { - match Command::new(program).args(args).output().await { - // ETXTBSY means exec never started; only that launch error is safe to retry. - Err(error) if error.kind() == io::ErrorKind::ExecutableFileBusy && attempt < 4 => { - tokio::time::sleep(std::time::Duration::from_millis(10 << attempt)).await; - attempt += 1; - } - output => { - break output - .map_err(|error| format!("could not run {}: {error}", program.display()))?; - } - } - }; - if !output.status.success() { - return Err(format!( - "{} {} failed: {}", - program.display(), - args.join(" "), - String::from_utf8_lossy(&output.stderr).trim() - ) - .into()); - } - String::from_utf8(output.stdout) - .map_err(|_| format!("{} returned non-UTF-8 output", program.display()).into()) -} - #[derive(Deserialize)] struct CreateRequest { #[serde(rename = "Name")] @@ -475,9 +134,17 @@ pub(super) fn inherited_listener() -> io::Result { /// # Errors /// /// Returns an error when the listener cannot become asynchronous or serving fails. -pub(super) async fn run(listener: StdUnixListener) -> io::Result<()> { +pub(super) async fn run( + listener: StdUnixListener, + data_dir: &std::path::Path, + run_dir: &std::path::Path, +) -> io::Result<()> { listener.set_nonblocking(true)?; - serve(UnixListener::from_std(listener)?, VolumeStorage::new()).await + serve( + UnixListener::from_std(listener)?, + VolumeStorage::new(data_dir, run_dir), + ) + .await } async fn serve(listener: UnixListener, storage: VolumeStorage) -> io::Result<()> { diff --git a/crates/ployzd/src/volume_plugin/capacity.rs b/crates/ployzd/src/volume_plugin/capacity.rs index b091bac650..51ab1d21aa 100644 --- a/crates/ployzd/src/volume_plugin/capacity.rs +++ b/crates/ployzd/src/volume_plugin/capacity.rs @@ -4,7 +4,7 @@ use axum::{Json, extract::State}; use ployz_core::{ProvisionedVolumeMaximumBytes, StorageCapacity, StorageCapacityError}; use std::collections::BTreeMap; -use super::{CapacityAdmission, DockerVolumeName, VolumeStorage}; +use super::{DockerVolumeName, VolumeStorage}; type Volumes = BTreeMap; @@ -65,11 +65,37 @@ impl VolumeStorage { }) } + async fn inspect_capacity(&self) -> Result { + let admission = self.admit_mutation().await.map_err(unknown)?; + let storage = self.clone(); + tokio::spawn(async move { + let _admission = admission; + let _pool_guard = storage.pool.lock_mutation().await.map_err(unknown)?; + storage.capacity().await.map_err(unknown) + }) + .await + .unwrap_or_else(|error| Err(unknown(error))) + } + async fn prepare( &self, requested: &Volumes, ) -> Result, ployz_core::RpcError> { - let _guard = self.mutation.lock().await; + let admission = self.admit_mutation().await.map_err(storage_error)?; + let storage = self.clone(); + let requested = requested.clone(); + tokio::spawn(async move { + let _admission = admission; + storage.prepare_admitted(&requested).await + }) + .await + .unwrap_or_else(|error| Err(unknown(error))) + } + + async fn prepare_admitted( + &self, + requested: &Volumes, + ) -> Result, ployz_core::RpcError> { let _pool_guard = self.pool.lock_mutation().await.map_err(storage_error)?; let capacity = self.capacity().await.map_err(unknown)?; let budget = capacity @@ -78,78 +104,53 @@ impl VolumeStorage { if requested.is_empty() { return Ok(Vec::new()); } + let existing_pool = self.pool.one_usable().await.map_err(storage_error)?; + if let Some(pool) = &existing_pool { + let datasets = self.datasets(pool).await.map_err(storage_error)?; + for (name, maximum) in requested { + let plugin_name = name + .as_str() + .parse::() + .map_err(storage_error)?; + if let Some(existing) = + Self::dataset(&datasets, pool, &plugin_name).map_err(storage_error)? + { + existing + .require_requested(&plugin_name, maximum.get()) + .map_err(storage_error)?; + } + } + } let commitment = capacity .volumes .values() .map(|maximum| maximum.get()) .try_fold(budget.additional_commitment_bytes, u64::checked_add) .ok_or_else(|| unknown("Volume commitments overflow u64"))?; - let pool = match self.pool.one_usable().await.map_err(storage_error)? { + match existing_pool { Some(pool) => { self.pool .ensure_capacity(&pool, commitment, capacity.unmanaged_used_bytes) .await .map_err(storage_error)?; - pool } None => { self.pool.create(commitment).await.map_err(storage_error)?; - self.one_pool().await.map_err(storage_error)? - } - }; - let mut prepared = Vec::new(); - for (name, maximum) in requested { - let plugin_name = name - .as_str() - .parse::() - .map_err(storage_error)?; - // The whole batch has physical backing; each dataset records its durable commitment. - if let Err(error) = self - .create_volume( - &pool, - &plugin_name, - maximum.get(), - CapacityAdmission::Ensured, - ) - .await - { - let mut error = storage_error(error); - error - .details - .as_object_mut() - .expect("storage errors have object details") - .insert("prepared_volumes".into(), serde_json::json!(prepared)); - return Err(error); } - prepared.push(name.clone()); } - Ok(prepared) + Ok(requested.keys().cloned().collect()) } } pub(super) async fn inspect( State(storage): State, ) -> Json> { - // Finish import recovery under the locks even if the observer disconnects. - Json( - tokio::spawn(async move { - let _guard = storage.mutation.lock().await; - let _pool_guard = storage.pool.lock_mutation().await.map_err(unknown)?; - storage.capacity().await.map_err(unknown) - }) - .await - .unwrap_or_else(|error| Err(unknown(error))), - ) + Json(storage.inspect_capacity().await) } pub(super) async fn prepare( State(storage): State, Json(requested): Json, ) -> Json, ployz_core::RpcError>> { - // Finish admitted allocation even if the requesting connection disappears. - Json( - tokio::spawn(async move { storage.prepare(&requested).await }) - .await - .unwrap_or_else(|error| Err(unknown(error))), - ) + Json(storage.prepare(&requested).await) } diff --git a/crates/ployzd/src/volume_plugin/pool.rs b/crates/ployzd/src/volume_plugin/pool.rs index cd1030d176..3dd750cded 100644 --- a/crates/ployzd/src/volume_plugin/pool.rs +++ b/crates/ployzd/src/volume_plugin/pool.rs @@ -70,7 +70,7 @@ impl VolumeStorage { options: &BTreeMap, ) -> Result<()> { let requested = parse_size(options)?; - let _guard = self.mutation.lock().await; + let _guard = self.admit_mutation().await?; let _pool_guard = self.pool.lock_mutation().await?; let existing = match self.pool.one_usable().await? { Some(pool) => Some(pool), diff --git a/crates/ployzd/src/volume_plugin/removal.rs b/crates/ployzd/src/volume_plugin/removal.rs index 8a055cfe39..b01f8af6c3 100644 --- a/crates/ployzd/src/volume_plugin/removal.rs +++ b/crates/ployzd/src/volume_plugin/removal.rs @@ -11,7 +11,7 @@ use super::{ impl VolumeStorage { async fn remove(&self, name: &DockerVolumeName) -> Result<()> { - let _guard = self.mutation.lock().await; + let _guard = self.admit_mutation().await?; let Some(pool) = self.pool.one_usable().await? else { return Ok(()); }; diff --git a/crates/ployzd/src/volume_plugin/storage.rs b/crates/ployzd/src/volume_plugin/storage.rs new file mode 100644 index 0000000000..a324026a4d --- /dev/null +++ b/crates/ployzd/src/volume_plugin/storage.rs @@ -0,0 +1,379 @@ +//! Durable ZFS Volume storage ownership and mutation admission. + +use std::{collections::BTreeMap, io, path::PathBuf, sync::Arc}; + +use ployzd::machine_pool::MachinePool; +use tokio::{ + process::Command, + sync::{Mutex, OwnedMutexGuard}, +}; + +use super::{DockerVolumeName, Result, VolumeError, pool::PoolStorage}; + +pub(super) const DATASET_ROOT: &str = "ployz"; +pub(super) const MOUNT_ROOT: &str = "/var/lib/ployz-volumes"; + +#[derive(Clone)] +pub(super) struct VolumeStorage { + pub(super) pool: PoolStorage, + pub(super) zfs: PathBuf, + pub(super) mutation: Arc>, + pub(super) installation: ployzd::mutation::MutationGate, +} + +pub(super) enum CapacityAdmission { + Required, + Ensured, +} + +impl VolumeStorage { + pub(super) fn new(data_dir: impl Into, run_dir: impl Into) -> Self { + Self { + pool: PoolStorage::new("zpool"), + zfs: "zfs".into(), + mutation: Arc::new(Mutex::new(())), + installation: ployzd::mutation::MutationGate::new(run_dir, data_dir), + } + } + + #[cfg(test)] + pub(super) fn with_programs(zpool: impl Into, zfs: impl Into) -> Self { + let zpool = zpool.into(); + let backing = zpool.with_file_name("machine-pool"); + let fixture = zpool + .parent() + .expect("test command has a fixture directory") + .to_owned(); + Self { + pool: PoolStorage::new(zpool).with_backing(backing), + zfs: zfs.into(), + mutation: Arc::new(Mutex::new(())), + installation: ployzd::mutation::MutationGate::new( + fixture.join("admission-run"), + fixture.join("admission-data"), + ), + } + } + + pub(super) async fn admit_mutation( + &self, + ) -> Result<(OwnedMutexGuard<()>, ployzd::mutation::MutationGuard)> { + let local = Arc::clone(&self.mutation).lock_owned().await; + let installation = self + .installation + .try_mutation() + .map_err(|error| VolumeError::from(error.to_string()))?; + Ok((local, installation)) + } + + pub(super) async fn create_volume( + &self, + pool: &MachinePool, + name: &DockerVolumeName, + requested: u64, + origin: CapacityAdmission, + ) -> Result<()> { + let datasets = self.datasets(pool).await?; + let root = format!("{}/{DATASET_ROOT}", pool.name()); + let volume = format!("{root}/{name}"); + + if let Some(existing) = Self::dataset(&datasets, pool, name)? { + return existing.require_requested(name, requested); + } + + if matches!(origin, CapacityAdmission::Required) { + let commitment = datasets + .iter() + .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) + .map(|dataset| dataset.refquota) + .try_fold(requested, u64::checked_add) + .ok_or_else(|| { + VolumeError::from("Provisioned Volume commitments overflowed u64") + })?; + let managed_used = datasets + .iter() + .filter(|dataset| dataset.name.starts_with(&format!("{root}/"))) + .map(|dataset| dataset.active_used_bytes) + .try_fold(0u64, u64::checked_add) + .ok_or("Dataset occupancy overflows u64")?; + self.pool + .ensure_capacity( + pool, + commitment, + pool.used_bytes().saturating_sub(managed_used), + ) + .await?; + } + + if !datasets.iter().any(|dataset| dataset.name == root) { + self.zfs(&[ + "create", + "-o", + "canmount=off", + "-o", + &format!("mountpoint={MOUNT_ROOT}"), + &root, + ]) + .await?; + } + self.zfs(&["create", "-o", &format!("refquota={requested}"), &volume]) + .await?; + Ok(()) + } + + pub(super) async fn mountpoint(&self, name: &DockerVolumeName) -> Result { + let _guard = self.admit_mutation().await?; + let pool = self.one_pool().await?; + let datasets = self.datasets(&pool).await?; + let dataset = Self::dataset(&datasets, &pool, name)? + .ok_or_else(|| format!("Provisioned Volume {name} does not exist"))?; + dataset.require_provisioned(name)?; + dataset.require_writable()?; + if dataset.mounted { + return Ok(dataset.mountpoint.clone()); + } + self.zfs(&["mount", &dataset.name]).await?; + let datasets = self.datasets(&pool).await?; + let dataset = Self::dataset(&datasets, &pool, name)? + .ok_or_else(|| format!("Provisioned Volume {name} disappeared while mounting"))?; + dataset.require_provisioned(name)?; + dataset.require_writable()?; + if !dataset.mounted { + return Err(format!("Provisioned Volume {name} did not mount").into()); + } + Ok(dataset.mountpoint.clone()) + } + + pub(super) async fn one_pool(&self) -> Result { + self.pool + .one_usable() + .await? + .ok_or_else(|| "no usable existing Machine Pool".into()) + } + + pub(super) async fn datasets(&self, pool: &MachinePool) -> Result> { + let output = self + .zfs(&[ + "list", + "-Hp", + "-o", + "name,refquota,used,usedbydataset,mountpoint,mounted,readonly", + "-r", + pool.name(), + ]) + .await?; + output + .lines() + .map(|line| Dataset::parse(line, pool.name())) + .collect() + } + + pub(super) fn dataset<'datasets>( + datasets: &'datasets [Dataset], + pool: &MachinePool, + name: &DockerVolumeName, + ) -> Result> { + let root_name = format!("{}/{DATASET_ROOT}", pool.name()); + let root = datasets.iter().find(|dataset| dataset.name == root_name); + if let Some(root) = root { + root.require_mountpoint(MOUNT_ROOT)?; + root.require_writable()?; + } + + let requested = format!("{}/{DATASET_ROOT}/{name}", pool.name()); + let descendant_prefix = format!("{requested}/"); + if let Some(dataset) = datasets + .iter() + .find(|dataset| dataset.name.starts_with(&descendant_prefix)) + { + return Err(format!( + "ZFS dataset {} is a descendant of Provisioned Volume {name}; remove it before retrying", + dataset.name + ) + .into()); + } + let dataset = datasets.iter().find(|dataset| dataset.name == requested); + if dataset.is_some() && root.is_none() { + return Err(format!("ZFS did not report managed root dataset {root_name}").into()); + } + Ok(dataset) + } + + pub(super) async fn zfs(&self, args: &[&str]) -> Result { + checked_command(&self.zfs, args).await + } +} + +pub(super) struct Dataset { + pub(super) name: String, + pub(super) refquota: u64, + pub(super) used_bytes: u64, + pub(super) active_used_bytes: u64, + pub(super) mountpoint: String, + pub(super) mounted: bool, + pub(super) readonly: bool, +} + +impl Dataset { + pub(super) fn parse(line: &str, pool: &str) -> Result { + let mut fields = line.split('\t'); + let invalid = || format!("invalid ZFS dataset output for Pool {pool}: {line}"); + let ( + Some(name), + Some(refquota), + Some(used_bytes), + Some(active_used_bytes), + Some(mountpoint), + Some(mounted), + Some(readonly), + None, + ) = ( + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + fields.next(), + ) + else { + return Err(invalid().into()); + }; + let used_bytes = used_bytes.parse::().map_err(|_| invalid())?; + let active_used_bytes = active_used_bytes.parse::().map_err(|_| invalid())?; + if active_used_bytes > used_bytes { + return Err(invalid().into()); + } + Ok(Self { + name: name.to_owned(), + refquota: parse_zfs_bytes(refquota)?, + used_bytes, + active_used_bytes, + mountpoint: mountpoint.to_owned(), + mounted: mounted == "yes", + readonly: match readonly { + "off" => false, + "on" => true, + _ => return Err(invalid().into()), + }, + }) + } + + pub(super) fn require_mountpoint(&self, expected: &str) -> Result<()> { + if self.mountpoint == expected { + return Ok(()); + } + Err(format!( + "ZFS dataset {} has incompatible mountpoint {}; set it to {expected} before retrying", + self.name, self.mountpoint + ) + .into()) + } + + pub(super) fn require_writable(&self) -> Result<()> { + if !self.readonly { + return Ok(()); + } + Err(format!( + "ZFS dataset {} is read-only; make it writable before retrying", + self.name + ) + .into()) + } + + pub(super) fn require_provisioned(&self, name: &DockerVolumeName) -> Result<()> { + if self.refquota == 0 { + return Err(format!( + "ZFS dataset {} has no Provisioned Volume bound; refusing to use it", + self.name + ) + .into()); + } + self.require_mountpoint(&name.mountpoint()) + } + + pub(super) fn require_requested(&self, name: &DockerVolumeName, requested: u64) -> Result<()> { + self.require_mountpoint(&name.mountpoint())?; + self.require_writable()?; + if self.refquota == requested { + return Ok(()); + } + Err(format!( + "Volume {name} already has a {}-byte bound; changing it to {requested} bytes is a separate update operation", + self.refquota + ) + .into()) + } +} + +fn parse_zfs_bytes(value: &str) -> Result { + match value { + "none" | "-" => Ok(0), + _ => value + .parse() + .map_err(|_| VolumeError::from(format!("invalid byte count from ZFS: {value}"))), + } +} + +pub(super) fn parse_size(options: &BTreeMap) -> Result { + if options.len() != 1 || !options.contains_key("size") { + return Err("Volume option size is required and is the only supported option".into()); + } + let value = options + .get("size") + .expect("the only accepted option is size"); + let (amount, suffix) = value.split_at(value.len().saturating_sub(1)); + let multiplier = match suffix { + "b" => 1, + "k" => 1024_u64, + "m" => 1024_u64.pow(2), + "g" => 1024_u64.pow(3), + "t" => 1024_u64.pow(4), + _ => { + return Err(format!( + "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" + ) + .into()); + } + }; + let amount = amount.parse::().map_err(|_| { + format!( + "invalid Volume size {value:?}; use a positive integer followed by b, k, m, g, or t" + ) + })?; + if amount == 0 { + return Err("Volume size must be greater than zero".into()); + } + amount + .checked_mul(multiplier) + .ok_or_else(|| format!("Volume size {value:?} overflows bytes").into()) +} + +pub(super) async fn checked_command(program: &PathBuf, args: &[&str]) -> Result { + let mut attempt = 0; + let output = loop { + match Command::new(program).args(args).output().await { + // ETXTBSY means exec never started; only that launch error is safe to retry. + Err(error) if error.kind() == io::ErrorKind::ExecutableFileBusy && attempt < 4 => { + tokio::time::sleep(std::time::Duration::from_millis(10 << attempt)).await; + attempt += 1; + } + output => { + break output + .map_err(|error| format!("could not run {}: {error}", program.display()))?; + } + } + }; + if !output.status.success() { + return Err(format!( + "{} {} failed: {}", + program.display(), + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + String::from_utf8(output.stdout) + .map_err(|_| format!("{} returned non-UTF-8 output", program.display()).into()) +} diff --git a/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs b/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs index d957e9503b..cdb2729bce 100644 --- a/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs +++ b/crates/ployzd/src/volume_plugin/tests/capacity_tests.rs @@ -71,7 +71,42 @@ async fn storage_inspection_preserves_unrecoverable_backing() { } #[tokio::test] -async fn batch_preparation_checks_total_before_allocating_and_reuses_committed_volumes() { +async fn batch_preparation_rejects_an_existing_read_only_volume_before_dataset_creation() { + let test = TestDir::new(); + for marker in ["root", "volume", "readonly-volume"] { + fs::write(test.0.join(marker), "").unwrap(); + } + let (zpool, zfs) = fake_zfs(&test.0, USABLE_POOL); + let socket = test.0.join("plugin.sock"); + let server = tokio::spawn(serve( + UnixListener::bind(&socket).unwrap(), + VolumeStorage::with_programs(zpool, zfs), + )); + + let response = post( + &socket, + "/Storage.Prepare", + json!({"data":1073741824u64,"other":1073741824u64}), + ) + .await; + + assert!( + response + .pointer("/Err/message") + .and_then(Value::as_str) + .is_some_and(|message| message.contains("read-only")), + "{response}" + ); + assert!( + !fs::read_to_string(test.0.join("commands")) + .unwrap() + .contains("zfs create") + ); + server.abort(); +} + +#[tokio::test] +async fn batch_preparation_reserves_capacity_before_docker_creates_volumes() { let test = TestDir::new(); let socket = test.0.join("plugin.sock"); let server = tokio::spawn(serve( @@ -103,6 +138,28 @@ async fn batch_preparation_checks_total_before_allocating_and_reuses_committed_v post(&socket, "/Storage.Prepare", requested.clone()).await, json!({"Ok":["data","other"]}) ); + assert_eq!( + post(&socket, "/VolumeDriver.Get", json!({"Name":"data"})).await, + json!({"Err":"Provisioned Volume data does not exist"}) + ); + assert_eq!( + post(&socket, "/VolumeDriver.List", json!({})).await, + json!({"Volumes":[],"Err":""}) + ); + let capacity = post(&socket, "/Storage.Inspect", json!(null)).await; + assert_eq!(capacity.pointer("/Ok/volumes").unwrap(), &json!({})); + + for (name, size) in [("data", "1073741824b"), ("other", "2147483648b")] { + assert_eq!( + post( + &socket, + "/VolumeDriver.Create", + json!({"Name":name,"Opts":{"size":size}}), + ) + .await, + json!({"Err":""}) + ); + } let capacity = post(&socket, "/Storage.Inspect", json!(null)).await; assert_eq!( capacity.pointer("/Ok/volumes").unwrap(), diff --git a/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs b/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs index b5493b8979..23178d2f0b 100644 --- a/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs +++ b/crates/ployzd/src/volume_plugin/tests/first_pool_tests.rs @@ -2,9 +2,12 @@ use std::{ os::unix::fs::{MetadataExt, PermissionsExt}, + sync::Arc, time::Duration, }; +use tokio::sync::Mutex; + use super::super::pool::{POOL_BACKING_FILE, PoolStorage}; use super::*; @@ -426,6 +429,7 @@ async fn concurrent_and_retried_first_creates_converge_on_one_pool() { pool: first.pool.clone(), zfs: first.zfs.clone(), mutation: Arc::new(Mutex::new(())), + installation: first.installation.clone(), }; let first_socket = test.0.join("first-plugin.sock"); let second_socket = test.0.join("second-plugin.sock"); @@ -491,6 +495,7 @@ async fn a_second_process_cannot_use_a_pool_before_its_owner_finishes() { pool: first.pool.clone(), zfs: first.zfs.clone(), mutation: Arc::new(Mutex::new(())), + installation: first.installation.clone(), }; let first_socket = test.0.join("first-plugin.sock"); let second_socket = test.0.join("second-plugin.sock"); @@ -956,5 +961,9 @@ esac ), zfs: program("zfs"), mutation: Arc::new(Mutex::new(())), + installation: ployzd::mutation::MutationGate::new( + directory.join("admission-run"), + directory.join("admission-data"), + ), } } diff --git a/crates/ployzd/tests/machine_record.rs b/crates/ployzd/tests/machine_record.rs index 773278bdd6..42e9e88ec9 100644 --- a/crates/ployzd/tests/machine_record.rs +++ b/crates/ployzd/tests/machine_record.rs @@ -130,8 +130,8 @@ fn sample_cloud_pairing() -> CloudPairing { .unwrap() } -#[test] -fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { +#[tokio::test] +async fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { let dir = TestDir::new("ployzd-initialize-cloud-pairing"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); @@ -147,6 +147,7 @@ fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { wireguard_mtu: None, cloud_pairing: Some(pairing.clone()), }) + .await .unwrap(); assert_eq!( @@ -171,8 +172,8 @@ fn initialize_with_cloud_pairing_stores_relay_url_and_pairing_credential() { assert!(pairing_json.get("dial").is_none()); } -#[test] -fn set_cloud_pairing_after_initialize_persists() { +#[tokio::test] +async fn set_cloud_pairing_after_initialize_persists() { let dir = TestDir::new("ployzd-set-cloud-pairing"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); @@ -188,10 +189,14 @@ fn set_cloud_pairing_after_initialize_persists() { wireguard_mtu: None, cloud_pairing: None, }) + .await .unwrap(); assert_eq!(local.record().unwrap().cloud_pairing, None); - local.set_cloud_pairing(Some(pairing.clone())).unwrap(); + local + .set_cloud_pairing(Some(pairing.clone())) + .await + .unwrap(); assert_eq!( local.record().unwrap().cloud_pairing.as_ref(), Some(&pairing) @@ -201,8 +206,8 @@ fn set_cloud_pairing_after_initialize_persists() { assert_eq!(reopened.record().cloud_pairing.as_ref(), Some(&pairing)); } -#[test] -fn set_cloud_pairing_none_clears_persisted_pairing() { +#[tokio::test] +async fn set_cloud_pairing_none_clears_persisted_pairing() { let dir = TestDir::new("ployzd-clear-cloud-pairing"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); @@ -216,28 +221,30 @@ fn set_cloud_pairing_none_clears_persisted_pairing() { wireguard_mtu: None, cloud_pairing: Some(sample_cloud_pairing()), }) + .await .unwrap(); - local.set_cloud_pairing(None).unwrap(); + local.set_cloud_pairing(None).await.unwrap(); assert_eq!(local.record().unwrap().cloud_pairing, None); drop(local); let reopened = LocalMachineStore::open(&dir.0).unwrap(); assert_eq!(reopened.record().cloud_pairing, None); } -#[test] -fn set_cloud_pairing_before_initialize_is_not_participating() { +#[tokio::test] +async fn set_cloud_pairing_before_initialize_is_not_participating() { let dir = TestDir::new("ployzd-set-cloud-pairing-uninitialized"); let store = LocalMachineStore::open(&dir.0).unwrap(); let (reset, _) = tokio::sync::watch::channel(false); let local = LocalMachine::new(Arc::new(Mutex::new(store)), reset); let error = local .set_cloud_pairing(Some(sample_cloud_pairing())) + .await .unwrap_err(); assert!(matches!(error, LocalMachineError::NotParticipating)); } -#[test] -fn join_with_cloud_pairing_stores_the_same_two_fields() { +#[tokio::test] +async fn join_with_cloud_pairing_stores_the_same_two_fields() { let first_dir = TestDir::new("ployzd-join-cloud-pairing-first"); let mut first = LocalMachineStore::open(&first_dir.0).unwrap(); let initialized = first @@ -278,6 +285,7 @@ fn join_with_cloud_pairing_stores_the_same_two_fields() { wireguard_mtu: None, cloud_pairing: Some(pairing.clone()), }) + .await .unwrap(); assert_eq!( @@ -440,6 +448,7 @@ async fn inspect_reports_stored_cloud_pairing_without_the_secret() { wireguard_mtu: None, cloud_pairing: Some(sample_cloud_pairing()), }) + .await .unwrap(); let details = local.inspect(InspectRequest::default()).await.unwrap(); diff --git a/docs/RELEASE.md b/docs/RELEASE.md index e19ed90689..d1fad38f2a 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -35,7 +35,7 @@ Protected `main` pushes warm the shared R2 cache, including release archives and Fast CI on `main` is the merge gate. The ignored cluster suite is informing. A red nightly there does not block a tag or Publish. -Before Publish, run `scripts/qualify-release.sh` against real Linux Machines using the draft musl archives (`PLOYZ_ARTIFACT_DIR`). That script does not pick a cloud vendor. You pass SSH targets. Those hosts must be uninitialized Machines unless you set `PLOYZ_QUALIFY_RESET=1`, which accepts a reset and destroys managed containers. Pass a qualification key with `PLOYZ_QUALIFY_SSH_KEY` or `-i` in `PLOYZ_QUALIFY_SSH_OPTS` so `ssh` and `ployz machine init`/`add` use the same identity. +Before Publish, run `scripts/qualify-release.sh` against real Linux Machines using two draft musl archive sets: `PLOYZ_ARTIFACT_DIR` is the source release and `PLOYZ_UPGRADE_ARTIFACT_DIR` is the target release. The versions must differ. The script does not pick a cloud vendor. You pass SSH targets. Those hosts must be uninitialized Machines unless you set `PLOYZ_QUALIFY_RESET=1`, which accepts a reset and destroys managed containers. Pass a qualification key with `PLOYZ_QUALIFY_SSH_KEY`; the normal `ployz machine init`/`add` path installs the verified source, then the qualifier proves a target upgrade through client disconnection, persistent ZFS-backed traffic, corrupt preflight rejection, failed activation evidence, and explicit previous-binary repair. When the informing cluster suite and that run disagree, the real Machines are the authority. Testkit bugs do not block a release. @@ -85,7 +85,7 @@ Goreleaser does not touch the tap (`--skip=homebrew`); `scripts/promote-release. ## Machine daemon -`scripts/install.sh` on Linux. Same version tokens: `latest` / `stable` / `beta` / pin. Set `PLOYZ_VERSION`. +`ployzd install` on Linux installs or replaces a Machine daemon. It accepts `--version stable`, `--version beta`, or an exact version; use `--software-only` for ordinary replacement after the Machine has already been prepared. Setup downloads and verifies the CLI release's daemon as a temporary bootstrap, then that daemon installs the selected Machine release through this interface. ## Cloud Relay diff --git a/evidence/product-paths.tsv b/evidence/product-paths.tsv new file mode 100644 index 0000000000..65173f8970 --- /dev/null +++ b/evidence/product-paths.tsv @@ -0,0 +1,4 @@ +path rung1 rung2 rung3 rung4 rung5 +machine-installer crates/ployzd/src/installer/mod.rs::tests::{installation_interface_contract,zfs_candidate_download_contract}; crates/ployzd/src/installer/release.rs::tests - - - ployzd install --software-only with a foreign-owned freshly built musl release on the retained Ubuntu systemd Machine +machine-setup crates/ployz/src/provisioning.rs::tests; scripts/test-{missing-ssh-client,qualify-release}.sh crates/ployz/tests/cloud_init/daemon_sync.rs crates/ployz/tests/setup_cli.rs - scripts/qualify-release.sh with verified musl CLI and daemon archives on real Machines +machine-upgrade crates/ployzd/src/installer/upgrade.rs::tests::upgrade_attempt_contract; crates/ployzd/src/machine_api/local_tests.rs::upgrade_and_machine_mutations_refuse_each_other_at_the_rpc_boundary crates/ployz-core/tests/{contract,capabilities}.rs crates/ployz/tests/cli_shape.rs::machine_upgrade_requires_explicit_targets_and_has_typed_inspection - scripts/qualify-release.sh with real source/target musl releases, independent worker, disconnected client, persistent ZFS traffic, corrupt preflight, failed activation, and explicit repair diff --git a/scripts/install.sh b/scripts/install.sh deleted file mode 100755 index f64cd1589f..0000000000 --- a/scripts/install.sh +++ /dev/null @@ -1,587 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -export LC_ALL="${LC_ALL:-C.UTF-8}" - -INSTALL_ONLY=${INSTALL_ONLY:-false} -INSTALL_BIN_DIR=${INSTALL_BIN_DIR:-/usr/local/bin} -INSTALL_SYSTEMD_DIR=${INSTALL_SYSTEMD_DIR:-/etc/systemd/system} -PLOYZ_GITHUB_URL=${PLOYZ_GITHUB_URL:-https://github.com/getployz/ployz2} -PLOYZ_CHANNEL_URL=${PLOYZ_CHANNEL_URL:-https://ployz.sh} -PLOYZ_VERSION=${PLOYZ_VERSION:-latest} -PLOYZ_VERSION=${PLOYZ_VERSION#v} -PLOYZ_USER=ployz -PLOYZ_GROUP_ADD_USER=${PLOYZ_GROUP_ADD_USER:-} -PLOYZ_DATA_DIR=${PLOYZ_DATA_DIR:-/var/lib/ployz} -PLOYZ_RUN_DIR=${PLOYZ_RUN_DIR:-/run/ployz} -PLOYZ_STORAGE=${PLOYZ_STORAGE:-none} -DOCKER_DAEMON_CONFIG_FILE=${DOCKER_DAEMON_CONFIG_FILE:-/etc/docker/daemon.json} -APT_LOCK_TIMEOUT_SECONDS=300 -PLOYZ_APT_CONFIG= -DAEMON_REPLACED=false -DOCKER_DAEMON_CONFIG='{ - "features": { "containerd-snapshotter": true }, - "live-restore": true, - "log-driver": "json-file", - "log-opts": { "max-size": "10m", "max-file": "3" } -}' - -log() { echo "$1"; } -warning() { echo "WARNING: $1" >&2; } -error() { echo "ERROR: $1" >&2; exit 1; } -command_exists() { command -v "$1" >/dev/null 2>&1; } - -configure_apt_lock_wait() { - local inherited_apt_config=${APT_CONFIG:-} - PLOYZ_APT_CONFIG=$(mktemp) - if [ -n "$inherited_apt_config" ]; then - cat "$inherited_apt_config" > "$PLOYZ_APT_CONFIG" - printf '\n' >> "$PLOYZ_APT_CONFIG" - fi - printf 'DPkg::Lock::Timeout "%s";\n' "$APT_LOCK_TIMEOUT_SECONDS" >> "$PLOYZ_APT_CONFIG" - export APT_CONFIG="$PLOYZ_APT_CONFIG" -} - -run_with_apt_lock_wait() { - local deadline=0 error_dir error_log status tee_pid - while true; do - error_dir=$(mktemp -d) - error_log=$error_dir/stderr - mkfifo "$error_dir/pipe" - tee "$error_log" < "$error_dir/pipe" >&2 & - tee_pid=$! - if LC_ALL=C "$@" 2> "$error_dir/pipe"; then - status=0 - else - status=$? - fi - wait "$tee_pid" - if [ "$status" -eq 0 ]; then - rm -rf "$error_dir" - return 0 - fi - if grep -Eq '^E: Could not get lock .*/(lists|archives)/lock\. It is held by process' "$error_log"; then - [ "$deadline" -ne 0 ] || deadline=$((SECONDS + APT_LOCK_TIMEOUT_SECONDS)) - if [ "$SECONDS" -lt "$deadline" ]; then - rm -rf "$error_dir" - sleep 1 - continue - fi - fi - break - done - if grep -Eq '^E: (Could not get lock|Unable to acquire .* lock)' "$error_log"; then - rm -rf "$error_dir" - error "The package-manager lock named above stayed busy for five minutes. Let its owner finish, then retry the installer." - fi - rm -rf "$error_dir" - return "$status" -} - -channel_version_from_file() { - local version - version=$(tr -d ' \t\r\n' < "$1") - echo "$version" | grep -Eq '^v?[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$' || return 1 - echo "$version" -} - -resolve_install() { - local requested=${1#v} dest resolved name - case "$requested" in - latest | stable | '') name=stable ;; - beta) name=beta ;; - *) - printf '%s\n' "$requested" - return 0 - ;; - esac - dest=$(mktemp) - if ! curl -fsSL -o "$dest" "$PLOYZ_CHANNEL_URL/$name" || ! resolved=$(channel_version_from_file "$dest"); then - rm -f "$dest" - error "$name channel is unavailable" - fi - rm -f "$dest" - printf '%s\n' "${resolved#v}" -} - -daemon_archive() { - case "$1" in - x86_64) echo ployzd_linux_amd64.tar.gz ;; - aarch64) echo ployzd_linux_arm64.tar.gz ;; - *) return 1 ;; - esac -} - -daemon_action() { - local installed=$1 target=$2 mode=$3 - if [ -z "$installed" ]; then - echo replace - elif [ "$mode" = pin ]; then - [ "$installed" = "$target" ] && echo keep || echo replace - elif [ "$installed" = "$target" ]; then - echo keep - else - local newest - newest=$(printf '%s\n%s\n' "${installed//-/\~}" "$target" | sort -V | tail -n1) - [ "$newest" = "$target" ] && echo replace || echo keep - fi -} - -verify_system() { - [ "$(uname -s)" = Linux ] || error "Ployz Machine must be Linux" - daemon_archive "$(uname -m)" >/dev/null || error "Unsupported architecture: $(uname -m)" - if [ ! -d /run/systemd/system ] && [ "$INSTALL_ONLY" != true ]; then - error "Ployz requires systemd" - fi -} - -install_prerequisites() { - command_exists curl && return - if command_exists apt-get; then - run_with_apt_lock_wait apt-get update -qq >/dev/null - run_with_apt_lock_wait env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq curl ca-certificates - elif command_exists dnf; then dnf install -y curl ca-certificates - elif command_exists yum; then yum install -y curl ca-certificates - elif command_exists pacman; then pacman -Sy --noconfirm curl ca-certificates - elif command_exists zypper; then zypper --non-interactive install curl ca-certificates - else error "curl is required and no supported package manager was found" - fi -} - -operating_system_id() { - local id - [ -r /etc/os-release ] || return 1 - id=$(sed -n 's/^ID=//p' /etc/os-release | head -n1) - id=${id#\"} - id=${id%\"} - [ -n "$id" ] || return 1 - printf '%s\n' "$id" -} - -container_virtualization() { - systemd-detect-virt --container 2>/dev/null || true -} - -lxc_is_unprivileged() { - local inside outside _length - read -r inside outside _length < /proc/self/uid_map || return 1 - [ "$inside" = 0 ] && [ "$outside" != 0 ] -} - -zfs_smoke_bytes() { - echo 134217728 -} - -require_host_root_reserve() { - local allocation=$1 root_size='' root_available='' reserve - read -r root_size root_available < <(df -B1 --output=size,avail / | tail -n1) || \ - error "Could not inspect host-root capacity for ZFS storage preparation" - case "$root_size:$root_available" in - *[!0-9:]* | :* | *:) error "Could not read host-root capacity for ZFS storage preparation" ;; - esac - reserve=$((root_size / 4)) - [ "$reserve" -ge 10737418240 ] || reserve=10737418240 - if [ "$root_available" -lt $((reserve + allocation)) ]; then - error "Host root has ${root_available} bytes available; ZFS validation needs ${allocation} bytes while preserving the ${reserve}-byte host-root reserve" - fi -} - -package_file_list_has_zfs_module() { - local kernel=$1 - grep -F "/lib/modules/$kernel/" | grep -E '/zfs\.ko(\.[^/]*)?$' >/dev/null -} - -ubuntu_zfs_module_package() { - local kernel=$1 package package_dir archive download_failure='' archive_failure='' - for package in \ - "linux-main-modules-zfs-$kernel" \ - "linux-modules-zfs-$kernel" \ - "linux-modules-$kernel" \ - "linux-modules-extra-$kernel"; do - apt-cache show "$package" >/dev/null 2>&1 || continue - if dpkg-query -L "$package" 2>/dev/null | package_file_list_has_zfs_module "$kernel"; then - printf '%s\n' "$package" - return - fi - package_dir=$(mktemp -d) - if (cd "$package_dir" && run_with_apt_lock_wait apt-get download "$package" >/dev/null); then - archive=$(find "$package_dir" -maxdepth 1 -type f -name '*.deb' -print -quit) - if [ -z "$archive" ] || ! dpkg-deb -c "$archive" > "$package_dir/files"; then - archive_failure=$package - elif package_file_list_has_zfs_module "$kernel" < "$package_dir/files"; then - rm -rf "$package_dir" - printf '%s\n' "$package" - return - fi - else - download_failure=$package - fi - rm -rf "$package_dir" - done - if [ -n "$archive_failure" ]; then - error "Could not inspect downloaded running-kernel module package $archive_failure for ZFS support" - fi - if [ -n "$download_failure" ]; then - error "Could not download running-kernel module package $download_failure to inspect it for ZFS support" - fi - error "Ubuntu has no packaged ZFS module for the running kernel $kernel; install a supported Ubuntu kernel and retry" -} - -install_zfs_packages_ubuntu() { - local kernel=$1 module_package - if ! run_with_apt_lock_wait apt-get update -qq >/dev/null; then - error "Could not refresh Ubuntu packages needed for ZFS" - fi - if ! module_package=$(ubuntu_zfs_module_package "$kernel"); then - return 1 - fi - if ! run_with_apt_lock_wait env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends zfsutils-linux "$module_package"; then - error "Could not install zfsutils-linux and running-kernel module package $module_package" - fi - if ! dpkg-query -L "$module_package" | package_file_list_has_zfs_module "$kernel"; then - error "Installed package $module_package does not supply the ZFS module for running kernel $kernel" - fi -} - -zfs_arc_max_for_memory_kib() { - local memory_kib=$1 cap - cap=$((memory_kib * 1024 / 4)) - [ "$cap" -ge 268435456 ] || cap=268435456 - [ "$cap" -le 1073741824 ] || cap=1073741824 - printf '%s\n' "$cap" -} - -zfs_arc_max() { - local memory_kib - memory_kib=$(awk '$1 == "MemTotal:" { print $2; exit }' /proc/meminfo) - case "$memory_kib" in '' | *[!0-9]*) error "Could not read total RAM for the ZFS ARC limit" ;; esac - zfs_arc_max_for_memory_kib "$memory_kib" -} - -persist_zfs_arc_max() { - local cap=$1 config - config=$(mktemp) - printf 'options zfs zfs_arc_max=%s\n' "$cap" > "$config" - if ! install -D -m 0644 "$config" /etc/modprobe.d/ployz-zfs.conf; then - rm -f "$config" - error "Could not persist the ZFS ARC limit in /etc/modprobe.d/ployz-zfs.conf" - fi - rm -f "$config" -} - -set_and_verify_zfs_arc_max() { - local cap=$1 parameter=/sys/module/zfs/parameters/zfs_arc_max observed - [ -w "$parameter" ] || error "Loaded ZFS module does not expose a writable zfs_arc_max parameter" - printf '%s\n' "$cap" > "$parameter" || error "Could not apply zfs_arc_max=$cap to the loaded ZFS module" - observed=$(cat "$parameter") || error "Could not read zfs_arc_max from the loaded ZFS module" - [ "$observed" = "$cap" ] || \ - error "Loaded ZFS module reports zfs_arc_max=$observed instead of the required $cap" -} - -cleanup_zfs_smoke() { - local pool=$1 backing=$2 pool_created=$3 failure=$4 - local pool_names dataset_names mount_sources - local cleanup_failure='' - - if [ "$pool_created" = true ]; then - if ! zpool destroy -f "$pool"; then - cleanup_failure="Could not destroy temporary ZFS smoke Pool $pool; destroy it, then remove $backing" - fi - elif ! pool_names=$(zpool list -H -o name 2>&1); then - cleanup_failure="Could not verify whether temporary ZFS smoke Pool $pool remains; inspect and destroy it, then remove $backing: ${pool_names:-zpool returned no diagnostic output}" - elif printf '%s\n' "$pool_names" | grep -Fxq "$pool" && ! zpool destroy -f "$pool"; then - cleanup_failure="Could not destroy temporary ZFS smoke Pool $pool; destroy it, then remove $backing" - fi - if [ -z "$cleanup_failure" ]; then - if ! pool_names=$(zpool list -H -o name 2>&1); then - cleanup_failure="Could not verify cleanup of temporary ZFS smoke Pool $pool; inspect and destroy it, then remove $backing: ${pool_names:-zpool returned no diagnostic output}" - elif printf '%s\n' "$pool_names" | grep -Fxq "$pool"; then - cleanup_failure="Temporary ZFS smoke Pool $pool remains after destroy; destroy it, then remove $backing" - elif ! dataset_names=$(zfs list -H -o name 2>&1); then - cleanup_failure="Could not verify cleanup of temporary ZFS smoke dataset $pool; inspect and destroy its Pool, then remove $backing: ${dataset_names:-zfs returned no diagnostic output}" - elif printf '%s\n' "$dataset_names" | awk -v pool="$pool" \ - '$0 == pool || index($0, pool "/") == 1 { found = 1 } END { exit !found }'; then - cleanup_failure="Temporary ZFS smoke dataset $pool remains after destroy; destroy its Pool, then remove $backing" - elif ! mount_sources=$(awk ' - { for (i = 1; i <= NF; i++) if ($i == "-" && $(i + 1) == "zfs") print $(i + 2) } - ' /proc/self/mountinfo 2>&1); then - cleanup_failure="Could not verify cleanup of temporary ZFS smoke mounts for $pool; inspect and unmount them, then remove $backing: ${mount_sources:-mount inspection returned no diagnostic output}" - elif printf '%s\n' "$mount_sources" | awk -v pool="$pool" \ - '$0 == pool || index($0, pool "/") == 1 { found = 1 } END { exit !found }'; then - cleanup_failure="Temporary ZFS smoke mount for $pool remains after destroy; unmount it, destroy the Pool, then remove $backing" - elif ! rm -f "$backing"; then - cleanup_failure="Could not remove temporary ZFS smoke backing file $backing" - fi - fi - if [ -z "$cleanup_failure" ] && [ -e "$backing" ]; then - cleanup_failure="Temporary ZFS smoke backing file $backing remains after cleanup" - fi - if [ -n "$cleanup_failure" ]; then - [ -z "$failure" ] || cleanup_failure="$cleanup_failure (after: $failure)" - error "$cleanup_failure" - fi - [ -z "$failure" ] || error "$failure" -} - -restore_zfs_smoke_trap() { - local saved=$1 signal=$2 - if [ -n "$saved" ]; then - eval "$saved" - else - trap - "$signal" - fi -} - -interrupt_zfs_smoke() { - local signal=$1 pool=$2 backing=$3 pool_created=$4 - local previous_int_trap=$5 previous_term_trap=$6 - trap - INT TERM - [ -z "$backing" ] || cleanup_zfs_smoke "$pool" "$backing" "$pool_created" '' - restore_zfs_smoke_trap "$previous_int_trap" INT - restore_zfs_smoke_trap "$previous_term_trap" TERM - kill -s "$signal" "${BASHPID:-$$}" -} - -validate_zfs() { - local bytes backing='' pool blocks block_size allocated mount_target - local previous_int_trap previous_term_trap - local pool_created=false failure='' - bytes=$(zfs_smoke_bytes) - require_host_root_reserve "$bytes" - pool="ployz-smoke-${BASHPID:-$$}-${RANDOM}" - previous_int_trap=$(trap -p INT) - previous_term_trap=$(trap -p TERM) - trap 'interrupt_zfs_smoke INT "$pool" "$backing" "$pool_created" "$previous_int_trap" "$previous_term_trap"' INT - trap 'interrupt_zfs_smoke TERM "$pool" "$backing" "$pool_created" "$previous_int_trap" "$previous_term_trap"' TERM - if ! backing=$(mktemp /var/tmp/ployz-zfs-smoke.XXXXXX); then - error "Could not create a temporary host-root backing file for ZFS validation" - fi - - if ! fallocate -l "$bytes" "$backing"; then - failure="Could not preallocate the non-sparse ZFS smoke backing file $backing" - elif ! read -r blocks block_size < <(stat -c '%b %B' "$backing"); then - failure="Could not verify allocation of ZFS smoke backing file $backing" - else - allocated=$((blocks * block_size)) - [ "$allocated" -ge "$bytes" ] || \ - failure="ZFS smoke backing file $backing is sparse: $allocated of $bytes bytes are allocated" - fi - if [ -z "$failure" ]; then - mount_target=$(findmnt -n -o TARGET -T "$backing" 2>/dev/null || true) - [ "$mount_target" = / ] || \ - failure="ZFS smoke backing file $backing is on $mount_target instead of the host root filesystem" - fi - if [ -z "$failure" ]; then - if zpool create -f -m none -o cachefile=none "$pool" "$backing"; then - pool_created=true - else - failure="Could not create temporary ZFS smoke Pool $pool on $backing" - fi - fi - if [ -z "$failure" ] && ! zpool list -Hp -o name,size,alloc,free "$pool" >/dev/null; then - failure="Could not query temporary ZFS smoke Pool $pool" - fi - if [ -z "$failure" ] && ! zfs list -Hp -o name,mountpoint "$pool" >/dev/null; then - failure="Could not query temporary ZFS smoke dataset $pool" - fi - - cleanup_zfs_smoke "$pool" "$backing" "$pool_created" "$failure" - restore_zfs_smoke_trap "$previous_int_trap" INT - restore_zfs_smoke_trap "$previous_term_trap" TERM -} - -prepare_zfs() { - local os_id container kernel cap - os_id=$(operating_system_id) || error "Could not identify the Linux distribution for ZFS storage preparation" - [ "$os_id" = ubuntu ] || error "ZFS storage preparation is not supported on $os_id yet; use a supported Ubuntu release" - container=$(container_virtualization) - if [ "$container" = openvz ] || { [ -d /proc/vz ] && [ ! -d /proc/bc ]; }; then - error "OpenVZ does not allow this Machine to load the host ZFS kernel module" - fi - if [ "$container" = lxc ] && lxc_is_unprivileged; then - error "Unprivileged LXC does not allow this Machine to load the host ZFS kernel module" - fi - command_exists apt-get || error "Ubuntu apt-get is required for ZFS storage preparation" - kernel=$(uname -r) - require_host_root_reserve "$(zfs_smoke_bytes)" - cap=$(zfs_arc_max) - persist_zfs_arc_max "$cap" - install_zfs_packages_ubuntu "$kernel" - modprobe zfs || error "modprobe zfs failed for running kernel $kernel; verify kernel module support and container privileges" - set_and_verify_zfs_arc_max "$cap" - validate_zfs - log "ZFS storage preparation validated; no Machine Pool was created" -} - -prepare_storage() { - case "$PLOYZ_STORAGE" in - none) return ;; - zfs) prepare_zfs ;; - *) error "Unsupported storage preparation '$PLOYZ_STORAGE'; expected none or zfs" ;; - esac -} - -install_docker() { - if command_exists dockerd; then - if [ "$INSTALL_ONLY" != true ] && ! docker info -f '{{ .DriverStatus }}' 2>/dev/null | grep -q io.containerd.snapshotter; then - warning "Docker is retained unchanged; enable its containerd image store for best results" - fi - return - fi - run_with_apt_lock_wait bash -o pipefail -c 'curl -fsSL https://get.docker.com | sh' - mkdir -p "$(dirname "$DOCKER_DAEMON_CONFIG_FILE")" - printf '%s\n' "$DOCKER_DAEMON_CONFIG" > "$DOCKER_DAEMON_CONFIG_FILE" - [ "$INSTALL_ONLY" = true ] || systemctl restart docker -} - -create_user_and_directories() { - if ! id "$PLOYZ_USER" >/dev/null 2>&1; then - useradd --system --home-dir /nonexistent --shell /usr/sbin/nologin --user-group "$PLOYZ_USER" - fi - if [ -n "$PLOYZ_GROUP_ADD_USER" ]; then - gpasswd --add "$PLOYZ_GROUP_ADD_USER" "$PLOYZ_USER" >/dev/null - fi - install -d -m 0750 -o "$PLOYZ_USER" -g "$PLOYZ_USER" "$PLOYZ_DATA_DIR" "$PLOYZ_RUN_DIR" -} - -install_binaries() { - local archive installed_version target action base_url tmp_dir requested mode - archive=$(daemon_archive "$(uname -m)") || error "Unsupported architecture: $(uname -m)" - requested=$PLOYZ_VERSION - case "$requested" in - latest | stable | beta | '') mode=floating ;; - *) mode=pin ;; - esac - target=$(resolve_install "$requested") - installed_version="" - if [ -x "$INSTALL_BIN_DIR/ployzd" ]; then - installed_version=$("$INSTALL_BIN_DIR/ployzd" version 2>/dev/null || true) - fi - - base_url="$PLOYZ_GITHUB_URL/releases/download/v$target" - - action=$(daemon_action "$installed_version" "$target" "$mode") - if [ -n "${PLOYZ_RELEASE_DIR:-}" ]; then - action=replace - fi - if [ "$action" = keep ]; then - log "ployzd ${installed_version} retained" - return - fi - - tmp_dir=$(mktemp -d) - # shellcheck disable=SC2064 - trap "rm -rf '$tmp_dir'" EXIT - # Daemon downloads currently rely on TLS; published checksums are not verified here. - if [ -n "${PLOYZ_RELEASE_DIR:-}" ]; then - cp "$PLOYZ_RELEASE_DIR/$archive" "$tmp_dir/$archive" || error "PLOYZ_RELEASE_DIR is missing $archive" - else - curl -fsSL -o "$tmp_dir/$archive" "$base_url/$archive" || error "Failed to download $archive" - fi - tar -xzf "$tmp_dir/$archive" -C "$tmp_dir" - install -m 0755 "$tmp_dir/ployzd" "$INSTALL_BIN_DIR/ployzd" - install -m 0755 "$tmp_dir/ployz-uninstall" "$INSTALL_BIN_DIR/ployz-uninstall" - DAEMON_REPLACED=true - # Machines deliberately receive no CLI binary or alias. -} - -install_systemd() { - cat > "$INSTALL_SYSTEMD_DIR/ployz.service" < "$INSTALL_SYSTEMD_DIR/ployz-volume-plugin.socket" < "$INSTALL_SYSTEMD_DIR/ployz-volume-plugin.service" < checksums.txt + sha256sum $cli_archives $daemon_archives | sort -k2 > checksums.txt ) write_homebrew_formula_from_checksums "$DIST/checksums.txt" "$DIST/homebrew/ployz.rb" "$version" "$tag" "$repo" diff --git a/scripts/qualify-release.sh b/scripts/qualify-release.sh index a7c0d630ec..6c08a0b3ce 100755 --- a/scripts/qualify-release.sh +++ b/scripts/qualify-release.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -# Qualify draft musl archives against real Linux Machines. +# Qualify two draft musl releases against real Linux Machines. # Pass SSH targets. This script does not pick a cloud vendor. # # PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' \ -# PLOYZ_ARTIFACT_DIR=/path/to/dist \ +# PLOYZ_ARTIFACT_DIR=/path/to/source/dist \ +# PLOYZ_UPGRADE_ARTIFACT_DIR=/path/to/target/dist \ # scripts/qualify-release.sh # -# Optional: PLOYZ_QUALIFY_DRY_RUN=1, PLOYZ_QUALIFY_SSH_OPTS, PLOYZ_QUALIFY_SSH_KEY, +# Optional: PLOYZ_QUALIFY_DRY_RUN=1, PLOYZ_QUALIFY_SSH_KEY, # PLOYZ_QUALIFY_CONTEXT, PLOYZ_QUALIFY_RESET=1. # Hosts must be uninitialized unless PLOYZ_QUALIFY_RESET=1. Reset destroys # managed containers on that Machine. @@ -17,16 +18,44 @@ ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) COMPOSE_DIR=$ROOT/scripts/qualify-release HOSTS=${PLOYZ_QUALIFY_HOSTS:-} ARTIFACT_DIR=${PLOYZ_ARTIFACT_DIR:-} +UPGRADE_ARTIFACT_DIR=${PLOYZ_UPGRADE_ARTIFACT_DIR:-} DRY_RUN=${PLOYZ_QUALIFY_DRY_RUN:-0} -SSH_OPTS=${PLOYZ_QUALIFY_SSH_OPTS:-"-o StrictHostKeyChecking=accept-new"} CONTEXT=${PLOYZ_QUALIFY_CONTEXT:-qualify} RESET=${PLOYZ_QUALIFY_RESET:-0} +SSH_KEY=${PLOYZ_QUALIFY_SSH_KEY:-} CONFIG_DIR= +TRAFFIC_PID= +ENROLL_FIXTURE_PID= +ENROLL_PORT= +TRAFFIC_STOP=/var/lib/ployz/qualification/traffic.stop +REMOTE_RELEASE_ROOT=/var/lib/ployz/qualification/releases +APP_URL=http://127.0.0.1:18082/identity +APP_VALUE=qualify-persistent-data +APP_VOLUME=qualify-release_qualify-data +APP_VOLUME_MOUNT=/var/lib/ployz-volumes/$APP_VOLUME +PAIRING='{"relayUrl":"http://127.0.0.1:1/","secret":"qualification-synthetic-pairing-secret-v1"}' error() { echo "ERROR: $*" >&2; exit 1; } -need_archive() { - [ -f "$ARTIFACT_DIR/$1" ] || error "PLOYZ_ARTIFACT_DIR is missing $1" +need_file() { + local directory=$1 label=$2 file=$3 + [ -f "$directory/$file" ] || error "$label is missing $file" +} + +sha256() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +verify_archive() { + local directory=$1 label=$2 archive=$3 expected actual + expected=$(awk -v archive="$archive" '$2 == archive || $2 == "*" archive { print $1; exit }' "$directory/checksums.txt") + [ -n "$expected" ] || error "$label checksums.txt has no hash for $archive" + actual=$(sha256 "$directory/$archive") + [ "$actual" = "$expected" ] || error "$label $archive checksum was $actual, expected $expected" } cli_archive() { @@ -47,48 +76,206 @@ daemon_archive() { esac } -if [ -n "${PLOYZ_QUALIFY_SSH_KEY:-}" ]; then - SSH_OPTS="$SSH_OPTS -i $PLOYZ_QUALIFY_SSH_KEY" +SSH_ARGS=() +if [ -n "$SSH_KEY" ]; then + SSH_ARGS=(-i "$SSH_KEY") fi -ssh_identity() { - # shellcheck disable=SC2086 - set -- $SSH_OPTS - while [ $# -gt 0 ]; do - case $1 in - -i | --identity) - printf '%s\n' "${2:-}" - return +ssh_host() { + # shellcheck disable=SC2029 # Every post-host argument is intentionally interpreted remotely. + ssh "${SSH_ARGS[@]}" "$@" +} + +scp_host() { + scp "${SSH_ARGS[@]}" "$@" +} + +json_field() { + python3 -c 'import json, sys +value = json.load(sys.stdin) +for part in sys.argv[1].split("."): + value = value[part] +print(value)' "$1" +} + +machine_state_signature() { + ssh_host "$first" sudo cat /var/lib/ployz/machine.json | python3 -c 'import hashlib, json, sys +value = json.load(sys.stdin) +def stable(item): + if isinstance(item, dict): + return {key: stable(child) for key, child in item.items() if key not in {"runtime", "selected_endpoints"}} + if isinstance(item, list): + return [stable(child) for child in item] + return item +encoded = json.dumps(stable(value), sort_keys=True, separators=(",", ":")).encode() +print(hashlib.sha256(encoded).hexdigest())' +} + +machine_cloud_pairing() { + ssh_host "$first" sudo cat /var/lib/ployz/machine.json | python3 -c 'import json, sys +value = json.load(sys.stdin)["cloud_pairing"] +print(json.dumps(value, sort_keys=True, separators=(",", ":")))' +} + +start_enroll_fixture() { + local port_file=$work/cloud-enroll.port i + python3 "$COMPOSE_DIR/cloud-enroll-fixture.py" "$port_file" "$work/cloud-enroll-evidence.jsonl" >"$work/cloud-enroll.log" 2>&1 & + ENROLL_FIXTURE_PID=$! + for ((i = 0; i < 50; i++)); do + if [ -s "$port_file" ]; then + ENROLL_PORT=$(cat "$port_file") + return + fi + kill -0 "$ENROLL_FIXTURE_PID" 2>/dev/null || break + sleep 0.1 + done + cat "$work/cloud-enroll.log" >&2 || true + error "Cloud enrollment fixture did not start" +} + +finish_enroll_fixture() { + if ! wait "$ENROLL_FIXTURE_PID"; then + cat "$work/cloud-enroll.log" >&2 || true + error "Cloud enrollment fixture rejected the enrollment contract" + fi + ENROLL_FIXTURE_PID= + [ "$(wc -l <"$work/cloud-enroll-evidence.jsonl")" -eq 2 ] || error "Cloud enrollment fixture did not observe exactly two requests" +} + +read_receipt() { + ssh_host "$first" 'sudo cat /var/lib/ployz/upgrade-attempt.json 2>/dev/null || true' +} + +wait_for_new_receipt() { + local previous=$1 receipt attempt_id i + for ((i = 0; i < 150; i++)); do + receipt=$(read_receipt) + attempt_id=$(printf '%s' "$receipt" | json_field attempt.attempt_id 2>/dev/null || true) + if [ -n "$attempt_id" ] && [ "$attempt_id" != "$previous" ]; then + printf '%s\n' "$receipt" + return + fi + sleep 0.2 + done + error "Machine did not record a new upgrade attempt within 30 seconds" +} + +wait_for_outcome() { + local attempt_id=$1 expected=$2 receipt observed_id outcome i + for ((i = 0; i < 900; i++)); do + receipt=$(read_receipt) + observed_id=$(printf '%s' "$receipt" | json_field attempt.attempt_id 2>/dev/null || true) + outcome=$(printf '%s' "$receipt" | json_field attempt.outcome 2>/dev/null || true) + if [ "$observed_id" = "$attempt_id" ] && [ "$outcome" = "$expected" ]; then + printf '%s\n' "$receipt" + return + fi + case "$outcome" in + failed|interrupted|succeeded) + [ "$observed_id" != "$attempt_id" ] || error "upgrade $attempt_id ended as $outcome, expected $expected" ;; esac - shift + sleep 1 done + error "upgrade $attempt_id did not become $expected within 15 minutes" } -SSH_KEY=$(ssh_identity) +require_field() { + local json=$1 field=$2 expected=$3 actual + actual=$(printf '%s' "$json" | json_field "$field") + [ "$actual" = "$expected" ] || error "$field was $actual, expected $expected" +} -ssh_host() { - # shellcheck disable=SC2086 - ssh $SSH_OPTS "$1" "${@:2}" +wait_for_application() { + local value i + for ((i = 0; i < 120; i++)); do + value=$(ssh_host "$first" "curl -fsS --max-time 2 '$APP_URL'" 2>/dev/null || true) + if [ "$value" = "$APP_VALUE" ]; then + return + fi + sleep 1 + done + error "qualification application did not serve its persistent value" +} + +assert_provisioned_volume() { + local volumes + volumes=$("$PLOYZ" volume ls --context "$CONTEXT") + printf '%s\n' "$volumes" + printf '%s\n' "$volumes" | awk -F '\t' -v volume="$APP_VOLUME" ' + $2 == volume && $3 == "PROVISIONED" && $6 == "ployz" { found = 1 } + END { exit !found } + ' || error "$APP_VOLUME is not a provisioned Ployz Volume" + ssh_host "$first" "test \"\$(sudo docker volume inspect '$APP_VOLUME' --format '{{.Driver}}')\" = ployz" || error "$APP_VOLUME is not backed by the Ployz Docker driver" + ssh_host "$first" "sudo zfs list -H -o mountpoint | grep -Fqx '$APP_VOLUME_MOUNT'" || error "$APP_VOLUME has no mounted ZFS dataset" +} + +start_traffic() { + ssh_host "$first" "sudo rm -f '$TRAFFIC_STOP'" + # shellcheck disable=SC2016 # The loop and its variables belong to the remote shell. + ssh_host "$first" 'while [ ! -e /var/lib/ployz/qualification/traffic.stop ]; do value=$(curl -fsS --max-time 2 http://127.0.0.1:18082/identity 2>&1 || true); if [ "$value" = qualify-persistent-data ]; then echo ok; else printf "failure: %s\n" "$value"; fi; sleep 0.2; done' >"$work/traffic.log" 2>&1 & + TRAFFIC_PID=$! +} + +stop_traffic() { + if [ -z "$TRAFFIC_PID" ]; then + return 0 + fi + ssh_host "$first" "sudo touch '$TRAFFIC_STOP'" >/dev/null 2>&1 || true + wait "$TRAFFIC_PID" || true + TRAFFIC_PID= +} + +cleanup() { + stop_traffic + if [ -n "$ENROLL_FIXTURE_PID" ]; then + kill "$ENROLL_FIXTURE_PID" >/dev/null 2>&1 || true + wait "$ENROLL_FIXTURE_PID" >/dev/null 2>&1 || true + ENROLL_FIXTURE_PID= + fi + [ -z "${work:-}" ] || rm -rf "$work" + [ -z "$CONFIG_DIR" ] || rm -rf "$CONFIG_DIR" +} + +stage_remote_release() { + local label=$1 directory=$2 archive=$3 upload + upload=/tmp/ployz-qualify-$$-$label + ssh_host "$first" "rm -rf '$upload' && mkdir -p '$upload'" + scp_host "$directory/$archive" "$directory/checksums.txt" "$first:$upload/" + ssh_host "$first" "sudo rm -rf '$REMOTE_RELEASE_ROOT/$label' && sudo install -d -m 0755 '$REMOTE_RELEASE_ROOT/$label' && sudo install -m 0644 '$upload/$archive' '$upload/checksums.txt' '$REMOTE_RELEASE_ROOT/$label/' && rm -rf '$upload'" +} + +select_remote_release() { + ssh_host "$first" "sudo ln -sfn '$REMOTE_RELEASE_ROOT/$1' '$REMOTE_RELEASE_ROOT/current'" +} + +assert_remote_hash() { + local path=$1 expected=$2 + ssh_host "$first" "test \"\$(sudo sha256sum '$path' | awk '{print \$1}')\" = '$expected'" } [ -n "$HOSTS" ] || error "set PLOYZ_QUALIFY_HOSTS to one or more user@host targets" -[ -n "$ARTIFACT_DIR" ] || error "set PLOYZ_ARTIFACT_DIR to the draft archive directory" +[ -n "$ARTIFACT_DIR" ] || error "set PLOYZ_ARTIFACT_DIR to the source draft archive directory" +[ -n "$UPGRADE_ARTIFACT_DIR" ] || error "set PLOYZ_UPGRADE_ARTIFACT_DIR to the target draft archive directory" [ -f "$COMPOSE_DIR/compose.yaml" ] || error "missing $COMPOSE_DIR/compose.yaml" +command -v python3 >/dev/null 2>&1 || error "python3 is required to inspect qualification evidence" read -r -a HOST_LIST <<<"$HOSTS" [ "${#HOST_LIST[@]}" -ge 1 ] || error "PLOYZ_QUALIFY_HOSTS is empty" -need_archive ployz_linux_amd64.tar.gz -need_archive ployz_linux_arm64.tar.gz -need_archive ployzd_linux_amd64.tar.gz -need_archive ployzd_linux_arm64.tar.gz -need_archive "$(cli_archive)" +for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do + need_file "$ARTIFACT_DIR" PLOYZ_ARTIFACT_DIR "$archive" +done +need_file "$ARTIFACT_DIR" PLOYZ_ARTIFACT_DIR "$(cli_archive)" +need_file "$ARTIFACT_DIR" PLOYZ_ARTIFACT_DIR checksums.txt +need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR "$(cli_archive)" +need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR checksums.txt if [ "$DRY_RUN" != 0 ]; then echo "qualify dry-run" echo "hosts: ${HOST_LIST[*]}" - echo "artifacts: $ARTIFACT_DIR" + echo "source artifacts: $ARTIFACT_DIR" + echo "target artifacts: $UPGRADE_ARTIFACT_DIR" echo "compose: $COMPOSE_DIR/compose.yaml" echo "ssh-key: ${SSH_KEY:-cli-default}" if [ "$RESET" != 0 ]; then @@ -96,41 +283,31 @@ if [ "$DRY_RUN" != 0 ]; then else echo "reset: no (initialized hosts fail without PLOYZ_QUALIFY_RESET=1)" fi - echo "steps: install daemon from PLOYZ_RELEASE_DIR (always replace), machine init --no-install, machine add, deploy named volume qualify-data, volume ls" + echo "steps: normal ZFS machine init/add from the source release; synthetic loopback Cloud Pairing; persistent application traffic; target upgrade with client disconnect and reconnect; corrupt preflight; failed activation; explicit previous-binary repair" exit 0 fi work=$(mktemp -d) -trap 'rm -rf "$work" "$CONFIG_DIR"' EXIT -tar -xzf "$ARTIFACT_DIR/$(cli_archive)" -C "$work" -PLOYZ=$work/ployz -[ -x "$PLOYZ" ] || error "CLI archive did not contain ployz" -version=$("$PLOYZ" version) || error "could not read ployz version from the artifact" +trap cleanup EXIT +mkdir -p "$work/source-cli" "$work/target-cli" +verify_archive "$ARTIFACT_DIR" source "$(cli_archive)" +verify_archive "$UPGRADE_ARTIFACT_DIR" target "$(cli_archive)" +tar -xzf "$ARTIFACT_DIR/$(cli_archive)" -C "$work/source-cli" +tar -xzf "$UPGRADE_ARTIFACT_DIR/$(cli_archive)" -C "$work/target-cli" +PLOYZ=$work/source-cli/ployz +TARGET_PLOYZ=$work/target-cli/ployz +[ -x "$PLOYZ" ] || error "source CLI archive did not contain ployz" +[ -x "$TARGET_PLOYZ" ] || error "target CLI archive did not contain ployz" +source_version=$("$PLOYZ" version) || error "could not read source CLI version" +target_version=$("$TARGET_PLOYZ" version) || error "could not read target CLI version" +[ "$source_version" != "$target_version" ] || error "source and target versions must differ" CONFIG_DIR=$(mktemp -d) export PLOYZ_CONFIG=$CONFIG_DIR/config.yaml - -install_host() { - local host=$1 arch archive remote - arch=$(ssh_host "$host" uname -m) - archive=$(daemon_archive "$arch") - remote=/tmp/ployz-qualify-$$ - ssh_host "$host" mkdir -p "$remote" - # shellcheck disable=SC2086 - scp $SSH_OPTS "$ROOT/scripts/install.sh" "$ARTIFACT_DIR/$archive" "$host:$remote/" - ssh_host "$host" sudo env \ - PLOYZ_RELEASE_DIR="$remote" \ - PLOYZ_VERSION="$version" \ - bash "$remote/install.sh" -} - -for host in "${HOST_LIST[@]}"; do - echo "install $host" - install_host "$host" -done +export PLOYZ_RELEASE_DIR=$ARTIFACT_DIR first=${HOST_LIST[0]} -echo "machine init $first" -init_cmd=("$PLOYZ" machine init --no-install --context "$CONTEXT") +echo "machine init $first at $source_version" +init_cmd=("$PLOYZ" machine init --version "$source_version" --storage zfs --name qualify-1 --context "$CONTEXT" --no-dns --no-ingress) if [ "$RESET" != 0 ]; then init_cmd+=(--yes) fi @@ -142,7 +319,7 @@ fi i=1 while [ "$i" -lt "${#HOST_LIST[@]}" ]; do echo "machine add ${HOST_LIST[$i]}" - add_cmd=("$PLOYZ" machine add --yes --no-install --context "$CONTEXT") + add_cmd=("$PLOYZ" machine add --yes --version "$source_version" --storage zfs --name "qualify-$((i + 1))" --context "$CONTEXT") if [ -n "$SSH_KEY" ]; then add_cmd+=(--ssh-key "$SSH_KEY") fi @@ -150,11 +327,125 @@ while [ "$i" -lt "${#HOST_LIST[@]}" ]; do i=$((i + 1)) done -echo "deploy named volume" +echo "establish a synthetic non-null Cloud Pairing through Cloud enrollment" +start_enroll_fixture +"$PLOYZ" cloud enroll pmet_qualification --name qualify-1 --storage none --no-dns --no-ingress --cloud-url "http://127.0.0.1:$ENROLL_PORT" --context "$CONTEXT" +finish_enroll_fixture +pairing_before=$(machine_cloud_pairing) +[ "$pairing_before" = "$PAIRING" ] || error "Machine did not persist the exact synthetic Cloud Pairing" + +machine_arch=$(ssh_host "$first" uname -m) +machine_archive=$(daemon_archive "$machine_arch") +need_file "$UPGRADE_ARTIFACT_DIR" PLOYZ_UPGRADE_ARTIFACT_DIR "$machine_archive" +verify_archive "$UPGRADE_ARTIFACT_DIR" target "$machine_archive" + +mkdir -p "$work/releases/target" "$work/releases/corrupt" "$work/releases/failure" "$work/target-payload" +cp "$UPGRADE_ARTIFACT_DIR/$machine_archive" "$UPGRADE_ARTIFACT_DIR/checksums.txt" "$work/releases/target/" +cp "$UPGRADE_ARTIFACT_DIR/$machine_archive" "$UPGRADE_ARTIFACT_DIR/checksums.txt" "$work/releases/corrupt/" +printf 'corrupt\n' >>"$work/releases/corrupt/$machine_archive" +tar -xzf "$UPGRADE_ARTIFACT_DIR/$machine_archive" -C "$work/target-payload" +[ -x "$work/target-payload/ployzd" ] || error "target daemon archive did not contain ployzd" +[ -f "$work/target-payload/ployz-uninstall" ] || error "target daemon archive did not contain ployz-uninstall" +target_daemon_hash=$(sha256 "$work/target-payload/ployzd") +cat >"$work/target-payload/ployzd" <&2 +exit 42 +EOF +chmod 0755 "$work/target-payload/ployzd" +tar -czf "$work/releases/failure/$machine_archive" -C "$work/target-payload" ployzd ployz-uninstall +failure_daemon_hash=$(sha256 "$work/target-payload/ployzd") +printf '%s %s\n' "$(sha256 "$work/releases/failure/$machine_archive")" "$machine_archive" >"$work/releases/failure/checksums.txt" + +for release in target corrupt failure; do + stage_remote_release "$release" "$work/releases/$release" "$machine_archive" +done +select_remote_release target +ssh_host "$first" "sudo install -d -m 0755 /etc/systemd/system/ployz.service.d && printf '[Service]\nEnvironment=PLOYZ_UPGRADE_RELEASE_DIR=$REMOTE_RELEASE_ROOT/current\n' | sudo tee /etc/systemd/system/ployz.service.d/qualification.conf >/dev/null && sudo systemctl daemon-reload && sudo systemctl restart ployz.service" + +echo "deploy persistent ZFS-backed application" "$PLOYZ" deploy --yes --context "$CONTEXT" -f "$COMPOSE_DIR/compose.yaml" +assert_provisioned_volume +wait_for_application +state_before=$(machine_state_signature) +start_traffic + +echo "upgrade qualify-1 from $source_version to $target_version, then disconnect the requesting client" +"$PLOYZ" machine upgrade "$target_version" --machine qualify-1 --context "$CONTEXT" >"$work/success-client.log" 2>&1 & +client_pid=$! +success_receipt=$(wait_for_new_receipt "") +success_id=$(printf '%s' "$success_receipt" | json_field attempt.attempt_id) +kill "$client_pid" >/dev/null 2>&1 || true +wait "$client_pid" >/dev/null 2>&1 || true +success_receipt=$(wait_for_outcome "$success_id" succeeded) +require_field "$success_receipt" attempt.version "$target_version" +success_inspection=$("$PLOYZ" machine upgrade inspect qualify-1 --attempt "$success_id" --output json --context "$CONTEXT") +require_field "$success_inspection" outcome succeeded +require_field "$success_inspection" attempt_id "$success_id" +require_field "$success_inspection" version "$target_version" +assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" +wait_for_application +[ "$(machine_state_signature)" = "$state_before" ] || error "Machine identity, pairing, or configuration changed during upgrade" +[ "$(machine_cloud_pairing)" = "$pairing_before" ] || error "Cloud Pairing changed during upgrade" + +echo "reject a checksum-corrupt target before activation" +select_remote_release corrupt +if "$PLOYZ" machine upgrade "$target_version" --machine qualify-1 --context "$CONTEXT" >"$work/corrupt-client.log" 2>&1; then + error "checksum-corrupt upgrade unexpectedly succeeded" +fi +grep -qi checksum "$work/corrupt-client.log" || error "checksum-corrupt upgrade did not report checksum evidence" +corrupt_receipt=$(wait_for_new_receipt "$success_id") +corrupt_id=$(printf '%s' "$corrupt_receipt" | json_field attempt.attempt_id) +corrupt_receipt=$(wait_for_outcome "$corrupt_id" failed) +require_field "$corrupt_receipt" attempt.stage verifying +assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" +ssh_host "$first" 'test ! -e /var/lib/ployz/.upgrade-active' +wait_for_application + +echo "activate an intentionally failing daemon and preserve local diagnostic evidence" +select_remote_release failure +"$PLOYZ" machine upgrade "$target_version" --machine qualify-1 --context "$CONTEXT" >"$work/failure-client.log" 2>&1 & +client_pid=$! +failure_receipt=$(wait_for_new_receipt "$corrupt_id") +failure_id=$(printf '%s' "$failure_receipt" | json_field attempt.attempt_id) +kill "$client_pid" >/dev/null 2>&1 || true +wait "$client_pid" >/dev/null 2>&1 || true +failure_receipt=$(wait_for_outcome "$failure_id" failed) +failure_stage=$(printf '%s' "$failure_receipt" | json_field attempt.stage) +case "$failure_stage" in + restarting|readiness) ;; + *) error "failed activation was reported at $failure_stage" ;; +esac +ssh_host "$first" "sudo journalctl -u 'ployz-upgrade-$failure_id.service' --no-pager -n 200 || true" +sleep 3 +assert_remote_hash /usr/local/bin/ployzd "$failure_daemon_hash" +assert_remote_hash /usr/local/bin/ployzd.previous "$target_daemon_hash" +ssh_host "$first" 'test ! -e /var/lib/ployz/.upgrade-active && ! sudo systemctl is-active --quiet ployz.service' +wait_for_application -volumes=$("$PLOYZ" volume ls --context "$CONTEXT") -printf '%s\n' "$volumes" -printf '%s\n' "$volumes" | grep -Eq 'qualify-data' || error "named volume qualify-data did not appear in volume ls" +echo "repair explicitly from the one retained previous binary" +ssh_host "$first" 'sudo systemctl stop ployz.service || true; sudo cp --preserve=mode,ownership /usr/local/bin/ployzd.previous /usr/local/bin/.ployzd-repair; sudo sync /usr/local/bin/.ployzd-repair; sudo mv /usr/local/bin/.ployzd-repair /usr/local/bin/ployzd; sudo sync /usr/local/bin; sudo systemctl start ployz.service' +failure_inspection=$("$PLOYZ" machine upgrade inspect qualify-1 --attempt "$failure_id" --output json --context "$CONTEXT") +require_field "$failure_inspection" outcome failed +require_field "$failure_inspection" attempt_id "$failure_id" +require_field "$failure_inspection" stage "$failure_stage" +assert_remote_hash /usr/local/bin/ployzd "$target_daemon_hash" +wait_for_application +[ "$(machine_state_signature)" = "$state_before" ] || error "Machine identity, pairing, or configuration changed after explicit repair" +[ "$(machine_cloud_pairing)" = "$pairing_before" ] || error "Cloud Pairing changed after explicit repair" + +stop_traffic +if grep -q '^failure:' "$work/traffic.log"; then + cat "$work/traffic.log" >&2 + error "application traffic failed during daemon upgrade or repair" +fi +traffic_successes=$(grep -c '^ok$' "$work/traffic.log" || true) +[ "$traffic_successes" -ge 5 ] || error "continuous traffic collected only $traffic_successes successful probes" +printf 'continuous application traffic: %s successful probes, 0 failures\n' "$traffic_successes" -echo "qualify-release passed on ${HOST_LIST[*]}" +assert_provisioned_volume +echo "qualify-release passed on ${HOST_LIST[*]}: $source_version -> $target_version" diff --git a/scripts/qualify-release/cloud-enroll-fixture.py b/scripts/qualify-release/cloud-enroll-fixture.py new file mode 100644 index 0000000000..5be3e18e67 --- /dev/null +++ b/scripts/qualify-release/cloud-enroll-fixture.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Bounded loopback Cloud enrollment fixture for release qualification.""" + +import http.server +import json +import re +import sys + +port_file, evidence_file = sys.argv[1:] +token = "pmet_qualification" +secret = "qualification-synthetic-pairing-secret-v1" +expected_paths = [f"/api/enroll/{token}", f"/api/enroll/{token}/callback"] + + +class Handler(http.server.BaseHTTPRequestHandler): + requests = 0 + failure = None + + def do_POST(self): + try: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > 1024 * 1024: + raise AssertionError(f"invalid request size {length}") + body = json.loads(self.rfile.read(length)) + expected = expected_paths[Handler.requests] + if self.path != expected: + raise AssertionError( + f"request {Handler.requests + 1} path {self.path!r}, expected {expected!r}" + ) + if Handler.requests == 0: + if body.get("protocolVersion") != 2: + raise AssertionError("enroll protocolVersion was not 2") + if body.get("name") != "qualify-1": + raise AssertionError("enroll Machine Name was not qualify-1") + if body.get("requestedStorage") != "none": + raise AssertionError("enroll requestedStorage was not none") + if not isinstance(body.get("publicKey"), str) or not body["publicKey"]: + raise AssertionError("enroll publicKey was empty") + response = { + "kind": "initialize", + "resumed": True, + "storage": "none", + "pairing": { + "relayUrl": "http://127.0.0.1:1", + "secret": secret, + }, + } + evidence = {"request": "enroll", "identity": "accepted"} + else: + machine_id = body.get("machineId") + valid_machine_id = isinstance( + machine_id, str + ) and re.fullmatch(r"[0-9a-f]{32}", machine_id) + if not valid_machine_id: + raise AssertionError( + "callback Machine ID was not 32 lowercase hex characters" + ) + if body.get("pairingCredential") != secret: + raise AssertionError("callback Pairing Credential changed") + response = {} + evidence = { + "request": "callback", + "machineId": machine_id, + "pairingCredential": "matched", + } + with open(evidence_file, "a", encoding="utf-8") as evidence_stream: + evidence_stream.write(json.dumps(evidence, sort_keys=True) + "\n") + payload = json.dumps(response, separators=(",", ":")).encode() + self.send_response(200) + except Exception as error: + Handler.failure = str(error) + payload = json.dumps({"error": str(error)}).encode() + self.send_response(400) + Handler.requests += 1 + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.send_header("Connection", "close") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *_): + pass + + +server = http.server.HTTPServer(("127.0.0.1", 0), Handler) +server.timeout = 30 +with open(port_file, "w", encoding="ascii") as output: + output.write(str(server.server_port)) +for _ in expected_paths: + server.handle_request() +server.server_close() +if Handler.requests != len(expected_paths): + raise SystemExit( + f"Cloud enrollment fixture received {Handler.requests} of {len(expected_paths)} requests" + ) +if Handler.failure is not None: + raise SystemExit(Handler.failure) diff --git a/scripts/qualify-release/compose.yaml b/scripts/qualify-release/compose.yaml index 2ad6c43afc..1020f247ba 100644 --- a/scripts/qualify-release/compose.yaml +++ b/scripts/qualify-release/compose.yaml @@ -1,8 +1,13 @@ services: data: - image: alpine:3.23.3 - command: ["sleep", "infinity"] + image: busybox:1.37.0 + command: + - sh + - -c + - test -s /data/identity || printf 'qualify-persistent-data\n' > /data/identity; exec httpd -f -p 8080 -h /data volumes: - qualify-data:/data -volumes: - qualify-data: + x-machines: [qualify-1] + x-ports: [18082:8080/tcp@host] +x-volumes: + qualify-data: 256M diff --git a/scripts/test-daemon-lifecycle.sh b/scripts/test-daemon-lifecycle.sh index 4416b4ee71..ca84459f77 100755 --- a/scripts/test-daemon-lifecycle.sh +++ b/scripts/test-daemon-lifecycle.sh @@ -5,62 +5,27 @@ set -euo pipefail ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT -mkdir -p "$TMP/bin" "$TMP/release" "$TMP/install" "$TMP/systemd" "$TMP/state" "$TMP/run" +mkdir -p "$TMP/bin" "$TMP/install" "$TMP/systemd" "$TMP/state" "$TMP/run" LOG=$TMP/commands.log export LOG : > "$LOG" -cat > "$TMP/ployzd" <<'EOF' -#!/bin/sh -[ "$1" = version ] && echo 1.2.3 -EOF -chmod 0755 "$TMP/ployzd" -tar -czf "$TMP/release/ployzd_linux_amd64.tar.gz" \ - -C "$TMP" ployzd -C "$ROOT/scripts" --transform='s|uninstall.sh|ployz-uninstall|' uninstall.sh -printf 'v1.2.3\n' > "$TMP/release/stable" - -cat > "$TMP/bin/uname" <<'EOF' -#!/bin/sh -case "$1" in -s) echo Linux ;; -m) echo x86_64 ;; esac -EOF -cat > "$TMP/bin/curl" <<'EOF' -#!/bin/bash -while [ "$#" -gt 0 ]; do - if [ "$1" = -o ]; then output=$2; shift 2; else url=$1; shift; fi -done -src="$FAKE_RELEASE/${url##*/}" -[ -f "$src" ] || exit 1 -cp "$src" "$output" -EOF -cat > "$TMP/bin/install" <<'EOF' -#!/bin/bash -args=() -while [ "$#" -gt 0 ]; do - case "$1" in -o|-g) shift 2 ;; *) args+=("$1"); shift ;; esac -done -exec /usr/bin/install "${args[@]}" -EOF -cat > "$TMP/bin/id" <<'EOF' -#!/bin/sh -exit 0 -EOF -cat > "$TMP/bin/dockerd" <<'EOF' -#!/bin/sh -exit 0 -EOF cat > "$TMP/bin/docker" <<'EOF' #!/bin/sh echo "docker $*" >> "$LOG" case "$*" in - 'info -f {{ .DriverStatus }}') echo io.containerd.snapshotter ;; 'ps -aq --filter label=ployz.managed') echo managed-container ;; 'ps -aq --filter name=^/ployz-corrosion$') echo corrosion-container ;; 'network ls -q --filter name=^ployz$') echo ployz-network ;; 'rm -f '*) - if grep -Eq '^systemctl stop .*ployz-volume-plugin\.(socket|service)' "$LOG" || [ ! -x "$INSTALL_BIN_DIR/ployzd" ]; then - echo "Docker could not unmount after volume plugin shutdown" >&2 + grep -Eq '^systemctl stop .*ployz-volume-plugin\.(socket|service)' "$LOG" && { + echo "volume plugin stopped before managed containers were removed" >&2 exit 1 - fi + } + [ -x "$INSTALL_BIN_DIR/ployzd" ] || { + echo "daemon removed before managed containers were removed" >&2 + exit 1 + } ;; esac EOF @@ -78,94 +43,14 @@ for command in getent userdel groupdel; do done chmod 0755 "$TMP/bin"/* -run_installer() { - sudo env PATH="$TMP/bin:$PATH" LOG="$LOG" FAKE_RELEASE="$TMP/release" PLOYZ_GITHUB_URL=https://example.invalid \ - PLOYZ_VERSION="$1" INSTALL_BIN_DIR="$TMP/install" INSTALL_SYSTEMD_DIR="$TMP/systemd" \ - PLOYZ_DATA_DIR="$TMP/state" PLOYZ_RUN_DIR="$TMP/run" bash "$ROOT/scripts/install.sh" -} +touch "$TMP/install/ployzd" "$TMP/install/ployz-uninstall" "$TMP/install/ployz-corrosion" +chmod 0755 "$TMP/install/ployzd" "$TMP/install/ployz-uninstall" +touch "$TMP/docker" "$TMP/images" "$TMP/volumes" "$TMP/docker-config" -set_installed_version() { - sudo tee "$TMP/install/ployzd" >/dev/null <&2 - exit 1 - fi -done -grep -Fq 'systemctl enable --now ployz-volume-plugin.socket' "$LOG" -if grep -Fq 'systemctl enable --now ployz-volume-plugin.service' "$LOG"; then - echo "Volume plugin service was enabled instead of socket-activated" >&2 - exit 1 -fi -grep -Fq 'systemctl restart ployz.service' "$LOG" -grep -Fq 'systemctl try-restart ployz-volume-plugin.service' "$LOG" -socket_enable_line=$(grep -nF 'systemctl enable --now ployz-volume-plugin.socket' "$LOG" | head -n 1 | cut -d: -f1) -docker_check_line=$(grep -nF 'docker info -f {{ .DriverStatus }}' "$LOG" | head -n 1 | cut -d: -f1) -if [ "$socket_enable_line" -ge "$docker_check_line" ]; then - echo "Volume plugin socket was activated after Docker installation" >&2 - exit 1 -fi - -: > "$LOG" -run_installer latest -if grep -Fq 'systemctl restart ployz.service' "$LOG"; then - echo "equal latest restarted ployz.service" >&2 - exit 1 -fi - -sudo rm -f "$TMP/systemd/ployz.service" -: > "$LOG" -run_installer latest -grep -Fq 'systemctl restart ployz.service' "$LOG" -[ -f "$TMP/systemd/ployz.service" ] - -: > "$LOG" -set_installed_version 1.2.2 -run_installer latest -grep -Fq 'systemctl restart ployz.service' "$LOG" -grep -Fxq 'RestartPreventExitStatus=78' "$TMP/systemd/ployz.service" -[ -f "$TMP/state/preserved" ] - -: > "$LOG" -set_installed_version 1.2.4 -run_installer latest -if grep -Fq 'systemctl restart ployz.service' "$LOG"; then - echo "newer-than-latest daemon was replaced" >&2 - exit 1 -fi -[ "$("$TMP/install/ployzd" version)" = 1.2.4 ] - -: > "$LOG" -run_installer 1.2.2 -grep -Fq 'systemctl restart ployz.service' "$LOG" -[ "$("$TMP/install/ployzd" version)" = 1.2.3 ] -[ -f "$TMP/state/preserved" ] - -touch "$TMP/docker" "$TMP/images" "$TMP/volumes" "$TMP/docker-config" "$TMP/install/ployz-corrosion" -: > "$LOG" sudo env PATH="$TMP/bin:$PATH" LOG="$LOG" PLOYZ_AUTO_CONFIRM=true INSTALL_BIN_DIR="$TMP/install" \ INSTALL_SYSTEMD_DIR="$TMP/systemd" PLOYZ_DATA_DIR="$TMP/state" PLOYZ_RUN_DIR="$TMP/run" \ bash "$ROOT/scripts/uninstall.sh" + [ ! -e "$TMP/install/ployzd" ] [ ! -e "$TMP/install/ployz-uninstall" ] [ -e "$TMP/install/ployz-corrosion" ] @@ -177,4 +62,4 @@ grep -Fq 'docker rm -f corrosion-container' "$LOG" grep -Fq 'docker network rm ployz-network' "$LOG" grep -Fq 'ip link delete ployz' "$LOG" -echo "daemon replacement and destructive uninstall contracts passed" +echo "destructive daemon uninstall contract passed" diff --git a/scripts/test-qualify-release.sh b/scripts/test-qualify-release.sh index c82a9e0bbe..8d88c1fda7 100755 --- a/scripts/test-qualify-release.sh +++ b/scripts/test-qualify-release.sh @@ -16,6 +16,7 @@ if grep -qi vultr "$ROOT/scripts/qualify-release.sh" "$ROOT/docs/RELEASE.md"; th fi grep -Fq 'qualify-data' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture has no named volume" +grep -Fq 'x-volumes:' "$ROOT/scripts/qualify-release/compose.yaml" || fail "compose fixture volume is not provisioned" if "$ROOT/scripts/qualify-release.sh" >/dev/null 2>&1; then fail "qualify-release accepted empty hosts" @@ -23,10 +24,17 @@ fi TMP=$(mktemp -d) trap 'rm -rf "$TMP"' EXIT -mkdir -p "$TMP/bin" "$TMP/installed" +SOURCE=$TMP/source +TARGET=$TMP/target +mkdir -p "$TMP/bin" "$SOURCE" "$TARGET" export LOG=$TMP/calls.log -cat > "$TMP/ployz" <<'CLI' +export SSH_LOG=$TMP/ssh.log +export QUALIFY_STATE=$TMP/current-release +printf 'target\n' >"$QUALIFY_STATE" + +cat >"$SOURCE/ployz" <<'CLI' #!/bin/sh +set -eu case "$1" in version) printf '1.2.3\n' ;; deploy|volume) @@ -46,116 +54,276 @@ case "$1" in esac shift done - printf '%s context=%s file=%s yes=%s\n' "$action" "$context" "$file" "$yes" >> "$LOG" - [ "$action" != volume ] || printf 'qualify-data\n' + printf '%s context=%s file=%s yes=%s\n' "$action" "$context" "$file" "$yes" >>"$LOG" + [ "$action" != volume ] || printf 'MACHINE\tVOLUME\tTYPE\tQUOTA\tUSED\tDRIVER\nqualify-1\tqualify-release_qualify-data\tPROVISIONED\t268435456\t4096\tployz\n' ;; - machine) - action=$2 - case "$action" in init|add) ;; *) exit 1 ;; esac - shift 2 - reset=no key= target= context= no_install=no + cloud) + [ "$2" = enroll ] || { echo "unexpected cloud action: $2" >&2; exit 1; } + token=$3 + shift 3 + name= storage= context= cloud_url= no_dns=no no_ingress=no while [ "$#" -gt 0 ]; do case "$1" in - --yes) reset=yes ;; - --ssh-key) key=$2; shift ;; + --name) name=$2; shift ;; + --storage) storage=$2; shift ;; --context) context=$2; shift ;; - --no-install) no_install=yes ;; - root@*) target=$1 ;; - *) echo "unexpected machine argument: $1" >&2; exit 1 ;; + --cloud-url) cloud_url=$2; shift ;; + --no-dns) no_dns=yes ;; + --no-ingress) no_ingress=yes ;; + *) echo "unexpected cloud enroll argument: $1" >&2; exit 1 ;; esac shift done - printf '%s target=%s reset=%s key=%s context=%s no_install=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" >> "$LOG" + printf 'cloud-enroll token=%s name=%s storage=%s context=%s cloud_url=%s no_dns=%s no_ingress=%s\n' "$token" "$name" "$storage" "$context" "$cloud_url" "$no_dns" "$no_ingress" >>"$LOG" + python3 - "$cloud_url" "$token" <<'PY' +import json +import sys +import urllib.request + +origin, token = sys.argv[1:] +url = f"{origin}/api/enroll/{token}" +identity = { + "protocolVersion": 2, + "name": "qualify-1", + "requestedStorage": "none", + "publicKey": "synthetic-public-key", +} +request = urllib.request.Request(url, json.dumps(identity).encode(), {"Content-Type": "application/json"}) +with urllib.request.urlopen(request) as response: + pairing = json.load(response)["pairing"] +callback = { + "machineId": "11111111111111111111111111111111", + "pairingCredential": pairing["secret"], +} +request = urllib.request.Request(f"{url}/callback", json.dumps(callback).encode(), {"Content-Type": "application/json"}) +with urllib.request.urlopen(request) as response: + json.load(response) +PY + : >"$QUALIFY_PAIRED" + ;; + machine) + action=$2 + shift 2 + case "$action" in + init|add) + reset=no key= target= context= no_install=no version= storage= name= no_dns=no no_ingress=no + while [ "$#" -gt 0 ]; do + case "$1" in + --yes) reset=yes ;; + --ssh-key) key=$2; shift ;; + --context) context=$2; shift ;; + --version) version=$2; shift ;; + --storage) storage=$2; shift ;; + --name) name=$2; shift ;; + --no-install) no_install=yes ;; + --no-dns) no_dns=yes ;; + --no-ingress) no_ingress=yes ;; + root@*) target=$1 ;; + *) echo "unexpected machine argument: $1" >&2; exit 1 ;; + esac + shift + done + printf '%s target=%s reset=%s key=%s context=%s no_install=%s version=%s storage=%s name=%s no_dns=%s no_ingress=%s release=%s\n' "$action" "$target" "$reset" "$key" "$context" "$no_install" "$version" "$storage" "$name" "$no_dns" "$no_ingress" "${PLOYZ_RELEASE_DIR:-}" >>"$LOG" + ;; + upgrade) + if [ "${1:-}" = inspect ]; then + shift + machine=$1 + shift + attempt= context= output= + while [ "$#" -gt 0 ]; do + case "$1" in + --attempt) attempt=$2; shift ;; + --context) context=$2; shift ;; + --output) output=$2; shift ;; + *) echo "unexpected upgrade inspect argument: $1" >&2; exit 1 ;; + esac + shift + done + current=$(cat "$QUALIFY_STATE") + printf 'upgrade-inspect machine=%s attempt=%s context=%s output=%s current=%s\n' "$machine" "$attempt" "$context" "$output" "$current" >>"$LOG" + case "$current" in + target) printf '{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","target":"1.2.4","version":"1.2.4"}\n' ;; + failure) printf '{"outcome":"failed","attempt_id":"cccccccccccccccccccccccccccccccc","target":"1.2.4","stage":"restarting","error":"restart daemon failed"}\n' ;; + *) exit 1 ;; + esac + exit + fi + release=$1 + shift + machine= context= + while [ "$#" -gt 0 ]; do + case "$1" in + --machine) machine=$2; shift ;; + --context) context=$2; shift ;; + *) echo "unexpected upgrade argument: $1" >&2; exit 1 ;; + esac + shift + done + current=$(cat "$QUALIFY_STATE") + printf 'upgrade release=%s machine=%s context=%s current=%s\n' "$release" "$machine" "$context" "$current" >>"$LOG" + case "$current" in + corrupt) echo 'daemon archive checksum mismatch' >&2; exit 1 ;; + target|failure) sleep 30 ;; + *) exit 1 ;; + esac + ;; + *) echo "unexpected machine action: $action" >&2; exit 1 ;; + esac ;; *) echo "unexpected command: $*" >&2; exit 1 ;; esac CLI -chmod 0755 "$TMP/ployz" -for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployz_macos_amd64.tar.gz ployz_macos_arm64.tar.gz ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do - tar -czf "$TMP/$archive" -C "$TMP" ployz -done +chmod 0755 "$SOURCE/ployz" -# Same-version local archives must still replace the installed payload. -for payload in old new; do - printf '#!/bin/sh\nif [ "$1" = version ]; then echo 1.2.3; else echo %s; fi\n' "$payload" > "$TMP/$payload" - chmod 0755 "$TMP/$payload" +cat >"$TARGET/ployz" <<'CLI' +#!/bin/sh +[ "${1:-}" = version ] && printf '1.2.4\n' +CLI +chmod 0755 "$TARGET/ployz" + +cat >"$SOURCE/ployzd" <<'DAEMON' +#!/bin/sh +[ "${1:-}" = version ] && printf '1.2.3\n' +DAEMON +cat >"$TARGET/ployzd" <<'DAEMON' +#!/bin/sh +[ "${1:-}" = version ] && printf '1.2.4\n' +DAEMON +printf '#!/bin/sh\nexit 0\n' >"$SOURCE/ployz-uninstall" +cp "$SOURCE/ployz-uninstall" "$TARGET/ployz-uninstall" +chmod 0755 "$SOURCE/ployzd" "$TARGET/ployzd" "$SOURCE/ployz-uninstall" "$TARGET/ployz-uninstall" + +for archive in ployz_linux_amd64.tar.gz ployz_linux_arm64.tar.gz ployz_macos_amd64.tar.gz ployz_macos_arm64.tar.gz; do + tar -czf "$SOURCE/$archive" -C "$SOURCE" ployz + tar -czf "$TARGET/$archive" -C "$TARGET" ployz done -cp "$TMP/old" "$TMP/installed/ployzd" -cp "$TMP/new" "$TMP/ployzd" -cp "$TMP/new" "$TMP/ployz-uninstall" for archive in ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz; do - tar -czf "$TMP/$archive" -C "$TMP" ployzd ployz-uninstall + tar -czf "$SOURCE/$archive" -C "$SOURCE" ployzd ployz-uninstall + tar -czf "$TARGET/$archive" -C "$TARGET" ployzd ployz-uninstall done ( - export PLOYZ_INSTALL_TEST_ONLY=true INSTALL_BIN_DIR="$TMP/installed" PLOYZ_RELEASE_DIR="$TMP" PLOYZ_VERSION=1.2.3 - source "$ROOT/scripts/install.sh" - install_binaries + cd "$SOURCE" + sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 >checksums.txt +) +( + cd "$TARGET" + sha256sum ployz_*.tar.gz ployzd_*.tar.gz | sort -k2 >checksums.txt ) -[ "$("$TMP/installed/ployzd")" = new ] || fail "same-version local archive was not installed" -export SSH_LOG=$TMP/ssh.log -cat > "$TMP/bin/ssh" <<'SSH' +cat >"$TMP/bin/ssh" <<'SSH' #!/bin/sh set -eu key= while [ "$#" -gt 0 ]; do case "$1" in -i) key=$2; shift 2 ;; - -o) shift 2 ;; *) break ;; esac done [ "$key" = /tmp/qualify-key ] || { echo 'remote command lost SSH identity' >&2; exit 1; } -printf '%s identity=%s\n' "${0##*/}" "$key" >> "$SSH_LOG" -case "$*" in *'uname -m') echo x86_64 ;; esac +host=$1 +shift +command=$* +printf 'ssh host=%s command=%s\n' "$host" "$command" >>"$SSH_LOG" +case "$command" in + *'uname -m'*) printf 'x86_64\n' ;; + *'ln -sfn'*'/target'*'/current'*) printf 'target\n' >"$QUALIFY_STATE" ;; + *'ln -sfn'*'/corrupt'*'/current'*) printf 'corrupt\n' >"$QUALIFY_STATE" ;; + *'ln -sfn'*'/failure'*'/current'*) printf 'failure\n' >"$QUALIFY_STATE" ;; + *'upgrade-attempt.json'*) + case "$(cat "$QUALIFY_STATE")" in + target) printf '{"requested":"1.2.4","attempt":{"outcome":"succeeded","attempt_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","target":"1.2.4","version":"1.2.4"}}\n' ;; + corrupt) printf '{"requested":"1.2.4","attempt":{"outcome":"failed","attempt_id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","target":"1.2.4","stage":"verifying","error":"checksum mismatch"}}\n' ;; + failure) printf '{"requested":"1.2.4","attempt":{"outcome":"failed","attempt_id":"cccccccccccccccccccccccccccccccc","target":"1.2.4","stage":"restarting","error":"restart daemon failed"}}\n' ;; + esac + ;; + *'sudo cat /var/lib/ployz/machine.json'*) + if [ -f "$QUALIFY_PAIRED" ]; then + pairing='{"relayUrl":"http://127.0.0.1:1/","secret":"qualification-synthetic-pairing-secret-v1"}' + else + pairing=null + fi + printf '{"body":{"phase":"participating","machine":{"id":"11111111111111111111111111111111","runtime":{"daemon_version":"1.2.3"}}},"wireguard_private_key":"secret","cloud_pairing":%s,"selected_endpoints":{}}\n' "$pairing" + ;; + *'while [ ! -e /var/lib/ployz/qualification/traffic.stop'*) + printf 'ok\nok\nok\nok\nok\nok\n' + ;; + *'curl -fsS --max-time 2'*) printf 'qualify-persistent-data\n' ;; + *'journalctl -u'*) printf 'intentional qualification activation failure\n' ;; +esac SSH chmod 0755 "$TMP/bin/ssh" -ln -s ssh "$TMP/bin/scp" + +cat >"$TMP/bin/scp" <<'SCP' +#!/bin/sh +set -eu +key= +if [ "${1:-}" = -i ]; then + key=$2 + shift 2 +fi +[ "$key" = /tmp/qualify-key ] || { echo 'remote copy lost SSH identity' >&2; exit 1; } +printf 'scp %s\n' "$*" >>"$SSH_LOG" +SCP +chmod 0755 "$TMP/bin/scp" +export QUALIFY_PAIRED=$TMP/paired + for reset in 0 1; do - : > "$LOG" - PATH="$TMP/bin:$PATH" PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$TMP" \ - PLOYZ_QUALIFY_SSH_OPTS='-i /tmp/qualify-key' PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ + : >"$LOG" + rm -f "$QUALIFY_PAIRED" + PATH="$TMP/bin:$PATH" PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$SOURCE" \ + PLOYZ_UPGRADE_ARTIFACT_DIR="$TARGET" PLOYZ_QUALIFY_SSH_KEY=/tmp/qualify-key \ + PLOYZ_QUALIFY_CONTEXT=qualify PLOYZ_QUALIFY_RESET="$reset" \ "$ROOT/scripts/qualify-release.sh" >/dev/null expected_reset=no [ "$reset" = 0 ] || expected_reset=yes - grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=yes" "$LOG" || fail "init lost its target, reset policy, or SSH identity" - grep -Fxq 'add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=yes' "$LOG" || fail "add lost its target or SSH identity" - grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "named-volume fixture was not deployed" - grep -Fxq 'volume context=qualify file= yes=no' "$LOG" || fail "volume ls did not query the qualification context" -done -for command in ssh scp; do - grep -Fxq "$command identity=/tmp/qualify-key" "$SSH_LOG" || fail "$command was not exercised" + grep -Fxq "init target=root@192.0.2.10 reset=$expected_reset key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-1 no_dns=yes no_ingress=yes release=$SOURCE" "$LOG" || fail "init lost its target, release, storage, name, version, reset policy, isolation policy, or SSH identity" + grep -Fxq "add target=root@192.0.2.11 reset=yes key=/tmp/qualify-key context=qualify no_install=no version=1.2.3 storage=zfs name=qualify-2 no_dns=no no_ingress=no release=$SOURCE" "$LOG" || fail "add lost its target, release, storage, name, version, or SSH identity" + grep -Eq '^cloud-enroll token=pmet_qualification name=qualify-1 storage=none context=qualify cloud_url=http://127\.0\.0\.1:[0-9]+ no_dns=yes no_ingress=yes$' "$LOG" || fail "Cloud Pairing fixture was not exercised through the source CLI" + grep -Fxq "deploy context=qualify file=$ROOT/scripts/qualify-release/compose.yaml yes=yes" "$LOG" || fail "persistent-volume fixture was not deployed" + grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=target' "$LOG" || fail "target upgrade was not requested" + grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=corrupt' "$LOG" || fail "corrupt preflight was not requested" + grep -Fxq 'upgrade release=1.2.4 machine=qualify-1 context=qualify current=failure' "$LOG" || fail "failed activation was not requested" + grep -Fxq 'upgrade-inspect machine=qualify-1 attempt=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa context=qualify output=json current=target' "$LOG" || fail "successful attempt was not inspected after reconnect" + grep -Fxq 'upgrade-inspect machine=qualify-1 attempt=cccccccccccccccccccccccccccccccc context=qualify output=json current=failure' "$LOG" || fail "failed attempt was not inspected after repair" done +grep -Fq 'ssh host=root@192.0.2.10 command=' "$SSH_LOG" || fail "SSH evidence path was not exercised" +grep -Fq 'docker volume inspect' "$SSH_LOG" || fail "qualification did not verify the Docker Volume driver" +grep -Fq 'zfs list -H -o mountpoint' "$SSH_LOG" || fail "qualification did not verify the ZFS dataset mount" +grep -Fq 'scp ' "$SSH_LOG" || fail "target release was not staged with SCP" -cat > "$TMP/bin/cargo" <<'CARGO' +cat >"$TMP/bin/cargo" <<'CARGO' #!/bin/sh case " $* " in *' --no-run '*' --ignored '*|*' --ignored '*' --no-run '*) echo 'informing tests must execute' >&2; exit 1 ;; *' --no-run '*) [ ! -s "$LOG" ] || exit 1 - printf 'compile\n' >> "$LOG" + printf 'compile\n' >>"$LOG" exit 0 ;; *' --no-fail-fast '*) case " $* " in - *' --ignored '*|*' --include-ignored '*) printf '%s\n' "$*" >> "$LOG" ;; + *' --ignored '*|*' --include-ignored '*) printf '%s\n' "$*" >>"$LOG" ;; *) echo 'informing tests must be selected' >&2; exit 1 ;; esac ;; *) echo 'test invocation must collect all binary failures' >&2; exit 1 ;; esac CARGO chmod 0755 "$TMP/bin/cargo" -: > "$LOG" +: >"$LOG" PATH="$TMP/bin:$PATH" "$ROOT/scripts/run-layer3-tests.sh" grep -Eq -- '--(include-)?ignored' "$LOG" || fail "layer3 runner did not execute tests" output=$( - PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$TMP" \ - PLOYZ_QUALIFY_DRY_RUN=1 PLOYZ_QUALIFY_RESET=0 "$ROOT/scripts/qualify-release.sh" + PLOYZ_QUALIFY_HOSTS='root@192.0.2.10 root@192.0.2.11' PLOYZ_ARTIFACT_DIR="$SOURCE" \ + PLOYZ_UPGRADE_ARTIFACT_DIR="$TARGET" PLOYZ_QUALIFY_DRY_RUN=1 \ + PLOYZ_QUALIFY_RESET=0 "$ROOT/scripts/qualify-release.sh" ) printf '%s\n' "$output" | grep -Fq 'qualify dry-run' || fail "dry-run did not print the plan" -printf '%s\n' "$output" | grep -Fq 'named volume' || fail "dry-run omitted the named-volume step" +printf '%s\n' "$output" | grep -Fq 'normal ZFS machine init/add' || fail "dry-run omitted normal storage setup" +printf '%s\n' "$output" | grep -Fq 'client disconnect and reconnect' || fail "dry-run omitted connection loss" +printf '%s\n' "$output" | grep -Fq 'failed activation' || fail "dry-run omitted failure evidence" printf '%s\n' "$output" | grep -Fq 'reset: no' || fail "dry-run omitted the default no-reset policy" -printf '%s\n' "$output" | grep -Fq 'always replace' || fail "dry-run omitted forced daemon replace" "$ROOT/scripts/check-layer3-runner.sh" echo "qualify-release contracts passed" diff --git a/scripts/verify-release.sh b/scripts/verify-release.sh index baa14cfdbe..d1804c7e93 100755 --- a/scripts/verify-release.sh +++ b/scripts/verify-release.sh @@ -49,8 +49,8 @@ check_archive() { check_checksums_and_formula() { checksum_names=$(awk '{print $2}' "$DIST/checksums.txt" | sort) - expected_checksums=$(cli_archives | sort) - [ "$checksum_names" = "$expected_checksums" ] || fail "checksums.txt must cover only CLI archives" + expected_checksums=$(printf '%s\n' "$(cli_archives)" ployzd_linux_amd64.tar.gz ployzd_linux_arm64.tar.gz | sort) + [ "$checksum_names" = "$expected_checksums" ] || fail "checksums.txt must cover CLI and daemon archives" formula=$(find "$DIST" -name 'ployz.rb' -print -quit) [ -n "$formula" ] || fail "Homebrew formula was not generated"