From 49f4af2cbe443b4c0ecf48be1597d64e709a5190 Mon Sep 17 00:00:00 2001 From: Wenyu Huang Date: Mon, 11 May 2026 16:16:20 +0000 Subject: [PATCH 1/6] feat(restart): add restart policy and stop state Introduce the restart policy model and persisted stop metadata needed for automatic crash recovery. This adds RestartPolicy variants, exponential backoff calculation, effective health-check selection, and the Crashed and Restarting BoxStatus values. It also extends BoxState and BoxInfo with StopInfo and last restart error fields so future runtime restart logic can persist crash context. SDK status string conversions are updated in the same commit because adding BoxStatus variants makes those matches exhaustive. Signed-off-by: Wenyu Huang --- sdks/c/src/info.rs | 2 + sdks/node/src/info.rs | 2 + sdks/python/src/info.rs | 2 + src/boxlite/src/lib.rs | 3 +- src/boxlite/src/litebox/mod.rs | 2 +- src/boxlite/src/litebox/state.rs | 140 +++++++++- src/boxlite/src/rest/types.rs | 2 + src/boxlite/src/runtime/advanced_options.rs | 277 ++++++++++++++++++++ src/boxlite/src/runtime/types.rs | 9 + 9 files changed, 429 insertions(+), 10 deletions(-) diff --git a/sdks/c/src/info.rs b/sdks/c/src/info.rs index d40bcde61..f03142b50 100644 --- a/sdks/c/src/info.rs +++ b/sdks/c/src/info.rs @@ -51,6 +51,8 @@ fn status_to_str(status: BoxStatus) -> &'static str { BoxStatus::Running => "running", BoxStatus::Stopping => "stopping", BoxStatus::Stopped => "stopped", + BoxStatus::Crashed => "crashed", + BoxStatus::Restarting => "restarting", BoxStatus::Paused => "paused", BoxStatus::Failed => "failed", } diff --git a/sdks/node/src/info.rs b/sdks/node/src/info.rs index b5644122e..c9920d0fc 100644 --- a/sdks/node/src/info.rs +++ b/sdks/node/src/info.rs @@ -75,6 +75,8 @@ fn status_to_string(status: BoxStatus) -> String { BoxStatus::Running => "running", BoxStatus::Stopping => "stopping", BoxStatus::Stopped => "stopped", + BoxStatus::Crashed => "crashed", + BoxStatus::Restarting => "restarting", BoxStatus::Paused => "paused", BoxStatus::Failed => "failed", } diff --git a/sdks/python/src/info.rs b/sdks/python/src/info.rs index 240de374f..c3b73882a 100644 --- a/sdks/python/src/info.rs +++ b/sdks/python/src/info.rs @@ -128,6 +128,8 @@ fn status_to_string(status: BoxStatus) -> String { BoxStatus::Running => "running", BoxStatus::Stopping => "stopping", BoxStatus::Stopped => "stopped", + BoxStatus::Crashed => "crashed", + BoxStatus::Restarting => "restarting", BoxStatus::Paused => "paused", BoxStatus::Failed => "failed", } diff --git a/src/boxlite/src/lib.rs b/src/boxlite/src/lib.rs index 8dc4c1eac..c8a012292 100644 --- a/src/boxlite/src/lib.rs +++ b/src/boxlite/src/lib.rs @@ -48,7 +48,7 @@ pub use litebox::{ }; pub use metrics::{BoxMetrics, RuntimeMetrics}; pub use runtime::advanced_options::{ - AdvancedBoxOptions, HealthCheckOptions, ResourceLimits, SecurityOptions, + AdvancedBoxOptions, HealthCheckOptions, ResourceLimits, RestartPolicy, SecurityOptions, }; pub use runtime::options::{ BoxArchive, BoxOptions, BoxliteOptions, CloneOptions, ExportOptions, ImageRegistry, @@ -56,6 +56,7 @@ pub use runtime::options::{ }; /// Boxlite library version (from CARGO_PKG_VERSION at compile time). pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +pub use litebox::{StopCause, StopInfo}; pub use runtime::id::{BaseDiskID, BaseDiskIDMint, BoxID, BoxIDMint}; pub use runtime::types::ContainerID; pub use runtime::types::{BoxInfo, BoxLifecyclePolicy, BoxState, BoxStateInfo, BoxStatus}; diff --git a/src/boxlite/src/litebox/mod.rs b/src/boxlite/src/litebox/mod.rs index 8b025ae46..c6e4bdb47 100644 --- a/src/boxlite/src/litebox/mod.rs +++ b/src/boxlite/src/litebox/mod.rs @@ -24,7 +24,7 @@ pub use exec::{BoxCommand, ExecResult, ExecStderr, ExecStdin, ExecStdout, Execut pub(crate) use manager::BoxManager; pub use network::{BoxConnection, BoxEndpoint, BoxTunnel, NetworkHandle}; pub use snapshot::SnapshotHandle; -pub use state::{BoxState, BoxStatus, HealthState, HealthStatus}; +pub use state::{BoxState, BoxStatus, HealthState, HealthStatus, StopCause, StopInfo}; pub(crate) use box_impl::SharedBoxImpl; pub(crate) use init::BoxBuilder; diff --git a/src/boxlite/src/litebox/state.rs b/src/boxlite/src/litebox/state.rs index 63bf1db29..5a8b5db07 100644 --- a/src/boxlite/src/litebox/state.rs +++ b/src/boxlite/src/litebox/state.rs @@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize}; /// SIGCONT → Running (VM resumed) /// stop() → Stopped (VM terminated, can restart) /// init err → Failed (record preserved with error_reason) +/// crash → Crashed (health check detected shim death) /// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] @@ -42,6 +43,14 @@ pub enum BoxStatus { /// Rootfs is preserved, box can be restarted. Stopped, + /// Box crashed (health check detected shim process death). + /// Evaluates restart policy to decide next action. + Crashed, + + /// Box is in the process of being restarted after a crash. + /// Transient state during backoff wait before new VM start. + Restarting, + /// Box VM is frozen via SIGSTOP (all vCPUs and virtio backends paused). /// Used during export/snapshot for point-in-time consistency. /// Equivalent to Docker's cgroup freezer pause. @@ -77,31 +86,48 @@ impl BoxStatus { /// Check if this status represents a transient state. pub fn is_transient(&self) -> bool { - matches!(self, BoxStatus::Stopping) + matches!(self, BoxStatus::Stopping | BoxStatus::Restarting) + } + + /// Check if this status requires health check monitoring. + /// Only Running boxes have an active VM process that needs monitoring. + /// Restarting is a transient state with no VM yet - monitoring starts after transition to Running. + pub fn requires_monitoring(&self) -> bool { + matches!(self, BoxStatus::Running) } /// Check if start() can be called from this state. /// Configured boxes need first start, Stopped and Failed boxes can be retried. + /// and Restarting resumes the crash-recovery restart pipeline. pub fn can_start(&self) -> bool { matches!( self, - BoxStatus::Configured | BoxStatus::Stopped | BoxStatus::Failed + BoxStatus::Configured | BoxStatus::Stopped | BoxStatus::Restarting | BoxStatus::Failed ) } /// Check if stop() can be called from this state. - /// Running and Paused boxes can be stopped. + /// Running, Crashed, Restarting, and Paused boxes can be stopped. + /// + /// `Crashed` has no live VM left, but stop() is still a useful explicit + /// transition to acknowledge the crash and mark the box as `Stopped`. pub fn can_stop(&self) -> bool { - matches!(self, BoxStatus::Running | BoxStatus::Paused) + matches!( + self, + BoxStatus::Running | BoxStatus::Crashed | BoxStatus::Restarting | BoxStatus::Paused + ) } /// Check if remove() can be called from this state. - /// Configured, Stopped, Failed, and Unknown boxes can be removed. - /// Failed is included so DESTROY_SANDBOX can clean up boxes whose init failed. + /// Configured, Stopped, Crashed, Failed, and Unknown boxes can be removed. pub fn can_remove(&self) -> bool { matches!( self, - BoxStatus::Configured | BoxStatus::Stopped | BoxStatus::Failed | BoxStatus::Unknown + BoxStatus::Configured + | BoxStatus::Stopped + | BoxStatus::Crashed + | BoxStatus::Failed + | BoxStatus::Unknown ) } @@ -136,9 +162,10 @@ impl BoxStatus { (Configured, Stopped) | (Configured, Failed) | (Configured, Unknown) | - // Running → Stopping (graceful), Stopped (crash), Paused (SIGSTOP), or Failed (runtime crash) + // Running → Stopping (graceful), Crashed (shim died), Paused (SIGSTOP), or Stopped (Running, Stopping) | (Running, Stopped) | + (Running, Crashed) | (Running, Paused) | (Running, Failed) | (Running, Unknown) | @@ -150,6 +177,14 @@ impl BoxStatus { (Stopped, Running) | (Stopped, Failed) | (Stopped, Unknown) | + // Crashed → Stopped (no policy) or Restarting (has policy) + (Crashed, Stopped) | + (Crashed, Restarting) | + (Crashed, Unknown) | + // Restarting → Running (success), Stopped (cancelled/max retries) + (Restarting, Running) | + (Restarting, Stopped) | + (Restarting, Unknown) | // Paused → Running (SIGCONT resume) or Stopped (killed while paused) (Paused, Running) | (Paused, Stopped) | @@ -169,6 +204,8 @@ impl BoxStatus { BoxStatus::Running => "running", BoxStatus::Stopping => "stopping", BoxStatus::Stopped => "stopped", + BoxStatus::Crashed => "crashed", + BoxStatus::Restarting => "restarting", BoxStatus::Paused => "paused", BoxStatus::Failed => "failed", } @@ -187,6 +224,8 @@ impl std::str::FromStr for BoxStatus { "running" => Ok(BoxStatus::Running), "stopping" => Ok(BoxStatus::Stopping), "stopped" => Ok(BoxStatus::Stopped), + "crashed" => Ok(BoxStatus::Crashed), + "restarting" => Ok(BoxStatus::Restarting), "paused" => Ok(BoxStatus::Paused), "failed" => Ok(BoxStatus::Failed), // Legacy: old transient statuses map to Stopped (DB backward compat) @@ -232,6 +271,48 @@ pub struct BoxState { /// keeps existing DB rows readable without migration. #[serde(default)] pub exit_code: Option, + /// Stop info (valid when status is Stopped/Crashed/Restarting). + #[serde(default)] + pub stop_info: StopInfo, + /// Error message from the last failed auto-restart attempt (cleared on success). + #[serde(default)] + pub last_restart_error: Option, +} + +/// Why the box stopped. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StopCause { + /// User called stop() or normal exit. + #[default] + Normal, + /// Crashed but no restart policy configured. + CrashedNoPolicy, + /// Restart policy exhausted max attempts. + MaxRetriesExceeded, + /// System reboot detected. + SystemReboot, + /// Restart attempt failed (e.g., VM failed to start). + RestartFailed, + /// Unknown/unexpected stop cause (should not happen in normal operation). + Unknown, +} + +/// Stop info for a stopped box (valid when status is Stopped/Crashed). +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct StopInfo { + /// Why the box stopped. + #[serde(default)] + pub cause: StopCause, + /// Exit code from the shim process (if available). + pub exit_code: Option, + /// When the box stopped/crashed (UTC). + pub exit_time: Option>, + /// How many restart attempts in the last sequence. + #[serde(default)] + pub restart_count: u32, + /// When the last successful restart happened (UTC). + pub restarted_at: Option>, } /// Health status of a box. @@ -331,6 +412,8 @@ impl BoxState { health_status: HealthStatus::new(), error_reason: None, exit_code: None, + stop_info: StopInfo::default(), + last_restart_error: None, } } @@ -381,6 +464,9 @@ impl BoxState { pub fn mark_stop(&mut self) { self.status = BoxStatus::Stopped; self.pid = None; + self.error_reason = None; + self.stop_info.cause = StopCause::Normal; + self.stop_info.exit_time = Some(Utc::now()); self.last_updated = Utc::now(); } @@ -406,6 +492,8 @@ impl BoxState { pub fn reset_for_reboot(&mut self) { if self.status.is_active() { self.status = BoxStatus::Stopped; + self.stop_info.cause = StopCause::SystemReboot; + self.stop_info.exit_time = Some(Utc::now()); } self.pid = None; self.last_updated = Utc::now(); @@ -478,6 +566,7 @@ mod tests { assert!(!BoxStatus::Running.can_start()); assert!(!BoxStatus::Stopping.can_start()); assert!(BoxStatus::Stopped.can_start()); + assert!(BoxStatus::Restarting.can_start()); assert!(!BoxStatus::Paused.can_start()); assert!(!BoxStatus::Unknown.can_start()); } @@ -488,6 +577,8 @@ mod tests { assert!(BoxStatus::Running.can_stop()); assert!(!BoxStatus::Stopping.can_stop()); assert!(!BoxStatus::Stopped.can_stop()); + assert!(BoxStatus::Crashed.can_stop()); + assert!(BoxStatus::Restarting.can_stop()); assert!(BoxStatus::Paused.can_stop()); assert!(!BoxStatus::Unknown.can_stop()); } @@ -1014,4 +1105,37 @@ mod tests { assert_eq!(state.health_status.failures, 0); assert!(state.health_status.last_check.is_none()); } + + // ==================================================================== + // requires_monitoring tests + // ==================================================================== + + #[test] + fn test_requires_monitoring_only_running() { + // Only Running status requires monitoring + assert!(!BoxStatus::Unknown.requires_monitoring()); + assert!(!BoxStatus::Configured.requires_monitoring()); + assert!(BoxStatus::Running.requires_monitoring()); + assert!(!BoxStatus::Stopping.requires_monitoring()); + assert!(!BoxStatus::Stopped.requires_monitoring()); + assert!(!BoxStatus::Crashed.requires_monitoring()); + assert!(!BoxStatus::Restarting.requires_monitoring()); // Transient, no VM yet + assert!(!BoxStatus::Paused.requires_monitoring()); + } + + // ==================================================================== + // can_start tests (includes restart from Stopped/Restarting) + // ==================================================================== + + #[test] + fn test_can_start_from_stopped_or_crashed() { + assert!(!BoxStatus::Unknown.can_start()); + assert!(BoxStatus::Configured.can_start()); // First start + assert!(!BoxStatus::Running.can_start()); + assert!(!BoxStatus::Stopping.can_start()); + assert!(BoxStatus::Stopped.can_start()); // Restart + assert!(!BoxStatus::Crashed.can_start()); // Must transition to Stopped first + assert!(BoxStatus::Restarting.can_start()); // Crash recovery restart + assert!(!BoxStatus::Paused.can_start()); + } } diff --git a/src/boxlite/src/rest/types.rs b/src/boxlite/src/rest/types.rs index 4ae496828..0be977339 100644 --- a/src/boxlite/src/rest/types.rs +++ b/src/boxlite/src/rest/types.rs @@ -293,6 +293,8 @@ impl BoxResponse { auto_resume: self.auto_resume, health_status: crate::litebox::HealthStatus::new(), // REST API doesn't provide health status exit_code: self.exit_code, + stop_info: crate::litebox::StopInfo::default(), // REST API doesn't provide stop info + last_restart_error: None, // REST API doesn't provide restart error }) } } diff --git a/src/boxlite/src/runtime/advanced_options.rs b/src/boxlite/src/runtime/advanced_options.rs index 0d2ca8132..24e06f76a 100644 --- a/src/boxlite/src/runtime/advanced_options.rs +++ b/src/boxlite/src/runtime/advanced_options.rs @@ -599,4 +599,281 @@ pub struct AdvancedBoxOptions { /// Most users should rely on the defaults. #[serde(default)] pub health_check: Option, + + /// Restart policy for automatic restart on crash. + /// + /// When set, the health check task will evaluate the policy when the + /// shim process dies and automatically restart the box if the policy allows it. + /// + /// If `health_check` is not configured but `restart_policy` is, a default + /// health check is auto-enabled (interval=5s, timeout=10s, retries=3, start_period=60s). + #[serde(default)] + pub restart_policy: Option, +} + +/// Restart policy for automatic restart on crash. +/// +/// Similar to Docker's restart policy. Controls what happens when a box's +/// shim process crashes. +/// +/// # Example +/// +/// ``` +/// use boxlite::runtime::advanced_options::{AdvancedBoxOptions, RestartPolicy}; +/// +/// let opts = AdvancedBoxOptions { +/// restart_policy: Some(RestartPolicy::Always), +/// ..Default::default() +/// }; +/// ``` +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum RestartPolicy { + /// Never restart (default). + No, + /// Always restart regardless of exit status. Unlimited retries with exponential backoff. + Always, + /// Restart only on non-zero exit code, limited to max_retries consecutive failures. + OnFailure { + /// Maximum consecutive restart attempts before giving up. + max_retries: u32, + }, + /// Always restart unless user explicitly called stop(). Unlimited retries. + UnlessStopped, +} + +impl RestartPolicy { + /// Evaluate whether a restart should happen based on exit code and restart count. + /// + /// # Arguments + /// * `exit_code` - Exit code from the shim process (None = signal/unknown) + /// * `restart_count` - Current consecutive restart attempt count + /// + /// # Returns + /// `true` if the box should be restarted + /// + /// # Warning + /// `Always` and `UnlessStopped` restart regardless of exit code. If the shim + /// exits cleanly (exit_code == 0) repeatedly, this results in an infinite + /// restart loop. Use `UnlessStopped` when you want a user-initiated `stop()` + /// to break the loop (the only way to stop an `Always` loop is to remove the box). + pub fn should_restart(&self, exit_code: Option, restart_count: u32) -> bool { + match self { + RestartPolicy::No => false, + RestartPolicy::Always => true, + RestartPolicy::OnFailure { max_retries } => { + // Only restart on non-zero exit code (explicit failure) + // None (no exit file) means normal/clean exit - no restart + let is_failure = exit_code.is_some_and(|code| code != 0); + is_failure && restart_count < *max_retries + } + RestartPolicy::UnlessStopped => true, + } + } + + /// Check if this policy has unlimited retries. + pub fn is_unlimited_retries(&self) -> bool { + matches!(self, RestartPolicy::Always | RestartPolicy::UnlessStopped) + } + + /// Default health check config to use when restart_policy is set but health_check is not. + pub fn default_health_check() -> HealthCheckOptions { + HealthCheckOptions { + interval: Duration::from_secs(5), + timeout: Duration::from_secs(10), + retries: 3, + start_period: Duration::from_secs(60), + } + } +} + +impl AdvancedBoxOptions { + /// Get the effective health check config. + /// + /// Returns user-configured health check if set, or auto-enables default + /// health check when restart_policy is configured without one. + pub fn effective_health_check(&self) -> Option { + if let Some(ref hc) = self.health_check { + Some(hc.clone()) + } else if self.restart_policy.is_some() { + Some(RestartPolicy::default_health_check()) + } else { + None + } + } +} + +/// Calculate exponential backoff delay for restart attempts. +/// +/// Base: 100ms, doubles each attempt, capped at 30s. +/// Sequence: 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, 6.4s, 12.8s, 25.6s, 30s (capped) +pub fn calculate_backoff(restart_count: u32) -> Duration { + let base_ms: u64 = 100; + let max_ms: u64 = 30_000; + let exp = 1u64.checked_shl(restart_count.min(18)).unwrap_or(u64::MAX); // Cap shift to avoid overflow + Duration::from_millis(base_ms.saturating_mul(exp).min(max_ms)) +} + +#[cfg(test)] +mod tests { + use super::*; + + // ==================================================================== + // RestartPolicy::should_restart tests + // ==================================================================== + + #[test] + fn test_restart_policy_no_never_restarts() { + let policy = RestartPolicy::No; + assert!(!policy.should_restart(Some(0), 0)); + assert!(!policy.should_restart(Some(1), 0)); + assert!(!policy.should_restart(None, 0)); + assert!(!policy.should_restart(Some(0), 100)); + } + + #[test] + fn test_restart_policy_always_always_restarts() { + let policy = RestartPolicy::Always; + assert!(policy.should_restart(Some(0), 0)); + assert!(policy.should_restart(Some(1), 0)); + assert!(policy.should_restart(None, 0)); + assert!(policy.should_restart(Some(0), 100)); // Unlimited retries + } + + #[test] + fn test_restart_policy_unless_stopped_always_restarts() { + let policy = RestartPolicy::UnlessStopped; + assert!(policy.should_restart(Some(0), 0)); + assert!(policy.should_restart(Some(1), 0)); + assert!(policy.should_restart(None, 0)); + assert!(policy.should_restart(Some(0), 100)); // Unlimited retries + } + + #[test] + fn test_restart_policy_on_failure_restarts_on_non_zero_exit() { + let policy = RestartPolicy::OnFailure { max_retries: 3 }; + // Non-zero exit code should restart + assert!(policy.should_restart(Some(1), 0)); + assert!(policy.should_restart(Some(127), 1)); + assert!(policy.should_restart(Some(-1), 2)); + } + + #[test] + fn test_restart_policy_on_failure_no_restart_on_zero_exit() { + let policy = RestartPolicy::OnFailure { max_retries: 3 }; + // Zero exit code should NOT restart + assert!(!policy.should_restart(Some(0), 0)); + assert!(!policy.should_restart(Some(0), 2)); + } + + #[test] + fn test_restart_policy_on_failure_no_restart_on_no_exit_file() { + let policy = RestartPolicy::OnFailure { max_retries: 3 }; + // No exit file (None) means clean/normal exit - no restart + assert!(!policy.should_restart(None, 0)); + assert!(!policy.should_restart(None, 1)); + } + + #[test] + fn test_restart_policy_on_failure_respects_max_retries() { + let policy = RestartPolicy::OnFailure { max_retries: 3 }; + // Should restart when under max_retries + assert!(policy.should_restart(Some(1), 0)); + assert!(policy.should_restart(Some(1), 1)); + assert!(policy.should_restart(Some(1), 2)); + // Should NOT restart when at max_retries + assert!(!policy.should_restart(Some(1), 3)); + assert!(!policy.should_restart(Some(1), 4)); + } + + #[test] + fn test_restart_policy_on_failure_zero_max_retries() { + let policy = RestartPolicy::OnFailure { max_retries: 0 }; + // With max_retries=0, should never restart + assert!(!policy.should_restart(Some(1), 0)); + assert!(!policy.should_restart(None, 0)); + } + + // ==================================================================== + // RestartPolicy::is_unlimited_retries tests + // ==================================================================== + + #[test] + fn test_is_unlimited_retries() { + assert!(!RestartPolicy::No.is_unlimited_retries()); + assert!(RestartPolicy::Always.is_unlimited_retries()); + assert!(RestartPolicy::UnlessStopped.is_unlimited_retries()); + assert!(!RestartPolicy::OnFailure { max_retries: 3 }.is_unlimited_retries()); + } + + // ==================================================================== + // calculate_backoff tests + // ==================================================================== + + #[test] + fn test_calculate_backoff_sequence() { + assert_eq!(calculate_backoff(0), Duration::from_millis(100)); + assert_eq!(calculate_backoff(1), Duration::from_millis(200)); + assert_eq!(calculate_backoff(2), Duration::from_millis(400)); + assert_eq!(calculate_backoff(3), Duration::from_millis(800)); + assert_eq!(calculate_backoff(4), Duration::from_millis(1600)); + assert_eq!(calculate_backoff(5), Duration::from_millis(3200)); + assert_eq!(calculate_backoff(6), Duration::from_millis(6400)); + assert_eq!(calculate_backoff(7), Duration::from_millis(12800)); + assert_eq!(calculate_backoff(8), Duration::from_millis(25600)); + } + + #[test] + fn test_calculate_backoff_capped_at_30s() { + // After 30s cap is reached + assert_eq!(calculate_backoff(9), Duration::from_millis(30000)); + assert_eq!(calculate_backoff(10), Duration::from_millis(30000)); + assert_eq!(calculate_backoff(100), Duration::from_millis(30000)); + } + + // ==================================================================== + // AdvancedBoxOptions::effective_health_check tests + // ==================================================================== + + #[test] + fn test_effective_health_check_user_config() { + let user_config = HealthCheckOptions { + interval: Duration::from_secs(10), + timeout: Duration::from_secs(5), + retries: 5, + start_period: Duration::from_secs(30), + }; + let opts = AdvancedBoxOptions { + health_check: Some(user_config.clone()), + restart_policy: Some(RestartPolicy::Always), + ..Default::default() + }; + let effective = opts.effective_health_check().unwrap(); + assert_eq!(effective.interval, Duration::from_secs(10)); + assert_eq!(effective.retries, 5); + } + + #[test] + fn test_effective_health_check_auto_enabled_for_restart_policy() { + let opts = AdvancedBoxOptions { + health_check: None, + restart_policy: Some(RestartPolicy::Always), + ..Default::default() + }; + let effective = opts.effective_health_check().unwrap(); + assert_eq!(effective.interval, Duration::from_secs(5)); + assert_eq!(effective.timeout, Duration::from_secs(10)); + assert_eq!(effective.retries, 3); + assert_eq!(effective.start_period, Duration::from_secs(60)); + } + + #[test] + fn test_effective_health_check_none_when_no_policy() { + let opts = AdvancedBoxOptions { + health_check: None, + restart_policy: None, + ..Default::default() + }; + assert!(opts.effective_health_check().is_none()); + } } diff --git a/src/boxlite/src/runtime/types.rs b/src/boxlite/src/runtime/types.rs index 2159c442f..e50cd4145 100644 --- a/src/boxlite/src/runtime/types.rs +++ b/src/boxlite/src/runtime/types.rs @@ -354,6 +354,13 @@ pub struct BoxInfo { /// Exit code of the container's init process, when the box stopped /// because its main command exited (docker semantics). pub exit_code: Option, + /// Stop info (valid when status is Stopped/Crashed/Restarting). + #[serde(default)] + pub stop_info: crate::litebox::StopInfo, + + /// Last restart error message (if any). + #[serde(default)] + pub last_restart_error: Option, } impl BoxInfo { @@ -383,6 +390,8 @@ impl BoxInfo { auto_resume: config.options.auto_resume.unwrap_or(true), health_status: state.health_status, exit_code: state.exit_code, + stop_info: state.stop_info.clone(), + last_restart_error: None, } } } From b3ef55dc7a08debe5cf08eed676ff5a71017fe91 Mon Sep 17 00:00:00 2001 From: Wenyu Huang Date: Mon, 11 May 2026 16:29:50 +0000 Subject: [PATCH 2/6] refactor(runtime): introduce stable box handles Add BoxHandle as the stable backend used by LiteBox and move the runtime cache from BoxImpl values to BoxHandle values. The handle delegates normal box and snapshot operations to the current BoxImpl. This keeps the public LiteBox value stable while allowing a later restart path to replace the underlying VM implementation. No restart behavior is introduced here; this is a behavior-preserving cache and handle refactor. Signed-off-by: Wenyu Huang --- src/boxlite/src/litebox/box_impl.rs | 2 +- src/boxlite/src/litebox/handle.rs | 166 ++++++++++++++++++++++++++++ src/boxlite/src/litebox/mod.rs | 3 +- src/boxlite/src/runtime/rt_impl.rs | 145 ++++++++++++------------ 4 files changed, 246 insertions(+), 70 deletions(-) create mode 100644 src/boxlite/src/litebox/handle.rs diff --git a/src/boxlite/src/litebox/box_impl.rs b/src/boxlite/src/litebox/box_impl.rs index a265f6f18..06daca902 100644 --- a/src/boxlite/src/litebox/box_impl.rs +++ b/src/boxlite/src/litebox/box_impl.rs @@ -718,7 +718,7 @@ impl BoxImpl { // Invalidate cache so new handles get fresh BoxImpl self.runtime - .invalidate_box_impl(self.id(), self.config.name.as_deref()); + .invalidate_box_handle(self.id(), self.config.name.as_deref()); for listener in &self.event_listeners { listener.on_box_stopped(&self.config.id, None); diff --git a/src/boxlite/src/litebox/handle.rs b/src/boxlite/src/litebox/handle.rs new file mode 100644 index 000000000..c5a155d2e --- /dev/null +++ b/src/boxlite/src/litebox/handle.rs @@ -0,0 +1,166 @@ +//! Stable handle for a box whose underlying VM implementation can be replaced. + +use std::net::SocketAddr; +use std::path::Path; +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::RwLock; + +use super::box_impl::SharedBoxImpl; +use super::copy::CopyOptions; +use super::local_snapshot::LocalSnapshotBackend; +use super::snapshot_mgr::SnapshotInfo; +use super::{BoxCommand, BoxTunnel, Execution, LiteBox}; +use crate::BoxID; +use crate::metrics::BoxMetrics; +use crate::runtime::backend::{BoxBackend, BoxNetworkBackend, SnapshotBackend}; +use crate::runtime::options::{BoxArchive, CloneOptions, ExportOptions, SnapshotOptions}; +use crate::runtime::types::BoxInfo; +use boxlite_shared::errors::BoxliteResult; + +pub(crate) type SharedBoxHandle = Arc; + +/// Stable API handle for a box. +/// +/// `LiteBox` points at this handle, while restart can replace the current +/// `BoxImpl` underneath it. This keeps existing user handles usable after an +/// automatic restart. +pub(crate) struct BoxHandle { + id: BoxID, + name: Option, + current: RwLock, +} + +impl BoxHandle { + pub(crate) fn new(inner: SharedBoxImpl) -> Self { + Self { + id: inner.id().clone(), + name: inner.config.name.clone(), + current: RwLock::new(inner), + } + } + + pub(crate) fn current(&self) -> SharedBoxImpl { + Arc::clone(&self.current.read()) + } + + pub(crate) fn id(&self) -> &BoxID { + &self.id + } + + pub(crate) fn info(&self) -> BoxInfo { + self.current().info() + } + +} + +#[async_trait] +impl BoxBackend for BoxHandle { + fn id(&self) -> &BoxID { + &self.id + } + + fn name(&self) -> Option<&str> { + self.name.as_deref() + } + + fn info(&self) -> BoxInfo { + self.current().info() + } + + async fn start(&self) -> BoxliteResult<()> { + self.current().start().await + } + + async fn exec(&self, command: BoxCommand) -> BoxliteResult { + self.current().exec(command).await + } + + async fn attach(&self, execution_id: Option<&str>) -> BoxliteResult { + crate::runtime::backend::BoxBackend::attach(&*self.current(), execution_id).await + } + + async fn metrics(&self) -> BoxliteResult { + self.current().metrics().await + } + + async fn stop(&self) -> BoxliteResult<()> { + self.current().stop().await + } + + async fn copy_into( + &self, + host_src: &Path, + container_dst: &str, + opts: CopyOptions, + ) -> BoxliteResult<()> { + self.current() + .copy_into(host_src, container_dst, opts) + .await + } + + async fn copy_out( + &self, + container_src: &str, + host_dst: &Path, + opts: CopyOptions, + ) -> BoxliteResult<()> { + self.current().copy_out(container_src, host_dst, opts).await + } + + async fn clone_box( + &self, + options: CloneOptions, + name: Option, + ) -> BoxliteResult { + self.current().clone_box(options, name).await + } + + async fn clone_boxes( + &self, + options: CloneOptions, + count: usize, + names: Vec, + ) -> BoxliteResult> { + self.current().clone_boxes(options, count, names).await + } + + async fn export_box(&self, options: ExportOptions, dest: &Path) -> BoxliteResult { + self.current().export_box(options, dest).await + } +} + +#[async_trait] +impl BoxNetworkBackend for BoxHandle { + async fn tunnel(&self, target: SocketAddr) -> BoxliteResult { + crate::runtime::backend::BoxNetworkBackend::tunnel(&*self.current(), target).await + } +} + +#[async_trait] +impl SnapshotBackend for BoxHandle { + async fn create(&self, options: SnapshotOptions, name: &str) -> BoxliteResult { + LocalSnapshotBackend::new(self.current()) + .create(options, name) + .await + } + + async fn list(&self) -> BoxliteResult> { + LocalSnapshotBackend::new(self.current()).list().await + } + + async fn get(&self, name: &str) -> BoxliteResult> { + LocalSnapshotBackend::new(self.current()).get(name).await + } + + async fn remove(&self, name: &str) -> BoxliteResult<()> { + LocalSnapshotBackend::new(self.current()).remove(name).await + } + + async fn restore(&self, name: &str) -> BoxliteResult<()> { + LocalSnapshotBackend::new(self.current()) + .restore(name) + .await + } +} diff --git a/src/boxlite/src/litebox/mod.rs b/src/boxlite/src/litebox/mod.rs index c6e4bdb47..5255f6286 100644 --- a/src/boxlite/src/litebox/mod.rs +++ b/src/boxlite/src/litebox/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod config; pub mod copy; mod crash_report; mod exec; +pub(crate) mod handle; mod init; pub(crate) mod local_snapshot; mod manager; @@ -27,8 +28,8 @@ pub use snapshot::SnapshotHandle; pub use state::{BoxState, BoxStatus, HealthState, HealthStatus, StopCause, StopInfo}; pub(crate) use box_impl::SharedBoxImpl; +pub(crate) use handle::{BoxHandle, SharedBoxHandle}; pub(crate) use init::BoxBuilder; -pub(crate) use local_snapshot::LocalSnapshotBackend; use std::path::Path; use std::sync::Arc; diff --git a/src/boxlite/src/runtime/rt_impl.rs b/src/boxlite/src/runtime/rt_impl.rs index 3311586d0..39dc6ffff 100644 --- a/src/boxlite/src/runtime/rt_impl.rs +++ b/src/boxlite/src/runtime/rt_impl.rs @@ -1,7 +1,7 @@ use crate::db::{BoxStore, Database}; use crate::images::{ImageDiskManager, ImageManager}; use crate::litebox::config::BoxConfig; -use crate::litebox::{BoxManager, LiteBox, LocalSnapshotBackend, SharedBoxImpl}; +use crate::litebox::{BoxHandle, BoxManager, LiteBox, SharedBoxHandle, SharedBoxImpl}; use crate::lock::{FileLockManager, LockManager}; use crate::metrics::{RuntimeMetrics, RuntimeMetricsStorage}; use crate::rootfs::guest::{GuestRootfs, GuestRootfsManager}; @@ -20,7 +20,7 @@ use std::sync::{Arc, RwLock, Weak}; use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; -fn litebox_from_impl(box_impl: SharedBoxImpl) -> LiteBox { +fn litebox_from_handle(handle: SharedBoxHandle) -> LiteBox { // Every handle this runtime hands out starts following its box's main // command, if the box is running and nobody is following it yet. // @@ -30,12 +30,11 @@ fn litebox_from_impl(box_impl: SharedBoxImpl) -> LiteBox { // (`boxlite serve`, the cloud), and without it such a box would run to // completion entirely unobserved and be reported Running forever: exactly // the lie the watcher exists to stop telling. - box_impl.arm_watcher(None); + handle.current().arm_watcher(None); - let box_backend: Arc = box_impl.clone(); - let network_backend: Arc = box_impl.clone(); - let snapshot_backend: Arc = - Arc::new(LocalSnapshotBackend::new(box_impl)); + let box_backend: Arc = handle.clone(); + let network_backend: Arc = handle.clone(); + let snapshot_backend: Arc = handle; LiteBox::new(box_backend, network_backend, snapshot_backend) } @@ -50,6 +49,7 @@ fn litebox_from_impl(box_impl: SharedBoxImpl) -> LiteBox { pub(crate) fn record_main_command_exit(state: &mut BoxState, exit_file: &std::path::Path) { if let Some(record) = boxlite_shared::layout::ExitRecord::read(exit_file) { state.exit_code = Some(record.exit_code); + state.stop_info.exit_code = Some(record.exit_code); } state.mark_stop(); } @@ -183,11 +183,11 @@ pub struct RuntimeImpl { /// Acquire this when you need atomicity across multiple operations on /// box_manager or image_manager. pub struct SynchronizedState { - /// Cache of active BoxImpl instances by ID. + /// Cache of active BoxHandle instances by ID. /// Uses Weak to allow automatic cleanup when all handles are dropped. - active_boxes_by_id: HashMap>, - /// Cache of active BoxImpl instances by name (only for named boxes). - active_boxes_by_name: HashMap>, + active_handles_by_id: HashMap>, + /// Cache of active BoxHandle instances by name (only for named boxes). + active_handles_by_name: HashMap>, } impl RuntimeImpl { @@ -296,8 +296,8 @@ impl RuntimeImpl { let inner = Arc::new(Self { sync_state: RwLock::new(SynchronizedState { - active_boxes_by_id: HashMap::new(), - active_boxes_by_name: HashMap::new(), + active_handles_by_id: HashMap::new(), + active_handles_by_name: HashMap::new(), }), box_manager: BoxManager::new(box_store), image_manager, @@ -389,15 +389,15 @@ impl RuntimeImpl { if let Some(ref name) = name && let Some((config, state)) = self.box_manager.lookup_box(name)? { - return if reuse_existing { - let (box_impl, _) = self.get_or_create_box_impl(config, state); - Ok((litebox_from_impl(box_impl), false)) + if reuse_existing { + let (handle, _) = self.get_or_create_box_handle(config, state); + return Ok((litebox_from_handle(handle), false)); } else { - Err(BoxliteError::InvalidArgument(format!( + return Err(BoxliteError::InvalidArgument(format!( "box with name '{}' already exists", name - ))) - }; + ))); + } } // Initialize box variables with defaults @@ -430,8 +430,8 @@ impl RuntimeImpl { && let Some(ref name) = name && let Some((config, state)) = self.box_manager.lookup_box(name)? { - let (box_impl, _) = self.get_or_create_box_impl(config, state); - return Ok((litebox_from_impl(box_impl), false)); + let (handle, _) = self.get_or_create_box_handle(config, state); + return Ok((litebox_from_handle(handle), false)); } return Err(e); @@ -445,7 +445,7 @@ impl RuntimeImpl { // Create LiteBox handle with shared BoxImpl // This also checks in-memory cache for duplicate names - let (box_impl, inserted) = self.get_or_create_box_impl(config, state); + let (handle, inserted) = self.get_or_create_box_handle(config, state); if !inserted { return Err(BoxliteError::InvalidArgument( "box with this name already exists".into(), @@ -457,7 +457,7 @@ impl RuntimeImpl { .boxes_created .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - Ok((litebox_from_impl(box_impl), true)) + Ok((litebox_from_handle(handle), true)) } /// Get a handle to an existing box by ID or name. @@ -476,19 +476,19 @@ impl RuntimeImpl { // Try as BoxID first if let Some(box_id) = BoxID::parse(id_or_name) - && let Some(weak) = sync.active_boxes_by_id.get(&box_id) + && let Some(weak) = sync.active_handles_by_id.get(&box_id) && let Some(strong) = weak.upgrade() { tracing::trace!(box_id = %box_id, "Found box in cache by ID"); - return Ok(Some(litebox_from_impl(strong))); + return Ok(Some(litebox_from_handle(strong))); } // Try as name - if let Some(weak) = sync.active_boxes_by_name.get(id_or_name) + if let Some(weak) = sync.active_handles_by_name.get(id_or_name) && let Some(strong) = weak.upgrade() { tracing::trace!(name = %id_or_name, "Found box in cache by name"); - return Ok(Some(litebox_from_impl(strong))); + return Ok(Some(litebox_from_handle(strong))); } } @@ -507,9 +507,9 @@ impl RuntimeImpl { "Retrieved box from DB, getting or creating BoxImpl" ); - let (box_impl, _) = self.get_or_create_box_impl(config, state); + let (handle, _) = self.get_or_create_box_handle(config, state); tracing::trace!(id_or_name = %id_or_name, "LiteBox created successfully"); - return Ok(Some(litebox_from_impl(box_impl))); + return Ok(Some(litebox_from_handle(handle))); } tracing::trace!(id_or_name = %id_or_name, "Box not found"); @@ -536,14 +536,14 @@ impl RuntimeImpl { // Try as BoxID first if let Some(box_id) = BoxID::parse(id_or_name) - && let Some(weak) = sync.active_boxes_by_id.get(&box_id) + && let Some(weak) = sync.active_handles_by_id.get(&box_id) && let Some(strong) = weak.upgrade() { return Ok(Some(strong.info())); } // Try as name - if let Some(weak) = sync.active_boxes_by_name.get(id_or_name) + if let Some(weak) = sync.active_handles_by_name.get(id_or_name) && let Some(strong) = weak.upgrade() { return Ok(Some(strong.info())); @@ -586,7 +586,7 @@ impl RuntimeImpl { // Add in-memory boxes not yet persisted { let sync = self.sync_state.read().unwrap(); - for (box_id, weak) in &sync.active_boxes_by_id { + for (box_id, weak) in &sync.active_handles_by_id { if !seen_ids.contains(box_id) && let Some(strong) = weak.upgrade() { @@ -611,14 +611,14 @@ impl RuntimeImpl { // Try as BoxID first if let Some(box_id) = BoxID::parse(id_or_name) - && let Some(weak) = sync.active_boxes_by_id.get(&box_id) + && let Some(weak) = sync.active_handles_by_id.get(&box_id) && weak.upgrade().is_some() { return Ok(true); } // Try as name - if let Some(weak) = sync.active_boxes_by_name.get(id_or_name) + if let Some(weak) = sync.active_handles_by_name.get(id_or_name) && weak.upgrade().is_some() { return Ok(true); @@ -678,9 +678,10 @@ impl RuntimeImpl { // Collect all active non-detached boxes let active_boxes: Vec = { let sync = self.sync_state.read().unwrap(); - sync.active_boxes_by_id + sync.active_handles_by_id .values() .filter_map(|weak| weak.upgrade()) + .map(|handle| handle.current()) .filter(|box_impl| !box_impl.config.options.detach) .collect() }; @@ -836,14 +837,14 @@ impl RuntimeImpl { // Try as BoxID first if let Some(box_id) = BoxID::parse(id_or_name) - && let Some(weak) = sync.active_boxes_by_id.get(&box_id) + && let Some(weak) = sync.active_handles_by_id.get(&box_id) && weak.upgrade().is_some() { return Ok(box_id); } // Try as name - if let Some(weak) = sync.active_boxes_by_name.get(id_or_name) + if let Some(weak) = sync.active_handles_by_name.get(id_or_name) && let Some(strong) = weak.upgrade() { return Ok(strong.id().clone()); @@ -973,7 +974,7 @@ impl RuntimeImpl { } // Invalidate cache - self.invalidate_box_impl(id, config.name.as_deref()); + self.invalidate_box_handle(id, config.name.as_deref()); tracing::info!(box_id = %id, "Removed box"); return Ok(()); @@ -982,9 +983,10 @@ impl RuntimeImpl { // Box not in database - check in-memory cache let box_impl = { let sync = self.sync_state.read().unwrap(); - sync.active_boxes_by_id + sync.active_handles_by_id .get(id) .and_then(|weak| weak.upgrade()) + .map(|handle| handle.current()) }; if let Some(box_impl) = box_impl { @@ -1020,7 +1022,7 @@ impl RuntimeImpl { } // Invalidate cache (removes from in-memory maps) - self.invalidate_box_impl(id, box_impl.config.name.as_deref()); + self.invalidate_box_handle(id, box_impl.config.name.as_deref()); // Delete box directory + its socket binding symlink box_impl.config.sockets().remove(); @@ -1482,20 +1484,26 @@ impl RuntimeImpl { // INTERNAL - BOX IMPL CACHE // ======================================================================== - /// Get existing BoxImpl from cache or create new one. + /// Create a fresh BoxImpl that is not installed in the handle cache. + fn create_box_impl(self: &Arc, config: BoxConfig, state: BoxState) -> SharedBoxImpl { + use crate::litebox::box_impl::BoxImpl; + + let box_token = self.shutdown_token.child_token(); + Arc::new(BoxImpl::new(config, state, Arc::clone(self), box_token)) + } + + /// Get existing BoxHandle from cache or create new one. /// - /// Returns `(SharedBoxImpl, inserted)` where `inserted` is true if a new BoxImpl + /// Returns `(SharedBoxHandle, inserted)` where `inserted` is true if a new BoxHandle /// was created, false if an existing one was returned. /// /// Checks both by name (if provided) and by ID. This prevents duplicate names /// even for boxes not yet persisted to database. - fn get_or_create_box_impl( + fn get_or_create_box_handle( self: &Arc, config: BoxConfig, state: BoxState, - ) -> (SharedBoxImpl, bool) { - use crate::litebox::box_impl::BoxImpl; - + ) -> (SharedBoxHandle, bool) { let box_id = config.id.clone(); let box_name = config.name.clone(); @@ -1503,54 +1511,55 @@ impl RuntimeImpl { // Check by name first (if provided) - prevents duplicate names if let Some(ref name) = box_name - && let Some(weak) = sync.active_boxes_by_name.get(name) + && let Some(weak) = sync.active_handles_by_name.get(name) { if let Some(strong) = weak.upgrade() { - tracing::trace!(name = %name, "Reusing cached BoxImpl by name"); + tracing::trace!(name = %name, "Reusing cached BoxHandle by name"); return (strong, false); } // Dead weak ref, clean it up - sync.active_boxes_by_name.remove(name); + sync.active_handles_by_name.remove(name); } // Check by ID - if let Some(weak) = sync.active_boxes_by_id.get(&box_id) { + if let Some(weak) = sync.active_handles_by_id.get(&box_id) { if let Some(strong) = weak.upgrade() { - tracing::trace!(box_id = %box_id, "Reusing cached BoxImpl by ID"); + tracing::trace!(box_id = %box_id, "Reusing cached BoxHandle by ID"); return (strong, false); } // Dead weak ref, clean it up - sync.active_boxes_by_id.remove(&box_id); + sync.active_handles_by_id.remove(&box_id); } - // Create new BoxImpl and cache in both maps - // Pass a child token so box can be cancelled independently or via runtime shutdown - let box_token = self.shutdown_token.child_token(); - let box_impl = Arc::new(BoxImpl::new(config, state, Arc::clone(self), box_token)); - let weak = Arc::downgrade(&box_impl); + // Create new BoxImpl, wrap it in a stable handle, and cache the handle + // in both maps. + let box_impl = self.create_box_impl(config, state); + let handle = Arc::new(BoxHandle::new(box_impl)); + let weak = Arc::downgrade(&handle); - sync.active_boxes_by_id.insert(box_id.clone(), weak.clone()); + sync.active_handles_by_id + .insert(box_id.clone(), weak.clone()); if let Some(name) = box_name { - sync.active_boxes_by_name.insert(name.clone(), weak); - tracing::trace!(box_id = %box_id, name = %name, "Created and cached new BoxImpl"); + sync.active_handles_by_name.insert(name.clone(), weak); + tracing::trace!(box_id = %box_id, name = %name, "Created and cached new BoxHandle"); } else { - tracing::trace!(box_id = %box_id, "Created and cached new BoxImpl (unnamed)"); + tracing::trace!(box_id = %box_id, "Created and cached new BoxHandle (unnamed)"); } - (box_impl, true) + (handle, true) } - /// Remove BoxImpl from cache. + /// Remove BoxHandle from cache. /// /// Called when box is stopped or removed. Existing handles become stale; - /// new handles from runtime.get() will get a fresh BoxImpl. - pub(crate) fn invalidate_box_impl(&self, box_id: &BoxID, box_name: Option<&str>) { + /// new handles from runtime.get() will get a fresh BoxHandle. + pub(crate) fn invalidate_box_handle(&self, box_id: &BoxID, box_name: Option<&str>) { let mut sync = self.sync_state.write().unwrap(); - sync.active_boxes_by_id.remove(box_id); + sync.active_handles_by_id.remove(box_id); if let Some(name) = box_name { - sync.active_boxes_by_name.remove(name); + sync.active_handles_by_name.remove(name); } - tracing::trace!(box_id = %box_id, name = ?box_name, "Invalidated BoxImpl cache"); + tracing::trace!(box_id = %box_id, name = ?box_name, "Invalidated BoxHandle cache"); } /// Acquire coordination lock for multi-step atomic operations. From 1d44038af8d22aff3c43e7b6b89b89a343177097 Mon Sep 17 00:00:00 2001 From: Wenyu Huang Date: Wed, 13 May 2026 16:06:37 +0000 Subject: [PATCH 3/6] refactor(lock): move box lock waits onto blocking pool Add an owned lock guard and async lock acquisition helpers that move blocking waits onto Tokio's blocking pool instead of retrying with yield_now(). Use the helper for box stop and live-state initialization so box lifecycle paths no longer spin or block a Tokio worker while waiting for the per-box lock. Signed-off-by: Wenyu Huang --- src/boxlite/src/litebox/box_impl.rs | 8 +- src/boxlite/src/lock/mod.rs | 109 ++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/src/boxlite/src/litebox/box_impl.rs b/src/boxlite/src/litebox/box_impl.rs index 06daca902..776ab5ce6 100644 --- a/src/boxlite/src/litebox/box_impl.rs +++ b/src/boxlite/src/litebox/box_impl.rs @@ -25,7 +25,7 @@ use crate::event_listener::EventListener; use crate::fs::BindMountHandle; use crate::litebox::BoxTunnel; use crate::litebox::copy::CopyOptions; -use crate::lock::LockGuard; +use crate::lock::acquire_owned_lock; use crate::metrics::{BoxMetrics, BoxMetricsStorage}; use crate::net::NetworkBackend; use crate::portal::GuestSession; @@ -1021,9 +1021,9 @@ impl BoxImpl { is_first_start ); - // Hold the lock for the duration of build operations. - // LockGuard acquires lock on creation and releases on drop. - let _guard = LockGuard::new(&*locker); + // Hold the lock for the duration of build operations without blocking + // a Tokio worker while another lifecycle operation owns it. + let _guard = acquire_owned_lock(locker).await?; // Build the box (lock is held) // The returned cleanup_guard stays armed until we disarm it after all diff --git a/src/boxlite/src/lock/mod.rs b/src/boxlite/src/lock/mod.rs index 0dfe6448d..ff2415323 100644 --- a/src/boxlite/src/lock/mod.rs +++ b/src/boxlite/src/lock/mod.rs @@ -17,6 +17,7 @@ pub use memory::InMemoryLockManager; use std::sync::Arc; use boxlite_shared::errors::{BoxliteError, BoxliteResult}; +use tokio_util::sync::CancellationToken; /// Unique identifier for a lock. /// @@ -192,6 +193,83 @@ impl Drop for LockGuard<'_> { } } +/// Owned RAII guard for async lock acquisition. +/// +/// Unlike [`LockGuard`], this guard owns an `Arc`, so it can be +/// created inside `spawn_blocking` and moved across async boundaries. +pub struct OwnedLockGuard { + lock: Arc, +} + +impl OwnedLockGuard { + /// Create a new owned guard, blocking the current thread until the lock is acquired. + pub fn new(lock: Arc) -> Self { + lock.lock(); + Self { lock } + } + + /// Try to create a new owned guard without blocking. + /// + /// Returns `None` if the lock is already held. + pub fn try_new(lock: Arc) -> Option { + if lock.try_lock() { + Some(Self { lock }) + } else { + None + } + } +} + +impl Drop for OwnedLockGuard { + fn drop(&mut self) { + self.lock.unlock(); + } +} + +/// Acquire a lock from async code without spinning on a Tokio worker thread. +pub async fn acquire_owned_lock(lock: Arc) -> BoxliteResult { + if let Some(guard) = OwnedLockGuard::try_new(Arc::clone(&lock)) { + return Ok(guard); + } + + tokio::task::spawn_blocking(move || OwnedLockGuard::new(lock)) + .await + .map_err(|e| BoxliteError::Internal(format!("lock acquisition task failed: {}", e))) +} + +/// Acquire a lock from async code, aborting the wait if shutdown is requested. +/// +/// The blocking lock acquisition itself cannot be cancelled once started. If +/// shutdown wins the select, the blocking task is detached; when it eventually +/// acquires the lock, its returned guard is dropped and releases the lock. +pub async fn acquire_owned_lock_or_cancel( + lock: Arc, + shutdown_token: &CancellationToken, + shutdown_message: impl Into, +) -> BoxliteResult { + if shutdown_token.is_cancelled() { + return Err(BoxliteError::Stopped(shutdown_message.into())); + } + + if let Some(guard) = OwnedLockGuard::try_new(Arc::clone(&lock)) { + return Ok(guard); + } + + let shutdown_message = shutdown_message.into(); + let lock_task = tokio::task::spawn_blocking(move || OwnedLockGuard::new(lock)); + + tokio::select! { + result = lock_task => { + result.map_err(|e| { + BoxliteError::Internal(format!("lock acquisition task failed: {}", e)) + }) + } + _ = shutdown_token.cancelled() => { + Err(BoxliteError::Stopped(shutdown_message)) + } + } +} + // Error helpers pub(crate) fn lock_exhausted() -> BoxliteError { BoxliteError::Internal("all locks have been allocated".to_string()) @@ -216,6 +294,7 @@ pub(crate) fn lock_invalid(id: LockId, max: u32) -> BoxliteError { #[cfg(test)] mod tests { use super::*; + use std::sync::Arc; fn test_lock_manager(manager: &dyn LockManager) { // Allocate a lock @@ -276,4 +355,34 @@ mod tests { assert!(lock.try_lock(), "should be able to acquire released lock"); lock.unlock(); } + + #[tokio::test] + async fn test_acquire_owned_lock_releases_on_drop() { + let manager = InMemoryLockManager::new(16); + let id = manager.allocate().expect("allocate"); + let lock = manager.retrieve(id).expect("retrieve"); + + { + let _guard = acquire_owned_lock(Arc::clone(&lock)).await.unwrap(); + assert!(!lock.try_lock(), "should not acquire while guard is held"); + } + + assert!(lock.try_lock(), "should acquire after guard is dropped"); + lock.unlock(); + } + + #[tokio::test] + async fn test_acquire_owned_lock_or_cancel_respects_pre_cancelled_token() { + let manager = InMemoryLockManager::new(16); + let id = manager.allocate().expect("allocate"); + let lock = manager.retrieve(id).expect("retrieve"); + let token = CancellationToken::new(); + token.cancel(); + + match acquire_owned_lock_or_cancel(lock, &token, "shutdown while waiting").await { + Ok(_) => panic!("pre-cancelled token should abort lock acquisition"), + Err(BoxliteError::Stopped(msg)) => assert_eq!(msg, "shutdown while waiting"), + Err(err) => panic!("unexpected error: {err}"), + } + } } From 1652e054934b6ebaf8dddbe1c7df3ccb97b0902f Mon Sep 17 00:00:00 2001 From: Wenyu Huang Date: Mon, 11 May 2026 16:36:41 +0000 Subject: [PATCH 4/6] feat(restart): recover crashed boxes with restart policy Add runtime crash recovery for boxes whose shim process dies while the embedding process is still running. Health checks now report shim death to a crash coordinator instead of mutating box state directly. The coordinator deduplicates in-flight crash notifications, records crash metadata, evaluates restart policy, applies exponential backoff, and restarts eligible boxes through a fresh BoxImpl swapped into the stable BoxHandle. This also teaches the init pipeline to treat Restarting like Stopped so restart reuses the existing rootfs. Integration tests cover No, Always, OnFailure, and UnlessStopped policies. Startup-time recovery of persisted crashed boxes is intentionally not included in this commit. Signed-off-by: Wenyu Huang --- src/boxlite/src/litebox/box_impl.rs | 193 +- src/boxlite/src/litebox/handle.rs | 3 + src/boxlite/src/litebox/init/mod.rs | 4 +- src/boxlite/src/litebox/state.rs | 43 +- src/boxlite/src/litebox/watcher.rs | 22 +- src/boxlite/src/runtime/advanced_options.rs | 17 +- src/boxlite/src/runtime/layout.rs | 8 +- src/boxlite/src/runtime/options.rs | 4 + src/boxlite/src/runtime/rt_impl.rs | 1827 ++++++++++++++++++- src/boxlite/tests/restart_policy.rs | 459 +++++ 10 files changed, 2474 insertions(+), 106 deletions(-) create mode 100644 src/boxlite/tests/restart_policy.rs diff --git a/src/boxlite/src/litebox/box_impl.rs b/src/boxlite/src/litebox/box_impl.rs index 776ab5ce6..ce07c53c3 100644 --- a/src/boxlite/src/litebox/box_impl.rs +++ b/src/boxlite/src/litebox/box_impl.rs @@ -231,6 +231,19 @@ impl BoxImpl { BoxInfo::new(&self.config, &state) } + /// Abort the watcher task (including its optional health probe). + /// + /// Used by restart() to retire this implementation before installing a fresh one. + pub(crate) fn abort_health_check(&self) { + if let Some(task) = self.watcher.get() { + tracing::debug!( + box_id = %self.config.id, + "Aborting box watcher" + ); + task.abort(); + } + } + // ======================================================================== // OPERATIONS (require LiveState) // ======================================================================== @@ -606,6 +619,19 @@ impl BoxImpl { return Ok(()); } + let lock_id = { + let state = self.state.read(); + if state.status.is_configured() && state.lock_id.is_none() { + return Ok(()); + } + state.lock_id.ok_or_else(|| { + BoxliteError::Internal(format!( + "box {} is missing lock_id (status: {:?})", + self.config.id, state.status + )) + })? + }; + // Abort the box watcher (if armed) so it does not run past stop(). // `stop()` also cancels the shutdown token the watcher selects on, but the // abort stops it immediately even if it is mid-probe. `abort` takes `&self`, @@ -618,24 +644,34 @@ impl BoxImpl { task.abort(); } - // Clear health status (box is no longer running) { - let mut state = self.state.write(); - state.clear_health_status(); - } + let locker = self.runtime.lock_manager.retrieve(lock_id)?; + let _lock_guard = acquire_owned_lock(locker).await?; - // Cancel the token - signals all in-flight operations to abort - self.shutdown_token.cancel(); + if self.shutdown_token.is_cancelled() && !self.runtime.shutdown_token.is_cancelled() { + tracing::debug!( + box_id = %self.config.id, + "Ignoring stop on retired box implementation" + ); + return Ok(()); + } + + // Clear health status (box is no longer running) + { + let mut state = self.state.write(); + state.clear_health_status(); + } + + // Cancel the token - signals all in-flight operations to abort + self.shutdown_token.cancel(); + } // Only attempt graceful shutdown for boxes that should have a live // shim. Calling live_state() on Configured/Failed would route - // through the restart pipeline and spawn a new VM — exactly what - // stop() must NOT do. + // through the restart pipeline and spawn a new VM. let should_attach = self.state.read().status == BoxStatus::Running; if should_attach && let Ok(live) = self.live_state().await { - // Recovered boxes lazy-attach here via vmm_attach (now - // ProcessIdentity-gated). Live boxes hit the cached LiveState. - // Either way the teardown is identical: + // Recovered boxes lazy-attach here via vmm_attach (ProcessIdentity-gated). let guest_shutdown = async { if let Ok(mut guest) = live.guest_session.guest().await { let _ = guest.shutdown().await; @@ -653,66 +689,66 @@ impl BoxImpl { handler.stop()?; } } - // If live_state() failed (vmm_attach said Absent — shim is gone), - // or status wasn't Running, fall through to cleanup. - - // Clean up PID file (single source of truth) - let pid_path = self.layout.pid_file_path(); - match std::fs::remove_file(&pid_path) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => tracing::warn!( - box_id = %self.config.id, - path = %pid_path.display(), - error = %e, - "Failed to remove PID file" - ), - } - - // Check if box was persisted - let was_persisted = self.state.read().lock_id.is_some(); + // If live_state() failed (vmm_attach said Absent), or status wasn't + // Running, fall through to cleanup without spawning a VM. - // Update state { - let mut state = self.state.write(); + let locker = self.runtime.lock_manager.retrieve(lock_id)?; + let _lock_guard = acquire_owned_lock(locker).await?; - // Only transition to Stopped if we were Running (or other active state). - // If we were Configured (never started), stay Configured so next start() - // triggers full initialization (creating disks). - if !state.status.is_configured() { - // Take the exit code the guest recorded on its way down, as - // docker does: `docker stop` leaves ExitCode 137, not 0. The - // guest writes the exit file when init dies — including when it - // dies because *we* killed it — before it checks whether the - // teardown was host-driven. The watcher cannot do this: stop() - // cancels its token, and it stands down precisely so it does not - // race this path. - crate::runtime::rt_impl::record_main_command_exit( - &mut state, - &self - .layout - .container_exit_file(self.config.container.id.as_str()), - ); + // Clean up PID file (single source of truth) + let pid_path = self.layout.pid_file_path(); + match std::fs::remove_file(&pid_path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => tracing::warn!( + box_id = %self.config.id, + path = %pid_path.display(), + error = %e, + "Failed to remove PID file" + ), } - if was_persisted { - // Box was persisted - sync to DB - // Note: If the box was already removed (e.g., by cleanup after init failure), - // this will return NotFound. We ignore that error since the box is already gone. - match self.runtime.box_manager.save_box(&self.config.id, &state) { - Ok(()) => {} - Err(BoxliteError::NotFound(_)) => { - tracing::debug!( - box_id = %self.config.id, - "Box already removed from DB during stop (likely cleanup after init failure)" - ); - return Ok(()); + // Check if box was persisted + let was_persisted = self.state.read().lock_id.is_some(); + + // Update state + { + let mut state = self.state.write(); + + // Only transition to Stopped if we were Running (or other active state). + // If we were Configured (never started), stay Configured so next start() + // triggers full initialization (creating disks). + if !state.status.is_configured() { + // Take the exit code the guest recorded on its way down, as + // docker does: `docker stop` leaves ExitCode 137, not 0. + crate::runtime::rt_impl::record_main_command_exit( + &mut state, + &self + .layout + .container_exit_file(self.config.container.id.as_str()), + ); + } + + if was_persisted { + // Box was persisted - sync to DB + // Note: If the box was already removed (e.g., by cleanup after init failure), + // this will return NotFound. We ignore that error since the box is already gone. + match self.runtime.box_manager.save_box(&self.config.id, &state) { + Ok(()) => {} + Err(BoxliteError::NotFound(_)) => { + tracing::debug!( + box_id = %self.config.id, + "Box already removed from DB during stop (likely cleanup after init failure)" + ); + return Ok(()); + } + Err(e) => return Err(e), } - Err(e) => return Err(e), + } else { + // Box was never started - persist now so it survives restarts + self.runtime.box_manager.add_box(&self.config, &state)?; } - } else { - // Box was never started - persist now so it survives restarts - self.runtime.box_manager.add_box(&self.config, &state)?; } } @@ -1031,6 +1067,7 @@ impl BoxImpl { // cleanup the VM process and directory. let builder = BoxBuilder::new(Arc::clone(&self.runtime), self.config.clone(), state)?; let (live_state, mut cleanup_guard) = builder.build().await?; + let health_config = self.config.options.advanced.effective_health_check(); // The box is up. If we adopted one whose init was already running, that // init needs no `Container.Start`; recording it now keeps @@ -1057,7 +1094,7 @@ impl BoxImpl { // Fetched before the state lock — the await must not run under it — and // before publishing Running below. let health_guest = - if self.config.options.advanced.health_check.is_some() && !adopting_running { + if health_config.is_some() && !adopting_running { Some(live_state.guest_session.guest().await?) } else { None @@ -1079,9 +1116,10 @@ impl BoxImpl { // clears ExitCode on start too). The guest drops its matching // exit file in Container.Init. state.exit_code = None; + state.stop_info.exit_code = None; - // Initialize health status if health check is configured - if self.config.options.advanced.health_check.is_some() { + // Initialize health status if an explicit or restart-policy health check is active. + if health_config.is_some() { state.init_health_status(); } @@ -1096,10 +1134,7 @@ impl BoxImpl { let health = health_guest.map(|guest| { super::watcher::HealthProbe::new( guest, - self.config - .options - .advanced - .health_check + health_config .clone() .expect("guest is fetched only when a health check is configured"), state.health_status, @@ -1198,7 +1233,13 @@ impl BoxImpl { { let mut state = self.state.write(); state.force_status(BoxStatus::Paused); - let _ = self.runtime.box_manager.save_box(self.id(), &state); + if let Err(e) = self.runtime.box_manager.save_box(self.id(), &state) { + tracing::warn!( + box_id = %self.id(), + error = %e, + "Failed to persist paused state during quiesce" + ); + } } // Phase 3: Caller's operation @@ -1215,7 +1256,13 @@ impl BoxImpl { if unsafe { libc::kill(pid, 0) } == 0 { let mut state = self.state.write(); state.force_status(BoxStatus::Running); - let _ = self.runtime.box_manager.save_box(self.id(), &state); + if let Err(e) = self.runtime.box_manager.save_box(self.id(), &state) { + tracing::warn!( + box_id = %self.id(), + error = %e, + "Failed to persist running state after quiesce" + ); + } } // Phase 5: Thaw guest I/O (always, best-effort) diff --git a/src/boxlite/src/litebox/handle.rs b/src/boxlite/src/litebox/handle.rs index c5a155d2e..8c190c041 100644 --- a/src/boxlite/src/litebox/handle.rs +++ b/src/boxlite/src/litebox/handle.rs @@ -53,6 +53,9 @@ impl BoxHandle { self.current().info() } + pub(crate) fn swap_current(&self, inner: SharedBoxImpl) -> SharedBoxImpl { + std::mem::replace(&mut *self.current.write(), inner) + } } #[async_trait] diff --git a/src/boxlite/src/litebox/init/mod.rs b/src/boxlite/src/litebox/init/mod.rs index 0f6113dc2..b1ed5b182 100644 --- a/src/boxlite/src/litebox/init/mod.rs +++ b/src/boxlite/src/litebox/init/mod.rs @@ -80,7 +80,7 @@ fn get_execution_plan(status: BoxStatus) -> BoxliteResult // Stopped and Failed both run the restart pipeline. A Failed box // has its rootfs preserved (per BoxStatus::Failed doc) and is // retryable per BoxStatus::can_start. - BoxStatus::Stopped | BoxStatus::Failed => vec![ + BoxStatus::Stopped | BoxStatus::Failed | BoxStatus::Restarting => vec![ // Restart: Same flow but rootfs tasks reuse existing COW disks // (preserves user modifications from previous run) Stage::sequential(vec![Box::new(FilesystemTask)]), @@ -198,7 +198,7 @@ impl BoxBuilder { } = self; let status = state.status; - let reuse_rootfs = status == BoxStatus::Stopped; + let reuse_rootfs = matches!(status, BoxStatus::Stopped | BoxStatus::Restarting); let skip_guest_wait = status == BoxStatus::Running; let ctx = InitPipelineContext::new(config, runtime.clone(), reuse_rootfs, skip_guest_wait); diff --git a/src/boxlite/src/litebox/state.rs b/src/boxlite/src/litebox/state.rs index 5a8b5db07..06c9dec70 100644 --- a/src/boxlite/src/litebox/state.rs +++ b/src/boxlite/src/litebox/state.rs @@ -277,6 +277,13 @@ pub struct BoxState { /// Error message from the last failed auto-restart attempt (cleared on success). #[serde(default)] pub last_restart_error: Option, + /// Monotonic lifecycle intent version. + /// + /// User-visible lifecycle operations bump this value so delayed background + /// work, such as crash-restart attempts after backoff, can detect that its + /// original decision is stale before committing a new state transition. + #[serde(default)] + pub lifecycle_epoch: u64, } /// Why the box stopped. @@ -414,9 +421,21 @@ impl BoxState { exit_code: None, stop_info: StopInfo::default(), last_restart_error: None, + lifecycle_epoch: 0, } } + /// Return the current lifecycle intent version. + pub fn lifecycle_epoch(&self) -> u64 { + self.lifecycle_epoch + } + + /// Bump the lifecycle intent version and update the state timestamp. + pub fn bump_lifecycle_epoch(&mut self) { + self.lifecycle_epoch = self.lifecycle_epoch.saturating_add(1); + self.last_updated = Utc::now(); + } + /// Set lock ID and update timestamp. pub fn set_lock_id(&mut self, lock_id: LockId) { self.lock_id = Some(lock_id); @@ -467,7 +486,7 @@ impl BoxState { self.error_reason = None; self.stop_info.cause = StopCause::Normal; self.stop_info.exit_time = Some(Utc::now()); - self.last_updated = Utc::now(); + self.bump_lifecycle_epoch(); } /// Mark the box as Failed with the captured init/runtime error. @@ -1086,6 +1105,27 @@ mod tests { let state = BoxState::new(); assert_eq!(state.health_status.state, HealthState::None); assert_eq!(state.health_status.failures, 0); + assert_eq!(state.lifecycle_epoch(), 0); + } + + #[test] + fn test_lifecycle_epoch_bumps_on_stop() { + let mut state = BoxState::new(); + assert_eq!(state.lifecycle_epoch(), 0); + + state.mark_stop(); + + assert_eq!(state.lifecycle_epoch(), 1); + } + + #[test] + fn test_lifecycle_epoch_saturates() { + let mut state = BoxState::new(); + state.lifecycle_epoch = u64::MAX; + + state.bump_lifecycle_epoch(); + + assert_eq!(state.lifecycle_epoch(), u64::MAX); } #[test] @@ -1104,6 +1144,7 @@ mod tests { assert_eq!(state.health_status.state, HealthState::None); assert_eq!(state.health_status.failures, 0); assert!(state.health_status.last_check.is_none()); + assert_eq!(state.lifecycle_epoch(), 0); } // ==================================================================== diff --git a/src/boxlite/src/litebox/watcher.rs b/src/boxlite/src/litebox/watcher.rs index 5ab0ab87b..2a8e8f8f4 100644 --- a/src/boxlite/src/litebox/watcher.rs +++ b/src/boxlite/src/litebox/watcher.rs @@ -102,6 +102,7 @@ pub(crate) struct BoxWatcher { box_name: Option, exit_file: std::path::PathBuf, removes_on_exit: bool, + has_restart_policy: bool, /// `None` ⇒ exit-only watcher. `Some` ⇒ also probe the guest's health. health: Option, } @@ -121,6 +122,7 @@ impl BoxWatcher { .layout .container_exit_file(bx.config.container.id.as_str()), removes_on_exit: bx.config.options.removes_on_stop(), + has_restart_policy: bx.config.options.advanced.restart_policy.is_some(), health, } } @@ -184,6 +186,24 @@ impl BoxWatcher { return; }; + // Restart-policy boxes are finalized by the central crash coordinator, + // which owns the Crashed/Restarting transitions and handle replacement. + // Leave the persisted state Running until it acquires the lifecycle lock. + if self.has_restart_policy { + let crash_tx = runtime.crash_sender(); + let box_id = self.box_id.clone(); + tokio::spawn(async move { + if let Err(error) = crash_tx.send(box_id.clone()).await { + tracing::error!( + box_id = %box_id, + error = %error, + "Crash handler channel closed, notification dropped" + ); + } + }); + return; + } + let stopped = { let mut state = self.state.write(); @@ -234,7 +254,7 @@ impl BoxWatcher { // Without it a long-lived runtime keeps handing out the spent handle from // its cache, and a remove-on-stop box — the default — that ran to // completion is never cleaned up, because nobody called stop() to do it. - runtime.invalidate_box_impl(&self.box_id, self.box_name.as_deref()); + runtime.invalidate_box_handle(&self.box_id, self.box_name.as_deref()); if self.removes_on_exit && let Err(e) = runtime.remove_box(&self.box_id, false) { diff --git a/src/boxlite/src/runtime/advanced_options.rs b/src/boxlite/src/runtime/advanced_options.rs index 24e06f76a..ca4d0c383 100644 --- a/src/boxlite/src/runtime/advanced_options.rs +++ b/src/boxlite/src/runtime/advanced_options.rs @@ -604,6 +604,9 @@ pub struct AdvancedBoxOptions { /// /// When set, the health check task will evaluate the policy when the /// shim process dies and automatically restart the box if the policy allows it. + /// For detached boxes, this only applies while a runtime is attached and + /// monitoring the box; it is not daemon-style recovery after the embedding + /// process exits. /// /// If `health_check` is not configured but `restart_policy` is, a default /// health check is auto-enabled (interval=5s, timeout=10s, retries=3, start_period=60s). @@ -662,9 +665,9 @@ impl RestartPolicy { RestartPolicy::No => false, RestartPolicy::Always => true, RestartPolicy::OnFailure { max_retries } => { - // Only restart on non-zero exit code (explicit failure) - // None (no exit file) means normal/clean exit - no restart - let is_failure = exit_code.is_some_and(|code| code != 0); + // Restart on explicit non-zero exits and unknown exits. Unknown + // usually means the shim died before writing structured exit info. + let is_failure = exit_code.is_none_or(|code| code != 0); is_failure && restart_count < *max_retries } RestartPolicy::UnlessStopped => true, @@ -767,11 +770,11 @@ mod tests { } #[test] - fn test_restart_policy_on_failure_no_restart_on_no_exit_file() { + fn test_restart_policy_on_failure_restarts_on_unknown_exit() { let policy = RestartPolicy::OnFailure { max_retries: 3 }; - // No exit file (None) means clean/normal exit - no restart - assert!(!policy.should_restart(None, 0)); - assert!(!policy.should_restart(None, 1)); + // Unknown exit means the shim died before writing structured exit info. + assert!(policy.should_restart(None, 0)); + assert!(policy.should_restart(None, 1)); } #[test] diff --git a/src/boxlite/src/runtime/layout.rs b/src/boxlite/src/runtime/layout.rs index 6c7b9ee06..6fe43b780 100644 --- a/src/boxlite/src/runtime/layout.rs +++ b/src/boxlite/src/runtime/layout.rs @@ -42,6 +42,9 @@ pub mod dirs { /// Subdirectory for per-entity locks pub const LOCKS_DIR: &str = "locks"; + + /// Exit info file written by shim on crash (contains exit code, signal, etc.) + pub const EXIT_FILE: &str = "exit"; } /// Configuration for filesystem layout behavior. @@ -537,10 +540,9 @@ impl BoxFilesystemLayout { /// Exit file path: ~/.boxlite/boxes/{box_id}/exit /// /// Written by the shim process on exit (normal or panic). - /// Format: First line is exit code, subsequent lines contain error details. - /// Follows Podman's conmon pattern for capturing exit information. + /// Format: JSON with exit_code, type, and optional message/signal. pub fn exit_file_path(&self) -> PathBuf { - self.box_dir.join("exit") + self.box_dir.join(dirs::EXIT_FILE) } /// Archived exit file from the previous lifecycle: `~/.boxlite/boxes/{box_id}/exit.previous`. diff --git a/src/boxlite/src/runtime/options.rs b/src/boxlite/src/runtime/options.rs index 0e8c6b857..30f79d31d 100644 --- a/src/boxlite/src/runtime/options.rs +++ b/src/boxlite/src/runtime/options.rs @@ -368,6 +368,10 @@ pub struct BoxOptions { /// any process can reattach via `runtime.get(box_id)`. The only ways /// to stop a detached box are `runtime.get(box_id).stop()` and /// `boxlite stop `. Similar to Docker's `-d` (detach) flag. + /// + /// Detached boxes are skipped by runtime shutdown and may survive parent + /// process exit. Health checks and restart policy still run inside the + /// embedding process and resume after a runtime reattaches to the box. #[serde(default = "default_detach")] pub detach: bool, diff --git a/src/boxlite/src/runtime/rt_impl.rs b/src/boxlite/src/runtime/rt_impl.rs index 39dc6ffff..93e4951f0 100644 --- a/src/boxlite/src/runtime/rt_impl.rs +++ b/src/boxlite/src/runtime/rt_impl.rs @@ -1,25 +1,33 @@ use crate::db::{BoxStore, Database}; use crate::images::{ImageDiskManager, ImageManager}; use crate::litebox::config::BoxConfig; -use crate::litebox::{BoxHandle, BoxManager, LiteBox, SharedBoxHandle, SharedBoxImpl}; -use crate::lock::{FileLockManager, LockManager}; +use crate::litebox::{BoxHandle, BoxManager, LiteBox, SharedBoxHandle, SharedBoxImpl, StopCause}; +use crate::lock::{ + FileLockManager, LockId, LockManager, acquire_owned_lock, acquire_owned_lock_or_cancel, +}; use crate::metrics::{RuntimeMetrics, RuntimeMetricsStorage}; use crate::rootfs::guest::{GuestRootfs, GuestRootfsManager}; +use crate::runtime::advanced_options::{RestartPolicy, calculate_backoff}; use crate::runtime::id::{BoxID, BoxIDMint}; +use crate::runtime::layout::dirs::EXIT_FILE; use crate::runtime::layout::{BoxFilesystemLayout, FilesystemLayout, FsLayoutConfig}; use crate::runtime::lock::RuntimeLock; use crate::runtime::options::{BoxArchive, BoxOptions, BoxliteOptions}; use crate::runtime::signal_handler::timeout_to_duration; use crate::runtime::types::{BoxInfo, BoxState, BoxStatus, ContainerID}; -use crate::vmm::VmmKind; use crate::vmm::controller::{ShimHandler, VmmHandler}; +use crate::vmm::{ExitInfo, VmmKind}; use boxlite_shared::{BoxliteError, BoxliteResult}; use chrono::Utc; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::{Arc, RwLock, Weak}; -use tokio::sync::OnceCell; +use tokio::sync::{Mutex as AsyncMutex, OnceCell, mpsc}; +use tokio::task::{Id as TaskId, JoinHandle, JoinSet}; +use tokio::time::Instant; use tokio_util::sync::CancellationToken; +const CRASH_COORDINATOR_GRACE_PERIOD: std::time::Duration = std::time::Duration::from_secs(5); + fn litebox_from_handle(handle: SharedBoxHandle) -> LiteBox { // Every handle this runtime hands out starts following its box's main // command, if the box is running and nobody is following it yet. @@ -106,6 +114,238 @@ fn copy_dir_recursive(src: &std::path::Path, dst: &std::path::Path) -> std::io:: Ok(()) } +fn read_box_exit_code(config: &BoxConfig) -> Option { + let container_exit_file = + boxlite_shared::layout::SharedGuestLayout::new(config.box_home.join("shared")) + .container(config.container.id.as_str()) + .exit_file(); + + boxlite_shared::layout::ExitRecord::read(&container_exit_file) + .map(|record| record.exit_code) + .or_else(|| { + let shim_exit_file = config.box_home.join(EXIT_FILE); + ExitInfo::from_file(&shim_exit_file).map(|info| info.exit_code()) + }) +} + +fn crash_restart_matches_plan(state: &BoxState, expected_epoch: u64) -> bool { + state.lifecycle_epoch() == expected_epoch + && matches!(state.status, BoxStatus::Crashed | BoxStatus::Restarting) +} + +fn stop_cause_when_restart_denied( + restart_policy: Option<&RestartPolicy>, + exit_code: Option, + max_retries_exhausted: bool, +) -> StopCause { + match restart_policy { + None | Some(RestartPolicy::No) => StopCause::CrashedNoPolicy, + Some(RestartPolicy::OnFailure { .. }) if exit_code == Some(0) => StopCause::Normal, + Some(RestartPolicy::OnFailure { .. }) if max_retries_exhausted => { + StopCause::MaxRetriesExceeded + } + Some(RestartPolicy::OnFailure { .. }) => { + // This branch should be unreachable: + // - exit_code != 0 (caught by guard above) + // - !max_retries_exhausted (caught by guard above) + // For OnFailure, non-zero exit + under max_retries should restart, not deny. + tracing::error!( + exit_code = ?exit_code, + max_retries_exhausted, + "BUG: Unreachable branch reached in stop_cause_when_restart_denied" + ); + debug_assert!(false, "Unreachable branch reached"); + StopCause::Unknown + } + Some(RestartPolicy::Always) => { + // Always policy should never deny restart - this is a bug + tracing::error!("BUG: Always policy should not reach stop_cause_when_restart_denied"); + debug_assert!(false, "Always policy should never deny restart"); + StopCause::Unknown + } + Some(RestartPolicy::UnlessStopped) => { + // UnlessStopped only denies restart when user explicitly stopped (cause == Normal) + // At this point, exit_code == 0 indicates a clean exit from user stop + StopCause::Normal + } + } +} + +struct CrashRestartPlan { + restart_policy: Option, + expected_epoch: u64, + current_restart_count: u32, + new_restart_count: u32, + exit_code: Option, +} + +enum RestartOutcome { + Stale, + Started { + box_impl: SharedBoxImpl, + lock_id: LockId, + }, +} + +struct CrashCoordinator { + runtime: Weak, + shutdown_token: CancellationToken, + force_cancel_token: CancellationToken, + crash_rx: mpsc::Receiver, + crash_rx_closed: bool, + pending_crashes: HashSet, + running_crashes: JoinSet, + task_boxes: HashMap, +} + +impl CrashCoordinator { + fn new(runtime: SharedRuntimeImpl, crash_rx: mpsc::Receiver) -> Self { + let shutdown_token = runtime.shutdown_token.clone(); + let force_cancel_token = runtime.crash_force_cancel_token.clone(); + Self { + runtime: Arc::downgrade(&runtime), + shutdown_token, + force_cancel_token, + crash_rx, + crash_rx_closed: false, + pending_crashes: HashSet::new(), + running_crashes: JoinSet::new(), + task_boxes: HashMap::new(), + } + } + + fn spawn(runtime: SharedRuntimeImpl, crash_rx: mpsc::Receiver) -> JoinHandle<()> { + tokio::spawn(async move { + Self::new(runtime, crash_rx).run().await; + }) + } + + async fn run(mut self) { + tracing::info!("Crash coordinator task started"); + let mut force_cancelled = false; + + loop { + if !force_cancelled && self.force_cancel_token.is_cancelled() { + self.abort_running_crashes(); + force_cancelled = true; + } + + let accepting_crashes = !self.shutdown_token.is_cancelled() && !self.crash_rx_closed; + if (!accepting_crashes || self.crash_rx_closed) && self.running_crashes.is_empty() { + break; + } + + tokio::select! { + crash = self.crash_rx.recv(), if accepting_crashes => { + match crash { + Some(box_id) => { + self.schedule_crash(box_id); + } + None => { + tracing::debug!("Crash notification channel closed"); + self.crash_rx_closed = true; + } + } + } + result = self.running_crashes.join_next_with_id(), if !self.running_crashes.is_empty() => { + if let Some(result) = result { + self.finish_crash_task(result); + } + } + _ = self.shutdown_token.cancelled(), if !self.shutdown_token.is_cancelled() => { + tracing::debug!("Crash coordinator received shutdown signal"); + } + _ = self.force_cancel_token.cancelled(), if !force_cancelled => { + self.abort_running_crashes(); + force_cancelled = true; + } + } + } + + tracing::info!("Crash coordinator task stopped"); + } + + fn abort_running_crashes(&mut self) { + tracing::warn!( + task_count = self.running_crashes.len(), + "Crash coordinator grace period expired; aborting crash tasks" + ); + self.running_crashes.abort_all(); + } + + fn schedule_crash(&mut self, box_id: BoxID) -> bool { + if !Self::mark_pending_crash(&mut self.pending_crashes, &box_id) { + tracing::debug!( + box_id = %box_id, + "Crash already being handled, skipping duplicate notification" + ); + return false; + } + + tracing::info!(box_id = %box_id, "Received crash notification"); + + if self.runtime.upgrade().is_none() { + tracing::debug!( + box_id = %box_id, + "Runtime already dropped, skipping crash notification" + ); + self.pending_crashes.remove(&box_id); + self.crash_rx_closed = true; + return false; + } + + let runtime_weak = Weak::clone(&self.runtime); + let shutdown_token = self.shutdown_token.clone(); + let task_box_id = box_id.clone(); + let abort_handle = self.running_crashes.spawn(async move { + RuntimeImpl::handle_box_crash(runtime_weak, shutdown_token, task_box_id.clone()).await; + task_box_id + }); + self.task_boxes.insert(abort_handle.id(), box_id); + true + } + + fn finish_crash_task(&mut self, result: Result<(TaskId, BoxID), tokio::task::JoinError>) { + match result { + Ok((task_id, returned_box_id)) => { + let box_id = self.task_boxes.remove(&task_id).unwrap_or_else(|| { + tracing::error!( + task_id = %task_id, + box_id = %returned_box_id, + "Completed crash task was not tracked" + ); + returned_box_id + }); + self.pending_crashes.remove(&box_id); + } + Err(error) => { + let task_id = error.id(); + if let Some(box_id) = self.task_boxes.remove(&task_id) { + self.pending_crashes.remove(&box_id); + tracing::error!( + task_id = %task_id, + box_id = %box_id, + is_cancelled = error.is_cancelled(), + is_panic = error.is_panic(), + error = %error, + "Crash handler task failed" + ); + } else { + tracing::error!( + task_id = %task_id, + error = %error, + "Untracked crash handler task failed" + ); + } + } + } + } + + fn mark_pending_crash(pending_crashes: &mut HashSet, box_id: &BoxID) -> bool { + pending_crashes.insert(box_id.clone()) + } +} + /// Internal runtime state protected by single lock. /// /// **Shared via Arc**: This is the actual shared state that can be cloned cheaply. @@ -176,6 +416,29 @@ pub struct RuntimeImpl { /// Use `.is_cancelled()` for sync checks, `.cancelled()` for async select!. /// Child tokens are passed to each box via `.child_token()`. pub(crate) shutdown_token: CancellationToken, + + // ======================================================================== + // CRASH HANDLER + // ======================================================================== + /// Channel sender for box crash notifications. + /// Used by health check tasks to notify runtime of crashes. + crash_tx: mpsc::Sender, + + /// Handle to the crash coordinator task (for graceful shutdown). + crash_handler_handle: AsyncMutex>>, + + /// Requests forced cancellation of supervised crash tasks after the + /// cooperative shutdown grace period expires. + crash_force_cancel_token: CancellationToken, + + /// Pending crash notification receiver, consumed once when the coordinator starts. + pending_crash_rx: std::sync::Mutex>>, + + /// One-time gate for lazy background task initialization. + /// + /// `new()` is synchronous and may run outside a Tokio runtime, so background + /// tasks are spawned by the first async runtime method instead. + services_started: tokio::sync::OnceCell<()>, } /// Synchronized state protected by RwLock. @@ -188,6 +451,13 @@ pub struct SynchronizedState { active_handles_by_id: HashMap>, /// Cache of active BoxHandle instances by name (only for named boxes). active_handles_by_name: HashMap>, + /// Strong runtime ownership for boxes kept alive by restart policy. + /// + /// Each entry is keyed by Box ID, so repeated restarts of the same box + /// replace the existing handle instead of growing this map. Entries are + /// removed when the box is explicitly stopped or removed via + /// `invalidate_box_handle`. + restart_owned_handles_by_id: HashMap, } impl RuntimeImpl { @@ -294,10 +564,14 @@ impl RuntimeImpl { ImageDiskManager::new(layout.image_layout().disk_images_dir(), layout.temp_dir()); let guest_rootfs_mgr = GuestRootfsManager::new(base_disk_mgr.clone(), layout.temp_dir()); + // Create crash notification channel + let (crash_tx, crash_rx) = mpsc::channel::(100); + let inner = Arc::new(Self { sync_state: RwLock::new(SynchronizedState { active_handles_by_id: HashMap::new(), active_handles_by_name: HashMap::new(), + restart_owned_handles_by_id: HashMap::new(), }), box_manager: BoxManager::new(box_store), image_manager, @@ -312,6 +586,11 @@ impl RuntimeImpl { network_factory: crate::net::default_factory(), _runtime_lock: runtime_lock, shutdown_token: CancellationToken::new(), + crash_tx, + crash_handler_handle: AsyncMutex::new(None), + crash_force_cancel_token: CancellationToken::new(), + pending_crash_rx: std::sync::Mutex::new(Some(crash_rx)), + services_started: tokio::sync::OnceCell::new(), }); tracing::debug!("initialized runtime"); @@ -322,6 +601,555 @@ impl RuntimeImpl { Ok(inner) } + // ======================================================================== + // CRASH HANDLER + // ======================================================================== + + /// Spawn the central crash coordinator task. + fn spawn_crash_handler( + runtime: SharedRuntimeImpl, + crash_rx: mpsc::Receiver, + ) -> JoinHandle<()> { + CrashCoordinator::spawn(runtime, crash_rx) + } + + /// Handle a single box crash (driven by the crash coordinator). + async fn handle_box_crash( + runtime_weak: Weak, + shutdown_token: CancellationToken, + box_id: BoxID, + ) { + let plan = { + let Some(runtime_impl) = runtime_weak.upgrade() else { + tracing::debug!(box_id = %box_id, "Runtime dropped before crash handling"); + return; + }; + + let Some(plan) = Self::record_crash_and_plan_restart(&runtime_impl, &box_id).await + else { + return; + }; + plan + }; + + tracing::info!( + box_id = %box_id, + restart_count = plan.new_restart_count, + exit_code = ?plan.exit_code, + "Box crashed, scheduling restart with backoff" + ); + + Self::run_restart_loop(runtime_weak, shutdown_token, box_id, plan).await; + } + + async fn record_crash_and_plan_restart( + runtime: &RuntimeImpl, + box_id: &BoxID, + ) -> Option { + let state = match runtime.box_manager.box_by_id(box_id) { + Ok(Some((_, s))) => s, + Ok(None) => { + tracing::debug!(box_id = %box_id, "Box not found, ignoring crash"); + return None; + } + Err(e) => { + tracing::error!(box_id = %box_id, error = %e, "Failed to read box state"); + return None; + } + }; + + let lock_id = match state.lock_id { + Some(id) => id, + None => { + tracing::warn!(box_id = %box_id, "Box has no lock_id"); + return None; + } + }; + + let locker = match runtime.lock_manager.retrieve(lock_id) { + Ok(l) => l, + Err(e) => { + tracing::error!(box_id = %box_id, error = %e, "Failed to retrieve lock"); + return None; + } + }; + + let _lock_guard = match acquire_owned_lock_or_cancel( + locker, + &runtime.shutdown_token, + format!("Runtime is shutting down while waiting to record crash for box {box_id}"), + ) + .await + { + Ok(guard) => guard, + Err(BoxliteError::Stopped(_)) => { + tracing::debug!(box_id = %box_id, "Shutdown while waiting for lock"); + return None; + } + Err(e) => { + tracing::error!(box_id = %box_id, error = %e, "Failed to acquire lock"); + return None; + } + }; + + let (config, mut state) = match runtime.box_manager.box_by_id(box_id) { + Ok(Some((c, s))) => (c, s), + Ok(None) => { + tracing::debug!(box_id = %box_id, "Box removed before crash handling"); + return None; + } + Err(e) => { + tracing::error!(box_id = %box_id, error = %e, "Failed to re-read box state"); + return None; + } + }; + + match state.status { + BoxStatus::Running | BoxStatus::Crashed => {} + BoxStatus::Restarting => { + tracing::debug!(box_id = %box_id, "Box already restarting, ignoring"); + return None; + } + _ => { + tracing::debug!( + box_id = %box_id, + status = ?state.status, + "Box not running, ignoring crash" + ); + return None; + } + } + + let exit_code = read_box_exit_code(&config); + let restart_policy = config.options.advanced.restart_policy.clone(); + let current_restart_count = state.stop_info.restart_count; + let expected_epoch = state.lifecycle_epoch(); + let new_restart_count = state.stop_info.restart_count.saturating_add(1); + let max_retries_exhausted = matches!( + restart_policy.as_ref(), + Some(RestartPolicy::OnFailure { max_retries }) if current_restart_count >= *max_retries + ); + + state.force_status(BoxStatus::Crashed); + state.set_pid(None); + state.exit_code = exit_code; + state.health_status.state = crate::litebox::HealthState::Unhealthy; + state.stop_info = crate::litebox::StopInfo { + cause: StopCause::CrashedNoPolicy, + exit_code, + exit_time: Some(Utc::now()), + restart_count: new_restart_count, + restarted_at: None, + }; + + if let Err(e) = runtime.box_manager.save_box(box_id, &state) { + tracing::error!(box_id = %box_id, error = %e, "Failed to save crashed state"); + } + + let should_restart = restart_policy + .as_ref() + .map(|policy| policy.should_restart(exit_code, current_restart_count)) + .unwrap_or(false); + + if !should_restart { + Self::mark_restart_denied( + runtime, + box_id, + state, + config.name.as_deref(), + restart_policy.as_ref(), + exit_code, + max_retries_exhausted, + ); + return None; + } + + Some(CrashRestartPlan { + restart_policy, + expected_epoch, + current_restart_count, + new_restart_count, + exit_code, + }) + } + + async fn commit_crash_restart_state( + runtime: &RuntimeImpl, + box_id: &BoxID, + expected_epoch: u64, + update_state: impl FnOnce(&mut BoxState), + ) -> BoxliteResult { + let Some((_, state)) = runtime.box_manager.box_by_id(box_id)? else { + tracing::debug!( + box_id = %box_id, + "Crash restart state commit skipped because box was removed" + ); + return Ok(false); + }; + + let lock_id = state + .lock_id + .ok_or_else(|| BoxliteError::Internal(format!("box {box_id} has no lock_id")))?; + let locker = runtime.lock_manager.retrieve(lock_id)?; + let _lock_guard = acquire_owned_lock(locker).await?; + + let Some((_, mut state)) = runtime.box_manager.box_by_id(box_id)? else { + tracing::debug!( + box_id = %box_id, + "Crash restart state commit skipped because box was removed" + ); + return Ok(false); + }; + + if !crash_restart_matches_plan(&state, expected_epoch) { + if state.lifecycle_epoch() != expected_epoch { + tracing::debug!( + box_id = %box_id, + expected_epoch, + actual_epoch = state.lifecycle_epoch(), + "Crash restart state commit skipped because lifecycle epoch changed" + ); + } else { + tracing::debug!( + box_id = %box_id, + status = ?state.status, + "Crash restart state commit skipped because box state changed" + ); + } + return Ok(false); + } + + update_state(&mut state); + runtime.box_manager.save_box(box_id, &state)?; + Ok(true) + } + + async fn commit_restart_success( + runtime: &RuntimeImpl, + box_id: &BoxID, + expected_epoch: u64, + box_impl: &SharedBoxImpl, + lock_id: LockId, + ) -> BoxliteResult { + let locker = runtime.lock_manager.retrieve(lock_id)?; + let _lock_guard = acquire_owned_lock_or_cancel( + locker, + &runtime.shutdown_token, + "Runtime is shutting down while finalizing restarted box", + ) + .await?; + + // The swapped-in BoxImpl is the live source here: update its in-memory + // state, then persist that snapshot. A DB-only update would leave + // existing handles stale. + let final_state = { + let mut state = box_impl.state.write(); + if state.lifecycle_epoch() != expected_epoch || state.status != BoxStatus::Running { + tracing::debug!( + box_id = %box_id, + expected_epoch, + actual_epoch = state.lifecycle_epoch(), + status = ?state.status, + "Restart success state commit skipped because box state changed" + ); + return Ok(false); + } + state.stop_info = crate::litebox::StopInfo::default(); + state.stop_info.restarted_at = Some(chrono::Utc::now()); + state.clone() + }; + + runtime.box_manager.save_box(box_id, &final_state)?; + Ok(true) + } + + async fn run_restart_loop( + runtime_weak: Weak, + shutdown_token: CancellationToken, + box_id: BoxID, + plan: CrashRestartPlan, + ) { + let mut attempt = plan.current_restart_count; + loop { + let backoff = calculate_backoff(attempt); + attempt += 1; + + tracing::info!( + box_id = %box_id, + attempt, + backoff_ms = backoff.as_millis() as u64, + "Scheduling restart attempt" + ); + + tokio::select! { + _ = tokio::time::sleep(backoff) => { + let Some(this) = runtime_weak.upgrade() else { + tracing::debug!(box_id = %box_id, "Runtime dropped during restart backoff"); + return; + }; + + // Re-read state after backoff: a manual stop/remove/restart may have happened. + let current_status = match this.box_manager.box_by_id(&box_id) { + Ok(Some((_, s))) => s.status, + Ok(None) => { + tracing::debug!(box_id = %box_id, "Box removed during backoff"); + return; + } + Err(e) => { + tracing::error!(box_id = %box_id, error = %e, "Failed to read state"); + return; + } + }; + + match current_status { + BoxStatus::Running => { + tracing::debug!( + box_id = %box_id, + "Box already running (manual restart), skipping auto-restart" + ); + return; + } + BoxStatus::Crashed | BoxStatus::Restarting => {} + status => { + tracing::debug!( + box_id = %box_id, + status = ?status, + "Box state changed during backoff, skipping auto-restart" + ); + return; + } + } + + tracing::info!(box_id = %box_id, attempt, "Executing restart"); + + match this.restart(&box_id, plan.expected_epoch).await { + Ok(RestartOutcome::Started { box_impl, lock_id }) => { + match Self::commit_restart_success( + &this, + &box_id, + plan.expected_epoch, + &box_impl, + lock_id, + ) + .await + { + Ok(true) => tracing::info!( + box_id = %box_id, + attempt, + "Box restarted successfully" + ), + Ok(false) => { + tracing::debug!( + box_id = %box_id, + attempt, + "Restart success commit became stale" + ); + return; + } + Err(e) => tracing::error!( + box_id = %box_id, + attempt, + error = %e, + "Box restarted but success state commit failed" + ), + } + break; + } + Ok(RestartOutcome::Stale) => { + tracing::debug!( + box_id = %box_id, + attempt, + "Crash restart attempt became stale" + ); + return; + } + Err(e) => { + tracing::error!( + box_id = %box_id, + attempt, + error = %e, + "Restart attempt failed" + ); + + let should_retry = plan.restart_policy + .as_ref() + .map(|p| p.should_restart(plan.exit_code, attempt)) + .unwrap_or(false); + + if should_retry { + match Self::mark_restart_failed( + &this, + &box_id, + plan.expected_epoch, + attempt, + ) + .await + { + Ok(true) => {} + Ok(false) => return, + Err(e) => tracing::error!( + box_id = %box_id, + attempt, + error = %e, + "Failed to save restart failure state" + ), + } + } else { + tracing::info!(box_id = %box_id, attempt, "Max retries exceeded"); + + if let Err(e) = Self::commit_crash_restart_state( + &this, + &box_id, + plan.expected_epoch, + |state| { + state.force_status(BoxStatus::Stopped); + state.stop_info.restart_count = attempt; + state.stop_info.cause = StopCause::MaxRetriesExceeded; + }, + ).await { + tracing::error!( + box_id = %box_id, + attempt, + error = %e, + "Failed to save max-retries-exceeded state" + ); + } + break; // Give up + } + // Continue loop for retry (_lock_guard dropped here) + } + } + // _lock_guard dropped here + } + _ = shutdown_token.cancelled() => { + tracing::debug!(box_id = %box_id, "Restart cancelled (shutdown)"); + if let Some(this) = runtime_weak.upgrade() + && let Err(e) = Self::commit_crash_restart_state( + &this, + &box_id, + plan.expected_epoch, + |state| { + state.force_status(BoxStatus::Stopped); + state.stop_info.restart_count = attempt; + state.stop_info.cause = StopCause::Normal; + }, + ).await { + tracing::error!( + box_id = %box_id, + attempt, + error = %e, + "Failed to save cancelled restart state" + ); + } + break; + } + } + } + } + + fn mark_restart_denied( + runtime: &RuntimeImpl, + box_id: &BoxID, + mut state: BoxState, + box_name: Option<&str>, + restart_policy: Option<&RestartPolicy>, + exit_code: Option, + max_retries_exhausted: bool, + ) { + tracing::info!( + box_id = %box_id, + policy = ?restart_policy, + "No restart policy or max retries exceeded, marking as stopped" + ); + + state.force_status(BoxStatus::Stopped); + state.stop_info.cause = + stop_cause_when_restart_denied(restart_policy, exit_code, max_retries_exhausted); + + if let Err(e) = runtime.box_manager.save_box(box_id, &state) { + tracing::error!(box_id = %box_id, error = %e, "Failed to save stopped state"); + } + + runtime.retire_cached_box_after_crash(box_id, box_name, &state); + } + + fn retire_cached_box_after_crash( + &self, + box_id: &BoxID, + box_name: Option<&str>, + stopped_state: &BoxState, + ) { + let handle = { + let sync = self.sync_state.read().unwrap(); + sync.active_handles_by_id + .get(box_id) + .and_then(|weak| weak.upgrade()) + }; + + if let Some(handle) = handle { + let box_impl = handle.current(); + box_impl.abort_health_check(); + box_impl.shutdown_token.cancel(); + *box_impl.state.write() = stopped_state.clone(); + } + + self.invalidate_box_handle(box_id, box_name); + } + + async fn mark_restart_failed( + runtime: &RuntimeImpl, + box_id: &BoxID, + expected_epoch: u64, + attempt: u32, + ) -> BoxliteResult { + Self::commit_crash_restart_state(runtime, box_id, expected_epoch, |state| { + state.force_status(BoxStatus::Crashed); + state.stop_info.restart_count = attempt; + state.stop_info.cause = StopCause::RestartFailed; + }) + .await + } + + /// Get crash sender for health check tasks. + pub(crate) fn crash_sender(&self) -> mpsc::Sender { + self.crash_tx.clone() + } + + // ======================================================================== + // LAZY SERVICES INITIALIZATION + // ======================================================================== + + /// Ensure the crash coordinator task is started. + /// + /// This is called lazily on the first async method, guaranteeing a Tokio + /// runtime exists. Uses `OnceCell` to record exactly one successful start. + async fn ensure_services_started(self: &Arc) { + let _ = self + .services_started + .get_or_try_init(|| { + let rt = Arc::clone(self); + async move { + let mut handle_slot = rt.crash_handler_handle.lock().await; + if handle_slot.is_some() { + return Ok(()); + } + if rt.shutdown_token.is_cancelled() { + return Err(()); + } + let crash_rx = rt + .pending_crash_rx + .lock() + .unwrap() + .take() + .expect("pending_crash_rx consumed twice"); + let handle = Self::spawn_crash_handler(Arc::clone(&rt), crash_rx); + *handle_slot = Some(handle); + Ok(()) + } + }) + .await; + } + // ======================================================================== // PUBLIC API - BOX OPERATIONS // ======================================================================== @@ -337,6 +1165,7 @@ impl RuntimeImpl { options: BoxOptions, name: Option, ) -> BoxliteResult { + self.ensure_services_started().await; let (litebox, _created) = self.create_inner(options, name, false).await?; Ok(litebox) } @@ -351,6 +1180,7 @@ impl RuntimeImpl { options: BoxOptions, name: Option, ) -> BoxliteResult<(LiteBox, bool)> { + self.ensure_services_started().await; self.create_inner(options, name, true).await } @@ -363,6 +1193,7 @@ impl RuntimeImpl { archive: BoxArchive, name: Option, ) -> BoxliteResult { + self.ensure_services_started().await; super::import::import_box(self, archive, name).await } @@ -468,6 +1299,7 @@ impl RuntimeImpl { /// If another handle to the same box exists, they share the same BoxImpl /// (and thus the same LiveState if initialized). pub async fn get(self: &Arc, id_or_name: &str) -> BoxliteResult> { + self.ensure_services_started().await; tracing::trace!(id_or_name = %id_or_name, "RuntimeInnerImpl::get called"); // Check in-memory cache first (for boxes created but not yet persisted) @@ -530,6 +1362,7 @@ impl RuntimeImpl { /// /// Checks in-memory cache first (for boxes not yet persisted), then database. pub async fn get_info(self: &Arc, id_or_name: &str) -> BoxliteResult> { + self.ensure_services_started().await; // Check in-memory cache first (for boxes created but not yet persisted) { let sync = self.sync_state.read().unwrap(); @@ -569,6 +1402,7 @@ impl RuntimeImpl { /// Includes both persisted boxes (from database) and in-memory boxes /// (created but not yet persisted). pub async fn list_info(self: &Arc) -> BoxliteResult> { + self.ensure_services_started().await; use std::collections::HashSet; // Get boxes from database - run on blocking thread pool @@ -605,6 +1439,7 @@ impl RuntimeImpl { /// /// Checks in-memory cache first (for boxes not yet persisted), then database. pub async fn exists(self: &Arc, id_or_name: &str) -> BoxliteResult { + self.ensure_services_started().await; // Check in-memory cache first { let sync = self.sync_state.read().unwrap(); @@ -660,13 +1495,18 @@ impl RuntimeImpl { /// survive parent process exit and runtime shutdown. /// /// # Arguments - /// * `timeout` - Seconds before force-kill. None=10s, Some(-1)=infinite + /// * `timeout` - Total shutdown deadline in seconds. None=10s, + /// Some(-1)=infinite /// /// # Returns /// Ok(()) if all boxes stopped successfully, Err if any box failed to stop. pub async fn shutdown(&self, timeout: Option) -> BoxliteResult<()> { + let shutdown_deadline = + timeout_to_duration(timeout).map(|duration| Instant::now() + duration); + // Check if already shut down (idempotent) if self.shutdown_token.is_cancelled() { + self.drain_crash_coordinator(shutdown_deadline).await; return Ok(()); } @@ -688,20 +1528,18 @@ impl RuntimeImpl { if active_boxes.is_empty() { tracing::info!("No active boxes to shutdown"); + self.drain_crash_coordinator(shutdown_deadline).await; return Ok(()); } tracing::info!(count = active_boxes.len(), "Stopping active boxes"); - // Convert timeout to duration - let timeout_duration = timeout_to_duration(timeout); - // Stop all boxes concurrently let stop_futures = active_boxes.iter().map(|box_impl| { let box_id = box_impl.id().to_string(); async move { - let result = if let Some(duration) = timeout_duration { - tokio::time::timeout(duration, box_impl.stop()).await + let result = if let Some(deadline) = shutdown_deadline { + tokio::time::timeout_at(deadline, box_impl.stop()).await } else { // Infinite timeout Ok(box_impl.stop().await) @@ -732,6 +1570,13 @@ impl RuntimeImpl { if errors.is_empty() { tracing::info!("Runtime shutdown complete"); + } else { + tracing::warn!("Shutdown completed with errors: {}", errors.join(", ")); + } + + self.drain_crash_coordinator(shutdown_deadline).await; + + if errors.is_empty() { Ok(()) } else { Err(BoxliteError::Internal(format!( @@ -741,6 +1586,58 @@ impl RuntimeImpl { } } + /// Wait for the crash coordinator within the remaining shutdown budget. + /// + /// If the grace period expires, the coordinator is asked to abort its + /// supervised tasks. Its handle remains stored so a later shutdown call can + /// reap it without detaching the coordinator. + async fn drain_crash_coordinator(&self, shutdown_deadline: Option) { + let grace_deadline = shutdown_deadline + .map(|deadline| deadline.min(Instant::now() + CRASH_COORDINATOR_GRACE_PERIOD)); + + let mut crash_handle = match grace_deadline { + Some(deadline) => { + match tokio::time::timeout_at(deadline, self.crash_handler_handle.lock()).await { + Ok(handle) => handle, + Err(_) => { + self.crash_force_cancel_token.cancel(); + tracing::warn!( + "Shutdown deadline reached while waiting to join crash coordinator" + ); + return; + } + } + } + None => self.crash_handler_handle.lock().await, + }; + + let Some(handle) = crash_handle.as_mut() else { + return; + }; + + let join_result = match grace_deadline { + Some(deadline) => tokio::time::timeout_at(deadline, &mut *handle).await.ok(), + None => Some((&mut *handle).await), + }; + + match join_result { + Some(Ok(())) => { + tracing::debug!("Crash coordinator stopped gracefully"); + *crash_handle = None; + } + Some(Err(e)) => { + tracing::warn!("Crash coordinator panicked: {:?}", e); + *crash_handle = None; + } + None => { + self.crash_force_cancel_token.cancel(); + tracing::warn!( + "Crash coordinator grace period expired; cancellation requested in background" + ); + } + } + } + /// Synchronous shutdown for atexit/Drop contexts. /// /// At atexit/Drop time, all `LiteBox` handles are gone (Weak refs dead), @@ -1109,6 +2006,7 @@ impl RuntimeImpl { options: BoxOptions, initial_status: BoxStatus, ) -> BoxliteResult { + self.ensure_services_started().await; use crate::litebox::config::ContainerRuntimeConfig; let box_id = BoxIDMint::mint(); @@ -1379,6 +2277,14 @@ impl RuntimeImpl { "Shim not verifiable (file missing, process dead, or PID reuse); \ marked Stopped" ); + } else if state.status == BoxStatus::Restarting { + state.force_status(BoxStatus::Stopped); + state.stop_info.cause = StopCause::RestartFailed; + state.stop_info.exit_time = Some(Utc::now()); + tracing::warn!( + box_id = %box_id, + "Interrupted restart found during recovery; marked Stopped" + ); } } } @@ -1481,15 +2387,121 @@ impl RuntimeImpl { } // ======================================================================== - // INTERNAL - BOX IMPL CACHE + // RESTART // ======================================================================== - /// Create a fresh BoxImpl that is not installed in the handle cache. - fn create_box_impl(self: &Arc, config: BoxConfig, state: BoxState) -> SharedBoxImpl { - use crate::litebox::box_impl::BoxImpl; - - let box_token = self.shutdown_token.child_token(); - Arc::new(BoxImpl::new(config, state, Arc::clone(self), box_token)) + /// Restart a box after crash handling if the lifecycle epoch still matches. + /// + /// Creates a fresh BoxImpl, starts it, and swaps it into the stable + /// BoxHandle so existing LiteBox handles continue to target the restarted + /// VM. The fresh BoxImpl has an empty OnceCell, so start() triggers + /// init_live_state() which runs the Restarting execution plan (same as + /// Stopped - reuse COW disks, spawn new VM). + /// + /// This method owns the per-box locking needed for the restart transition. + /// It holds the lock only while replacing cached state with Restarting, then + /// releases it before start(), because start() acquires the same lock while + /// rebuilding the VM. + /// + /// Returns `RestartOutcome::Stale` when a user lifecycle operation made + /// this delayed crash-restart attempt stale before it could start a VM. + /// The caller owns the success-state commit after `Started` is returned. + async fn restart( + self: &Arc, + box_id: &BoxID, + expected_epoch: u64, + ) -> BoxliteResult { + use crate::litebox::BoxStatus; + + tracing::info!(box_id = %box_id, "Restarting box"); + + // 1. Look up existing box config and state from DB + let Some((_, state)) = self.box_manager.box_by_id(box_id)? else { + tracing::debug!(box_id = %box_id, "Crash restart skipped because box was removed"); + return Ok(RestartOutcome::Stale); + }; + + let lock_id = state + .lock_id + .ok_or_else(|| BoxliteError::Internal(format!("box {box_id} has no lock_id")))?; + let locker = self.lock_manager.retrieve(lock_id)?; + + let box_impl = { + let _lock_guard = acquire_owned_lock_or_cancel( + locker, + &self.shutdown_token, + "Runtime is shutting down while waiting to restart box", + ) + .await?; + + // 2. Re-read under the lock, then replace cached state with Restarting. + let Some((config, mut state)) = self.box_manager.box_by_id(box_id)? else { + tracing::debug!(box_id = %box_id, "Crash restart skipped because box was removed"); + return Ok(RestartOutcome::Stale); + }; + + if !crash_restart_matches_plan(&state, expected_epoch) { + if state.lifecycle_epoch() != expected_epoch { + tracing::debug!( + box_id = %box_id, + expected_epoch, + actual_epoch = state.lifecycle_epoch(), + "Crash restart skipped because lifecycle epoch changed" + ); + } else { + tracing::debug!( + box_id = %box_id, + status = ?state.status, + "Crash restart skipped because box state changed" + ); + } + return Ok(RestartOutcome::Stale); + } + + // 3. Get the stable handle and retire its current BoxImpl. + // The handle remains cached so existing LiteBox values can observe the + // fresh BoxImpl after restart succeeds. + let (handle, _) = self.get_or_create_box_handle(config.clone(), state.clone()); + let old_box_impl = handle.current(); + old_box_impl.abort_health_check(); + old_box_impl.shutdown_token.cancel(); + + // 4. Update state to Restarting and persist + state.force_status(BoxStatus::Restarting); + state.set_pid(None); + state.clear_health_status(); + self.box_manager.save_box(box_id, &state)?; + + // 5. Create fresh BoxImpl with updated state. The transition lock is + // released at the end of this block before start() takes the same + // lock for VM rebuild. + let box_impl = self.create_box_impl(config, state); + let _old_box_impl = handle.swap_current(Arc::clone(&box_impl)); + { + let mut sync = self.sync_state.write().unwrap(); + sync.restart_owned_handles_by_id + .insert(box_id.clone(), handle); + } + box_impl + }; + + // 6. Call start() on fresh BoxImpl → empty OnceCell → init_live_state() + // BoxBuilder sees status=Restarting → same pipeline as Stopped + box_impl.start().await?; + + Ok(RestartOutcome::Started { box_impl, lock_id }) + } + + // ======================================================================== + // INTERNAL - BOX IMPL CACHE + // ======================================================================== + + /// Create a fresh BoxImpl that is not installed in the handle cache. + fn create_box_impl(self: &Arc, config: BoxConfig, state: BoxState) -> SharedBoxImpl { + use crate::litebox::box_impl::BoxImpl; + + let box_token = self.shutdown_token.child_token(); + Arc::new(BoxImpl::new(config, state, Arc::clone(self), box_token)) } /// Get existing BoxHandle from cache or create new one. @@ -1556,6 +2568,7 @@ impl RuntimeImpl { pub(crate) fn invalidate_box_handle(&self, box_id: &BoxID, box_name: Option<&str>) { let mut sync = self.sync_state.write().unwrap(); sync.active_handles_by_id.remove(box_id); + sync.restart_owned_handles_by_id.remove(box_id); if let Some(name) = box_name { sync.active_handles_by_name.remove(name); } @@ -1793,6 +2806,57 @@ mod tests { (runtime, temp_dir) } + fn create_state_commit_test_runtime() -> (SharedRuntimeImpl, TempDir) { + let temp_dir = TempDir::new_in("/tmp").expect("Failed to create temp dir"); + let fs_config = FsLayoutConfig::without_bind_mount(); + let layout = FilesystemLayout::new(temp_dir.path().to_path_buf(), fs_config); + layout.prepare().expect("prepare layout"); + let runtime_lock = RuntimeLock::acquire(layout.home_dir()).expect("acquire runtime lock"); + let db = Database::open(&layout.db_dir().join("boxlite.db")).expect("open database"); + let image_manager = ImageManager::new(layout.images_dir(), db.clone(), vec![]) + .expect("create image manager"); + let base_disk_store = crate::db::BaseDiskStore::new(db.clone()); + let base_disk_mgr = + crate::disk::BaseDiskManager::new(layout.bases_dir(), base_disk_store.clone()); + let snapshot_store = crate::db::SnapshotStore::new(db.clone()); + let snapshot_mgr = crate::litebox::snapshot_mgr::SnapshotManager::new(snapshot_store); + let box_store = BoxStore::new(db); + let lock_manager: Arc = + Arc::new(FileLockManager::new(layout.locks_dir()).expect("create lock manager")); + let image_disk_mgr = + ImageDiskManager::new(layout.image_layout().disk_images_dir(), layout.temp_dir()); + let guest_rootfs_mgr = GuestRootfsManager::new(base_disk_mgr.clone(), layout.temp_dir()); + let (crash_tx, crash_rx) = mpsc::channel::(100); + + let runtime = Arc::new(RuntimeImpl { + sync_state: RwLock::new(SynchronizedState { + active_handles_by_id: HashMap::new(), + active_handles_by_name: HashMap::new(), + restart_owned_handles_by_id: HashMap::new(), + }), + box_manager: BoxManager::new(box_store), + image_manager, + layout, + image_disk_mgr, + guest_rootfs_mgr, + guest_rootfs: Arc::new(OnceCell::new()), + runtime_metrics: RuntimeMetricsStorage::new(), + base_disk_mgr, + snapshot_mgr, + lock_manager, + network_factory: crate::net::default_factory(), + _runtime_lock: runtime_lock, + shutdown_token: CancellationToken::new(), + crash_tx, + crash_handler_handle: AsyncMutex::new(None), + crash_force_cancel_token: CancellationToken::new(), + pending_crash_rx: std::sync::Mutex::new(Some(crash_rx)), + services_started: tokio::sync::OnceCell::new(), + }); + + (runtime, temp_dir) + } + /// Create a minimal BoxConfig for testing. fn test_box_config(detach: bool) -> BoxConfig { BoxConfig { @@ -1871,6 +2935,78 @@ mod tests { // shutdown() tests // ==================================================================== + #[test] + fn test_new_does_not_start_background_tasks() { + let (runtime, _dir) = create_test_runtime(); + + // new() is sync — no crash coordinator or recovery tasks should be spawned + assert!(runtime.crash_handler_handle.try_lock().unwrap().is_none()); + assert!(runtime.services_started.get().is_none()); + } + + #[tokio::test] + async fn test_async_call_starts_background_services() { + let (runtime, _dir) = create_test_runtime(); + + // Before any async call, services are not started + assert!(runtime.services_started.get().is_none()); + + // Calling an async method triggers lazy initialization + runtime.list_info().await.unwrap(); + + // Services should now be started + assert!(runtime.services_started.get().is_some()); + assert!(runtime.crash_handler_handle.lock().await.is_some()); + } + + #[tokio::test] + async fn test_cancelled_service_initialization_preserves_crash_receiver() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let handle_slot = runtime.crash_handler_handle.lock().await; + let task_runtime = Arc::clone(&runtime); + let initialization = tokio::spawn(async move { + task_runtime.ensure_services_started().await; + }); + tokio::task::yield_now().await; + + assert!(runtime.pending_crash_rx.lock().unwrap().is_some()); + initialization.abort(); + assert!( + initialization + .await + .expect_err("initialization should be cancelled") + .is_cancelled() + ); + drop(handle_slot); + + assert!(runtime.services_started.get().is_none()); + runtime.ensure_services_started().await; + assert!(runtime.services_started.get().is_some()); + assert!(runtime.pending_crash_rx.lock().unwrap().is_none()); + assert!(runtime.crash_handler_handle.lock().await.is_some()); + + runtime.shutdown(None).await.unwrap(); + } + + #[tokio::test] + async fn test_shutdown_during_service_initialization_preserves_crash_receiver() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let handle_slot = runtime.crash_handler_handle.lock().await; + let task_runtime = Arc::clone(&runtime); + let initialization = tokio::spawn(async move { + task_runtime.ensure_services_started().await; + }); + tokio::task::yield_now().await; + + runtime.shutdown_token.cancel(); + drop(handle_slot); + initialization.await.unwrap(); + + assert!(runtime.services_started.get().is_none()); + assert!(runtime.pending_crash_rx.lock().unwrap().is_some()); + assert!(runtime.crash_handler_handle.lock().await.is_none()); + } + #[tokio::test] async fn test_shutdown_is_idempotent() { let (runtime, _dir) = create_test_runtime(); @@ -1896,11 +3032,664 @@ mod tests { #[tokio::test] async fn test_shutdown_with_empty_active_boxes() { - let (runtime, _dir) = create_test_runtime(); + let (runtime, _dir) = create_state_commit_test_runtime(); + let coordinator_drained = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let task_drained = Arc::clone(&coordinator_drained); + let task_shutdown = runtime.shutdown_token.clone(); + *runtime.crash_handler_handle.lock().await = Some(tokio::spawn(async move { + task_shutdown.cancelled().await; + task_drained.store(true, std::sync::atomic::Ordering::SeqCst); + })); // No boxes created — shutdown should complete cleanly let result = runtime.shutdown(Some(1)).await; assert!(result.is_ok()); + assert!(coordinator_drained.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn test_repeated_shutdown_drains_remaining_crash_coordinator() { + let (runtime, _dir) = create_state_commit_test_runtime(); + runtime.shutdown_token.cancel(); + let coordinator_drained = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let task_drained = Arc::clone(&coordinator_drained); + *runtime.crash_handler_handle.lock().await = Some(tokio::spawn(async move { + tokio::task::yield_now().await; + task_drained.store(true, std::sync::atomic::Ordering::SeqCst); + })); + + runtime.shutdown(None).await.unwrap(); + + assert!(coordinator_drained.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn test_shutdown_timeout_does_not_wait_forever_for_crash_coordinator() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let release_coordinator = Arc::new(tokio::sync::Semaphore::new(0)); + let task_release = Arc::clone(&release_coordinator); + *runtime.crash_handler_handle.lock().await = Some(tokio::spawn(async move { + let _permit = task_release + .acquire() + .await + .expect("release semaphore open"); + })); + + tokio::time::timeout(std::time::Duration::from_secs(2), runtime.shutdown(Some(1))) + .await + .expect("shutdown should respect its timeout") + .expect("shutdown should succeed without active boxes"); + assert!( + runtime.crash_handler_handle.lock().await.is_some(), + "timed-out coordinator handle must remain supervised" + ); + + release_coordinator.add_permits(1); + runtime + .shutdown(Some(1)) + .await + .expect("later shutdown should reap coordinator"); + assert!(runtime.crash_handler_handle.lock().await.is_none()); + } + + #[tokio::test] + async fn test_infinite_crash_coordinator_drain_waits_for_task() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let coordinator_drained = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let task_drained = Arc::clone(&coordinator_drained); + *runtime.crash_handler_handle.lock().await = Some(tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + task_drained.store(true, std::sync::atomic::Ordering::SeqCst); + })); + + runtime.drain_crash_coordinator(None).await; + + assert!(coordinator_drained.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn test_cancelled_drain_keeps_handle_for_concurrent_caller() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let release_coordinator = Arc::new(tokio::sync::Semaphore::new(0)); + let task_release = Arc::clone(&release_coordinator); + let coordinator_drained = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let task_drained = Arc::clone(&coordinator_drained); + *runtime.crash_handler_handle.lock().await = Some(tokio::spawn(async move { + let _permit = task_release + .acquire() + .await + .expect("release semaphore open"); + task_drained.store(true, std::sync::atomic::Ordering::SeqCst); + })); + + let mut first_drain = Box::pin(runtime.drain_crash_coordinator(None)); + assert!( + futures::poll!(&mut first_drain).is_pending(), + "first drain should wait for the coordinator" + ); + assert!( + runtime.crash_handler_handle.try_lock().is_err(), + "first drain should own the handle lock" + ); + + let mut second_drain = Box::pin(runtime.drain_crash_coordinator(None)); + assert!( + futures::poll!(&mut second_drain).is_pending(), + "concurrent drain must wait for the same coordinator handle" + ); + + drop(first_drain); + release_coordinator.add_permits(1); + tokio::time::timeout(std::time::Duration::from_secs(1), second_drain) + .await + .expect("second drain should take over the coordinator handle"); + + assert!(coordinator_drained.load(std::sync::atomic::Ordering::SeqCst)); + assert!(runtime.crash_handler_handle.lock().await.is_none()); + } + + // ==================================================================== + // crash coordinator tests + // ==================================================================== + + #[test] + fn test_crash_coordinator_marks_each_box_pending_once() { + let mut pending_crashes = HashSet::new(); + let box_id = BoxIDMint::mint(); + + assert!(CrashCoordinator::mark_pending_crash( + &mut pending_crashes, + &box_id + )); + assert!(!CrashCoordinator::mark_pending_crash( + &mut pending_crashes, + &box_id + )); + assert_eq!(pending_crashes.len(), 1); + } + + #[test] + fn test_crash_coordinator_skips_when_runtime_dropped() { + let (_tx, rx) = mpsc::channel(1); + let mut coordinator = CrashCoordinator { + runtime: Weak::new(), + shutdown_token: CancellationToken::new(), + force_cancel_token: CancellationToken::new(), + crash_rx: rx, + crash_rx_closed: false, + pending_crashes: HashSet::new(), + running_crashes: JoinSet::new(), + task_boxes: HashMap::new(), + }; + let box_id = BoxIDMint::mint(); + + assert!(!coordinator.schedule_crash(box_id)); + assert!(coordinator.pending_crashes.is_empty()); + assert!(coordinator.crash_rx_closed); + assert!(coordinator.running_crashes.is_empty()); + assert!(coordinator.task_boxes.is_empty()); + } + + #[tokio::test] + async fn test_crash_coordinator_force_cancel_aborts_supervised_tasks() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let (tx, rx) = mpsc::channel(1); + let mut coordinator = CrashCoordinator::new(runtime, rx); + let box_id = BoxIDMint::mint(); + coordinator.pending_crashes.insert(box_id.clone()); + let abort_handle = coordinator + .running_crashes + .spawn(std::future::pending::()); + coordinator.task_boxes.insert(abort_handle.id(), box_id); + coordinator.force_cancel_token.cancel(); + drop(tx); + + tokio::time::timeout(std::time::Duration::from_secs(1), coordinator.run()) + .await + .expect("forced cancellation should drain crash tasks"); + } + + #[tokio::test] + async fn test_crash_coordinator_cleans_up_completed_task() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let (_tx, rx) = mpsc::channel(1); + let mut coordinator = CrashCoordinator::new(runtime, rx); + let box_id = BoxIDMint::mint(); + coordinator.pending_crashes.insert(box_id.clone()); + let task_box_id = box_id.clone(); + let abort_handle = coordinator + .running_crashes + .spawn(async move { task_box_id }); + coordinator + .task_boxes + .insert(abort_handle.id(), box_id.clone()); + + let result = coordinator + .running_crashes + .join_next_with_id() + .await + .expect("completed task"); + coordinator.finish_crash_task(result); + + assert!(!coordinator.pending_crashes.contains(&box_id)); + assert!(coordinator.task_boxes.is_empty()); + assert!(CrashCoordinator::mark_pending_crash( + &mut coordinator.pending_crashes, + &box_id + )); + } + + #[tokio::test] + async fn test_crash_coordinator_cleans_up_panicked_task() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let (_tx, rx) = mpsc::channel(1); + let mut coordinator = CrashCoordinator::new(runtime, rx); + let box_id = BoxIDMint::mint(); + coordinator.pending_crashes.insert(box_id.clone()); + let abort_handle = coordinator + .running_crashes + .spawn(async move { panic!("test crash handler panic") }); + coordinator + .task_boxes + .insert(abort_handle.id(), box_id.clone()); + + let result = coordinator + .running_crashes + .join_next_with_id() + .await + .expect("panicked task"); + coordinator.finish_crash_task(result); + + assert!(!coordinator.pending_crashes.contains(&box_id)); + assert!(coordinator.task_boxes.is_empty()); + } + + #[tokio::test] + async fn test_crash_coordinator_cleans_up_cancelled_task() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let (_tx, rx) = mpsc::channel(1); + let mut coordinator = CrashCoordinator::new(runtime, rx); + let box_id = BoxIDMint::mint(); + coordinator.pending_crashes.insert(box_id.clone()); + let task_box_id = box_id.clone(); + let abort_handle = coordinator.running_crashes.spawn(async move { + std::future::pending::<()>().await; + task_box_id + }); + coordinator + .task_boxes + .insert(abort_handle.id(), box_id.clone()); + abort_handle.abort(); + + let result = coordinator + .running_crashes + .join_next_with_id() + .await + .expect("cancelled task"); + coordinator.finish_crash_task(result); + + assert!(!coordinator.pending_crashes.contains(&box_id)); + assert!(coordinator.task_boxes.is_empty()); + } + + #[tokio::test] + async fn test_crash_coordinator_drains_supervised_tasks_on_shutdown() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let (_tx, rx) = mpsc::channel(1); + let mut coordinator = CrashCoordinator::new(runtime, rx); + let box_id = BoxIDMint::mint(); + coordinator.pending_crashes.insert(box_id.clone()); + let task_box_id = box_id.clone(); + let task_shutdown = coordinator.shutdown_token.clone(); + let abort_handle = coordinator.running_crashes.spawn(async move { + task_shutdown.cancelled().await; + task_box_id + }); + coordinator.task_boxes.insert(abort_handle.id(), box_id); + coordinator.shutdown_token.cancel(); + + tokio::time::timeout(std::time::Duration::from_secs(1), coordinator.run()) + .await + .expect("coordinator should drain cancelled crash tasks"); + } + + // ==================================================================== + // crash restart epoch tests + // ==================================================================== + + #[test] + fn test_crash_restart_plan_rejects_stale_lifecycle_epoch() { + let mut state = BoxState::new(); + state.force_status(BoxStatus::Crashed); + state.lifecycle_epoch = 7; + + assert!(!crash_restart_matches_plan(&state, 6)); + } + + #[test] + fn test_crash_restart_plan_rejects_user_stopped_state() { + let mut state = BoxState::new(); + state.force_status(BoxStatus::Stopped); + state.lifecycle_epoch = 3; + + assert!(!crash_restart_matches_plan(&state, 3)); + } + + #[test] + fn test_crash_restart_plan_accepts_current_crashed_state() { + let mut state = BoxState::new(); + state.force_status(BoxStatus::Crashed); + state.lifecycle_epoch = 3; + + assert!(crash_restart_matches_plan(&state, 3)); + } + + #[test] + fn test_crash_restart_plan_accepts_current_restarting_state() { + let mut state = BoxState::new(); + state.force_status(BoxStatus::Restarting); + state.lifecycle_epoch = 3; + + assert!(crash_restart_matches_plan(&state, 3)); + } + + fn add_state_commit_test_box( + runtime: &RuntimeImpl, + status: BoxStatus, + lifecycle_epoch: u64, + ) -> BoxConfig { + let config = test_box_config(false); + let mut state = BoxState::new(); + state.force_status(status); + state.lifecycle_epoch = lifecycle_epoch; + state.stop_info.cause = StopCause::Normal; + let lock_id = runtime.lock_manager.allocate().expect("allocate lock"); + state.set_lock_id(lock_id); + + runtime + .box_manager + .add_box(&config, &state) + .expect("add box"); + config + } + + #[tokio::test] + async fn test_commit_restart_success_resets_stop_info() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = test_box_config(false); + let mut state = BoxState::new(); + state.force_status(BoxStatus::Running); + state.lifecycle_epoch = 3; + state.stop_info.cause = StopCause::RestartFailed; + state.stop_info.exit_code = Some(1); + state.stop_info.restart_count = 2; + let lock_id = runtime.lock_manager.allocate().expect("allocate lock"); + state.set_lock_id(lock_id); + runtime + .box_manager + .add_box(&config, &state) + .expect("add box"); + let box_impl = runtime.create_box_impl(config.clone(), state); + + let committed = + RuntimeImpl::commit_restart_success(&runtime, &config.id, 3, &box_impl, lock_id) + .await + .expect("commit restart success"); + + assert!(committed); + let in_memory = box_impl.state.read().clone(); + let (_, persisted) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert_eq!(in_memory.stop_info.cause, StopCause::Normal); + assert_eq!(in_memory.stop_info.exit_code, None); + assert_eq!(in_memory.stop_info.restart_count, 0); + assert!(in_memory.stop_info.restarted_at.is_some()); + assert_eq!(persisted.stop_info.cause, in_memory.stop_info.cause); + assert_eq!(persisted.stop_info.exit_code, in_memory.stop_info.exit_code); + assert_eq!(persisted.stop_info.exit_time, in_memory.stop_info.exit_time); + assert_eq!( + persisted.stop_info.restart_count, + in_memory.stop_info.restart_count + ); + assert_eq!( + persisted.stop_info.restarted_at, + in_memory.stop_info.restarted_at + ); + } + + #[tokio::test] + async fn test_commit_restart_success_rejects_stale_epoch() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = test_box_config(false); + let mut state = BoxState::new(); + state.force_status(BoxStatus::Running); + state.lifecycle_epoch = 4; + state.stop_info.cause = StopCause::RestartFailed; + state.stop_info.restart_count = 2; + let lock_id = runtime.lock_manager.allocate().expect("allocate lock"); + state.set_lock_id(lock_id); + runtime + .box_manager + .add_box(&config, &state) + .expect("add box"); + let box_impl = runtime.create_box_impl(config.clone(), state); + + let committed = + RuntimeImpl::commit_restart_success(&runtime, &config.id, 3, &box_impl, lock_id) + .await + .expect("commit restart success"); + + assert!(!committed); + let in_memory = box_impl.state.read().clone(); + let (_, persisted) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert_eq!(in_memory.stop_info.cause, StopCause::RestartFailed); + assert_eq!(in_memory.stop_info.restart_count, 2); + assert_eq!(persisted.stop_info.cause, in_memory.stop_info.cause); + assert_eq!( + persisted.stop_info.restart_count, + in_memory.stop_info.restart_count + ); + } + + #[tokio::test] + async fn test_commit_restart_success_rejects_stopped_state() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = test_box_config(false); + let mut state = BoxState::new(); + state.force_status(BoxStatus::Stopped); + state.lifecycle_epoch = 3; + state.stop_info.cause = StopCause::RestartFailed; + state.stop_info.restart_count = 2; + let lock_id = runtime.lock_manager.allocate().expect("allocate lock"); + state.set_lock_id(lock_id); + runtime + .box_manager + .add_box(&config, &state) + .expect("add box"); + let box_impl = runtime.create_box_impl(config.clone(), state); + + let committed = + RuntimeImpl::commit_restart_success(&runtime, &config.id, 3, &box_impl, lock_id) + .await + .expect("commit restart success"); + + assert!(!committed); + let in_memory = box_impl.state.read().clone(); + let (_, persisted) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert_eq!(in_memory.stop_info.cause, StopCause::RestartFailed); + assert_eq!(in_memory.stop_info.restart_count, 2); + assert_eq!(persisted.stop_info.cause, in_memory.stop_info.cause); + assert_eq!( + persisted.stop_info.restart_count, + in_memory.stop_info.restart_count + ); + } + + #[tokio::test] + async fn test_commit_crash_restart_state_rejects_stale_lifecycle_epoch() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = add_state_commit_test_box(&runtime, BoxStatus::Crashed, 4); + + let committed = RuntimeImpl::commit_crash_restart_state(&runtime, &config.id, 3, |state| { + state.stop_info.cause = StopCause::RestartFailed; + }) + .await + .expect("commit state"); + + let (_, saved_state) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert!(!committed); + assert_eq!(saved_state.status, BoxStatus::Crashed); + assert_eq!(saved_state.stop_info.cause, StopCause::Normal); + } + + #[tokio::test] + async fn test_commit_crash_restart_state_rejects_stopped_state_at_same_epoch() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = add_state_commit_test_box(&runtime, BoxStatus::Stopped, 3); + + let committed = RuntimeImpl::commit_crash_restart_state(&runtime, &config.id, 3, |state| { + state.stop_info.cause = StopCause::RestartFailed; + }) + .await + .expect("commit state"); + + let (_, saved_state) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert!(!committed); + assert_eq!(saved_state.status, BoxStatus::Stopped); + assert_eq!(saved_state.stop_info.cause, StopCause::Normal); + } + + #[tokio::test] + async fn test_commit_crash_restart_state_accepts_current_crashed_state() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = add_state_commit_test_box(&runtime, BoxStatus::Crashed, 3); + + let committed = RuntimeImpl::commit_crash_restart_state(&runtime, &config.id, 3, |state| { + state.force_status(BoxStatus::Stopped); + state.stop_info.cause = StopCause::MaxRetriesExceeded; + }) + .await + .expect("commit state"); + + let (_, saved_state) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert!(committed); + assert_eq!(saved_state.status, BoxStatus::Stopped); + assert_eq!(saved_state.stop_info.cause, StopCause::MaxRetriesExceeded); + } + + #[tokio::test] + async fn test_commit_crash_restart_state_accepts_current_restarting_state() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = add_state_commit_test_box(&runtime, BoxStatus::Restarting, 3); + + let committed = RuntimeImpl::commit_crash_restart_state(&runtime, &config.id, 3, |state| { + state.force_status(BoxStatus::Stopped); + state.stop_info.cause = StopCause::Normal; + }) + .await + .expect("commit state"); + + let (_, saved_state) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert!(committed); + assert_eq!(saved_state.status, BoxStatus::Stopped); + assert_eq!(saved_state.stop_info.cause, StopCause::Normal); + } + + #[tokio::test] + async fn test_mark_restart_failed_does_not_overwrite_stale_stopped_state() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = test_box_config(false); + let mut state = BoxState::new(); + state.force_status(BoxStatus::Stopped); + state.lifecycle_epoch = 4; + state.stop_info.cause = StopCause::Normal; + let lock_id = runtime.lock_manager.allocate().expect("allocate lock"); + state.set_lock_id(lock_id); + + runtime + .box_manager + .add_box(&config, &state) + .expect("add box"); + + let committed = RuntimeImpl::mark_restart_failed(&runtime, &config.id, 3, 5) + .await + .expect("mark restart failed"); + + let (_, saved_state) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert!(!committed); + assert_eq!(saved_state.status, BoxStatus::Stopped); + assert_eq!(saved_state.stop_info.cause, StopCause::Normal); + } + + #[tokio::test] + async fn test_cancelled_crash_restart_does_not_overwrite_stale_stopped_state() { + let (runtime, _dir) = create_state_commit_test_runtime(); + let config = test_box_config(false); + let mut state = BoxState::new(); + state.force_status(BoxStatus::Stopped); + state.lifecycle_epoch = 4; + state.stop_info.cause = StopCause::SystemReboot; + state.stop_info.restart_count = 99; + let lock_id = runtime.lock_manager.allocate().expect("allocate lock"); + state.set_lock_id(lock_id); + + runtime + .box_manager + .add_box(&config, &state) + .expect("add box"); + + let shutdown_token = CancellationToken::new(); + shutdown_token.cancel(); + RuntimeImpl::run_restart_loop( + Arc::downgrade(&runtime), + shutdown_token, + config.id.clone(), + CrashRestartPlan { + restart_policy: Some(RestartPolicy::Always), + expected_epoch: 3, + current_restart_count: 0, + new_restart_count: 1, + exit_code: Some(1), + }, + ) + .await; + + let (_, saved_state) = runtime + .box_manager + .box_by_id(&config.id) + .expect("load box") + .expect("box exists"); + assert_eq!(saved_state.status, BoxStatus::Stopped); + assert_eq!(saved_state.lifecycle_epoch(), 4); + assert_eq!(saved_state.stop_info.cause, StopCause::SystemReboot); + assert_eq!(saved_state.stop_info.restart_count, 99); + } + + #[test] + fn test_restart_denied_stop_cause_for_no_policy() { + assert_eq!( + stop_cause_when_restart_denied(None, Some(1), false), + StopCause::CrashedNoPolicy + ); + assert_eq!( + stop_cause_when_restart_denied(Some(&RestartPolicy::No), Some(1), false), + StopCause::CrashedNoPolicy + ); + } + + #[test] + fn test_restart_denied_stop_cause_for_on_failure_clean_exit() { + assert_eq!( + stop_cause_when_restart_denied( + Some(&RestartPolicy::OnFailure { max_retries: 3 }), + Some(0), + false, + ), + StopCause::Normal + ); + } + + #[test] + fn test_restart_denied_stop_cause_for_on_failure_max_retries() { + assert_eq!( + stop_cause_when_restart_denied( + Some(&RestartPolicy::OnFailure { max_retries: 3 }), + Some(1), + true, + ), + StopCause::MaxRetriesExceeded + ); } // ==================================================================== diff --git a/src/boxlite/tests/restart_policy.rs b/src/boxlite/tests/restart_policy.rs new file mode 100644 index 000000000..179df579f --- /dev/null +++ b/src/boxlite/tests/restart_policy.rs @@ -0,0 +1,459 @@ +//! Integration tests for restart policy functionality. +//! +//! # Prerequisites +//! +//! These tests require a real VM environment: +//! 1. Build the runtime: `make runtime:debug` +//! 2. Run with: `cargo test -p boxlite --test restart_policy -- --test-threads=1` + +mod common; + +use boxlite::StopCause; +use boxlite::litebox::HealthState; +use boxlite::runtime::advanced_options::{AdvancedBoxOptions, HealthCheckOptions, RestartPolicy}; +use boxlite::runtime::options::{BoxOptions, RootfsSpec}; +use boxlite::runtime::types::{BoxInfo, BoxStatus}; +use boxlite::{BoxID, BoxliteRuntime}; +use common::box_test::BoxTestBase; +use std::process::Command; +use std::time::Duration; +use tokio::time::{MissedTickBehavior, interval, timeout}; + +const STATUS_POLL_INTERVAL: Duration = Duration::from_millis(500); +const FAST_STATUS_POLL_INTERVAL: Duration = Duration::from_millis(10); +const STATUS_WAIT_TIMEOUT: Duration = Duration::from_secs(30); +const HEALTH_CHECK_INTERVAL: Duration = Duration::from_millis(200); +const HEALTH_CHECK_TIMEOUT: Duration = Duration::from_millis(500); +const HEALTH_CHECK_START_PERIOD: Duration = Duration::from_millis(0); + +/// Build `BoxOptions` with both restart policy and custom health check. +fn restart_and_health_opts(policy: RestartPolicy) -> BoxOptions { + BoxOptions { + rootfs: RootfsSpec::Image("alpine:latest".into()), + advanced: AdvancedBoxOptions { + restart_policy: Some(policy), + health_check: Some(HealthCheckOptions { + interval: HEALTH_CHECK_INTERVAL, + timeout: HEALTH_CHECK_TIMEOUT, + retries: 3, + start_period: HEALTH_CHECK_START_PERIOD, + }), + ..Default::default() + }, + auto_delete: Some(0), + ..Default::default() + } +} + +/// Build `BoxOptions` with restart policy only. +fn restart_only_opts(policy: RestartPolicy) -> BoxOptions { + BoxOptions { + rootfs: RootfsSpec::Image("alpine:latest".into()), + advanced: AdvancedBoxOptions { + restart_policy: Some(policy), + health_check: None, + ..Default::default() + }, + auto_delete: Some(0), + ..Default::default() + } +} + +async fn wait_for_info( + runtime: &BoxliteRuntime, + box_id: &BoxID, + wait_timeout: Duration, + description: &str, + mut predicate: impl FnMut(&BoxInfo) -> bool, +) -> Option { + wait_for_info_polling( + runtime, + box_id, + wait_timeout, + STATUS_POLL_INTERVAL, + description, + &mut predicate, + ) + .await +} + +async fn wait_for_info_polling( + runtime: &BoxliteRuntime, + box_id: &BoxID, + wait_timeout: Duration, + poll_interval: Duration, + description: &str, + mut predicate: impl FnMut(&BoxInfo) -> bool, +) -> Option { + timeout(wait_timeout, async { + let mut ticker = interval(poll_interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + + loop { + ticker.tick().await; + let info = runtime + .get_info(box_id.as_str()) + .await + .expect("get box info") + .unwrap_or_else(|| { + panic!("box {} disappeared while waiting for {description}", box_id) + }); + + if predicate(&info) { + return info; + } + } + }) + .await + .ok() +} + +async fn wait_for_info_fast( + runtime: &BoxliteRuntime, + box_id: &BoxID, + wait_timeout: Duration, + description: &str, + predicate: impl FnMut(&BoxInfo) -> bool, +) -> Option { + wait_for_info_polling( + runtime, + box_id, + wait_timeout, + FAST_STATUS_POLL_INTERVAL, + description, + predicate, + ) + .await +} + +async fn expect_info( + runtime: &BoxliteRuntime, + box_id: &BoxID, + description: &str, + predicate: impl FnMut(&BoxInfo) -> bool, +) -> BoxInfo { + wait_for_info(runtime, box_id, STATUS_WAIT_TIMEOUT, description, predicate) + .await + .unwrap_or_else(|| { + panic!( + "timed out after {:?} waiting for {description}", + STATUS_WAIT_TIMEOUT + ) + }) +} + +async fn expect_status(runtime: &BoxliteRuntime, box_id: &BoxID, status: BoxStatus) -> BoxInfo { + expect_info(runtime, box_id, &format!("status {status}"), |info| { + info.status == status + }) + .await +} + +async fn expect_restarted(runtime: &BoxliteRuntime, box_id: &BoxID, old_pid: u32) -> BoxInfo { + expect_info(runtime, box_id, "box restart with a new shim PID", |info| { + info.status == BoxStatus::Running && info.pid.is_some_and(|pid| pid != old_pid) + }) + .await +} + +fn kill_process(pid: u32) { + Command::new("kill") + .arg("-9") + .arg(pid.to_string()) + .output() + .expect("Failed to kill shim process"); +} + +async fn cleanup_box(t: &BoxTestBase, box_id: &BoxID) { + t.runtime + .remove(box_id.as_str(), true) + .await + .expect("remove restart-policy test box"); + t.runtime + .shutdown(Some(common::TEST_SHUTDOWN_TIMEOUT)) + .await + .expect("shutdown restart-policy test runtime"); +} + +// ============================================================================ +// RESTART POLICY: No +// ============================================================================ + +#[tokio::test] +async fn restart_policy_no_does_not_restart() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::No)).await; + + // Start the box + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + // Verify box is running + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + let shim_pid = info.pid.expect("No shim PID found"); + + // Kill the shim process + kill_process(shim_pid); + + // Box should be stopped, not restarted (No policy) + let info = expect_status(&t.runtime, &box_id, BoxStatus::Stopped).await; + assert_eq!( + info.status, + BoxStatus::Stopped, + "Expected box to be Stopped with No restart policy" + ); + + // Verify stop cause + assert_eq!(info.stop_info.cause, StopCause::CrashedNoPolicy); + assert_eq!(info.stop_info.restart_count, 1); + + cleanup_box(&t, &box_id).await; +} + +// ============================================================================ +// RESTART POLICY: Always +// ============================================================================ + +#[tokio::test] +async fn restart_policy_always_restarts_on_crash() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::Always)).await; + + // Start the box + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + // Verify box is running + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + let original_pid = info.pid.expect("No shim PID found"); + + // Kill the shim process + kill_process(original_pid); + + // Box should be running again with a new PID + let info = expect_restarted(&t.runtime, &box_id, original_pid).await; + assert_eq!( + info.status, + BoxStatus::Running, + "Expected box to be Running after auto-restart" + ); + + let new_pid = info.pid.expect("No shim PID found after restart"); + assert_ne!( + original_pid, new_pid, + "Expected new PID after restart, got same PID" + ); + + // After successful restart, stop_info is reset (restart_count=0, restarted_at set) + assert_eq!(info.stop_info.restart_count, 0); + assert!(info.stop_info.restarted_at.is_some()); + + cleanup_box(&t, &box_id).await; +} + +#[tokio::test] +async fn restart_policy_only_initializes_auto_health_status() { + let t = BoxTestBase::with_options(restart_only_opts(RestartPolicy::Always)).await; + + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + assert_eq!(info.health_status.state, HealthState::Starting); + assert_eq!(info.health_status.failures, 0); + assert!( + info.health_status.last_check.is_some(), + "auto-enabled health check should initialize health status on start" + ); + + cleanup_box(&t, &box_id).await; +} + +// ============================================================================ +// RESTART POLICY: OnFailure +// ============================================================================ + +#[tokio::test] +async fn restart_policy_on_failure_restarts_within_max_retries() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::OnFailure { + max_retries: 2, + })) + .await; + + // Start the box + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + // Verify box is running + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + + // First crash + let shim_pid = info.pid.expect("No shim PID found"); + kill_process(shim_pid); + + // Should restart because the first crash is within the retry budget. + let info = expect_restarted(&t.runtime, &box_id, shim_pid).await; + assert_eq!(info.status, BoxStatus::Running); + assert_eq!(info.stop_info.restart_count, 0); + assert!(info.stop_info.restarted_at.is_some()); + + cleanup_box(&t, &box_id).await; +} + +#[tokio::test] +async fn restart_policy_on_failure_zero_retries_stops_on_crash() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::OnFailure { + max_retries: 0, + })) + .await; + + // Start the box + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + // Crash the shim. + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + let shim_pid = info.pid.expect("No shim PID found"); + kill_process(shim_pid); + + // With zero retries, the first failure exhausts the retry budget. + let info = expect_status(&t.runtime, &box_id, BoxStatus::Stopped).await; + assert_eq!( + info.status, + BoxStatus::Stopped, + "Expected box to be Stopped when max_retries is zero" + ); + assert_eq!(info.stop_info.cause, StopCause::MaxRetriesExceeded); + assert_eq!(info.stop_info.restart_count, 1); + + cleanup_box(&t, &box_id).await; +} + +// ============================================================================ +// RESTART POLICY: UnlessStopped +// ============================================================================ + +#[tokio::test] +async fn restart_policy_unless_stopped_restarts_on_crash() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::UnlessStopped)).await; + + // Start the box + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + // Verify running + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + let shim_pid = info.pid.expect("No shim PID found"); + + // Kill the shim + kill_process(shim_pid); + + // Should be running again + let info = expect_restarted(&t.runtime, &box_id, shim_pid).await; + assert_eq!( + info.status, + BoxStatus::Running, + "Expected box to be Running after auto-restart (UnlessStopped)" + ); + + cleanup_box(&t, &box_id).await; +} + +#[tokio::test] +async fn restart_policy_unless_stopped_stays_stopped_after_user_stop() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::UnlessStopped)).await; + + // Start the box + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + // Verify running + expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + + // User explicitly stops the box + t.bx.stop().await.expect("Failed to stop box"); + + let info = expect_status(&t.runtime, &box_id, BoxStatus::Stopped).await; + assert_eq!(info.stop_info.cause, StopCause::Normal); + + // Verify it stays stopped (UnlessStopped should NOT restart after user stop) + assert!( + wait_for_info( + &t.runtime, + &box_id, + Duration::from_secs(5), + "unexpected restart after user stop", + |info| info.status == BoxStatus::Running, + ) + .await + .is_none(), + "UnlessStopped should not restart a user-stopped box" + ); + + let info = expect_status(&t.runtime, &box_id, BoxStatus::Stopped).await; + assert_eq!( + info.status, + BoxStatus::Stopped, + "UnlessStopped should not restart a user-stopped box" + ); + + cleanup_box(&t, &box_id).await; +} + +#[tokio::test] +async fn restart_policy_unless_stopped_user_stop_race_stays_stopped() { + let t = BoxTestBase::with_options(restart_and_health_opts(RestartPolicy::UnlessStopped)).await; + + t.bx.start().await.expect("Failed to start box"); + let box_id = t.bx.id().clone(); + + let info = expect_status(&t.runtime, &box_id, BoxStatus::Running).await; + let original_pid = info.pid.expect("No shim PID found"); + + kill_process(original_pid); + + let stop = timeout(STATUS_WAIT_TIMEOUT, t.bx.stop()); + let crash_or_restart_observed = wait_for_info_fast( + &t.runtime, + &box_id, + STATUS_WAIT_TIMEOUT, + "crash or restart handling", + |info| { + matches!( + info.status, + BoxStatus::Crashed | BoxStatus::Restarting | BoxStatus::Stopped + ) || (info.status == BoxStatus::Running && info.pid != Some(original_pid)) + }, + ); + + let (stop_result, observed_info) = tokio::join!(stop, crash_or_restart_observed); + observed_info.unwrap_or_else(|| { + panic!( + "timed out after {:?} waiting for crash or restart handling", + STATUS_WAIT_TIMEOUT + ) + }); + stop_result + .expect("stop timed out while racing with crash handling") + .expect("stop failed while racing with crash handling"); + + let info = expect_status(&t.runtime, &box_id, BoxStatus::Stopped).await; + assert_eq!(info.stop_info.cause, StopCause::Normal); + assert!( + info.pid.is_none(), + "stopped box should not retain a shim PID" + ); + + assert!( + wait_for_info( + &t.runtime, + &box_id, + Duration::from_secs(5), + "unexpected restart after user stop raced with crash recovery", + |info| info.status == BoxStatus::Running, + ) + .await + .is_none(), + "UnlessStopped should not restart after user stop wins the race" + ); + + cleanup_box(&t, &box_id).await; +} From 9bc874e54bb9ae43e1c675d1bbf6d0d741702d63 Mon Sep 17 00:00:00 2001 From: Wenyu Huang Date: Mon, 11 May 2026 16:40:51 +0000 Subject: [PATCH 5/6] feat(sdk): expose restart policy options Expose restart policy configuration through the Python and Node SDK option types. Python gets a RestartPolicy class with no, always, on_failure, and unless_stopped constructors and AdvancedBoxOptions support. Node gets a JsRestartPolicy object shape and conversion into the Rust RestartPolicy model, including validation for on_failure maxRetries. Signed-off-by: Wenyu Huang --- sdks/node/src/lib.rs | 5 +- sdks/node/src/options.rs | 149 +++++++++++++++++++++++++++- sdks/python/boxlite/__init__.py | 4 + sdks/python/src/advanced_options.rs | 103 ++++++++++++++++++- sdks/python/src/lib.rs | 5 +- sdks/python/src/options.rs | 3 + 6 files changed, 263 insertions(+), 6 deletions(-) diff --git a/sdks/node/src/lib.rs b/sdks/node/src/lib.rs index 5683d8daa..4fb91a3bc 100644 --- a/sdks/node/src/lib.rs +++ b/sdks/node/src/lib.rs @@ -30,8 +30,9 @@ pub use info::{JsBoxInfo, JsBoxStateInfo, JsHealthState, JsHealthStatus}; pub use metrics::{JsBoxMetrics, JsRuntimeMetrics}; pub use network::{JsBoxConnection, JsBoxTunnel, JsNetworkHandle}; pub use options::{ - ApiKeyCredential, JsAccessToken, JsBoxOptions, JsEnvVar, JsHealthCheckOptions, JsImageRegistry, - JsImageRegistryAuth, JsNetworkSpec, JsOptions, JsPortSpec, JsSecret, JsVolumeSpec, + ApiKeyCredential, JsAccessToken, JsBoxOptions, JsBoxliteRestOptions, JsEnvVar, + JsHealthCheckOptions, JsImageRegistry, JsImageRegistryAuth, JsNetworkSpec, JsOptions, + JsPortSpec, JsRestartPolicy, JsSecret, JsVolumeSpec, }; pub use runtime::JsBoxlite; // re-export for dist bundling pub use snapshot_options::{JsCloneOptions, JsExportOptions, JsSnapshotOptions}; diff --git a/sdks/node/src/options.rs b/sdks/node/src/options.rs index ab47cc2c1..9dcde9159 100644 --- a/sdks/node/src/options.rs +++ b/sdks/node/src/options.rs @@ -1,7 +1,9 @@ use std::path::PathBuf; use std::time::Duration; -use boxlite::runtime::advanced_options::{AdvancedBoxOptions, HealthCheckOptions, SecurityOptions}; +use boxlite::runtime::advanced_options::{ + AdvancedBoxOptions, HealthCheckOptions, RestartPolicy, SecurityOptions, +}; use boxlite::runtime::constants::images; use boxlite::runtime::options::{ BoxOptions, BoxliteOptions, ImageRegistry, ImageRegistryAuth, NetworkConfig, NetworkMode, @@ -12,6 +14,46 @@ use napi_derive::napi; use crate::advanced_options::JsSecurityOptions; +/// Restart policy for automatic restart on crash. +/// +/// Similar to Docker's restart policy. Controls what happens when a box's +/// shim process crashes. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct JsRestartPolicy { + /// Policy type: "no", "always", "on_failure", or "unless_stopped" + #[napi(js_name = "type")] + pub type_: String, + + /// Maximum retries for "on_failure" policy. + #[napi(js_name = "maxRetries")] + pub max_retries: Option, +} + +impl TryFrom for RestartPolicy { + type Error = boxlite_shared::errors::BoxliteError; + + fn try_from(js_policy: JsRestartPolicy) -> Result { + match js_policy.type_.as_str() { + "no" => Ok(RestartPolicy::No), + "always" => Ok(RestartPolicy::Always), + "on_failure" => { + let max_retries = js_policy.max_retries.ok_or_else(|| { + boxlite_shared::errors::BoxliteError::Config( + "on_failure restart policy requires maxRetries".into(), + ) + })?; + Ok(RestartPolicy::OnFailure { max_retries }) + } + "unless_stopped" => Ok(RestartPolicy::UnlessStopped), + _ => Err(boxlite_shared::errors::BoxliteError::Config(format!( + "invalid restart policy type: {}", + js_policy.type_ + ))), + } + } +} + /// Health check options for boxes. /// /// Defines how to periodically check if a box's guest agent is responsive. @@ -236,6 +278,10 @@ pub struct JsBoxOptions { #[napi(js_name = "healthCheck")] pub health_check: Option, + /// Restart policy for automatic restart on crash. + #[napi(js_name = "restartPolicy")] + pub restart_policy: Option, + /// Secrets to inject into outbound HTTPS requests via MITM proxy. pub secrets: Option>, } @@ -408,6 +454,10 @@ impl TryFrom for BoxOptions { .unwrap_or_default(); let health_check = js_opts.health_check.map(HealthCheckOptions::from); + let restart_policy = js_opts + .restart_policy + .map(RestartPolicy::try_from) + .transpose()?; let secrets = js_opts .secrets .unwrap_or_default() @@ -436,6 +486,7 @@ impl TryFrom for BoxOptions { advanced: AdvancedBoxOptions { security, health_check, + restart_policy, ..Default::default() }, auto_pause: js_opts.auto_pause, @@ -735,6 +786,7 @@ mod tests { user: None, security: None, health_check: None, + restart_policy: None, secrets: None, }; @@ -779,6 +831,7 @@ mod tests { user: None, security: None, health_check: None, + restart_policy: None, secrets: Some(vec![JsSecret { name: "openai".into(), value: "sk-test".into(), @@ -794,6 +847,100 @@ mod tests { assert_eq!(opts.secrets[0].placeholder, ""); } + #[test] + fn restart_policy_no() { + let js = JsRestartPolicy { + type_: "no".into(), + max_retries: None, + }; + let policy = RestartPolicy::try_from(js).unwrap(); + assert_eq!(policy, RestartPolicy::No); + } + + #[test] + fn restart_policy_always() { + let js = JsRestartPolicy { + type_: "always".into(), + max_retries: None, + }; + let policy = RestartPolicy::try_from(js).unwrap(); + assert_eq!(policy, RestartPolicy::Always); + } + + #[test] + fn restart_policy_on_failure() { + let js = JsRestartPolicy { + type_: "on_failure".into(), + max_retries: Some(3), + }; + let policy = RestartPolicy::try_from(js).unwrap(); + assert_eq!(policy, RestartPolicy::OnFailure { max_retries: 3 }); + } + + #[test] + fn restart_policy_on_failure_missing_max_retries() { + let js = JsRestartPolicy { + type_: "on_failure".into(), + max_retries: None, + }; + let err = RestartPolicy::try_from(js).unwrap_err(); + assert!(err.to_string().contains("maxRetries")); + } + + #[test] + fn restart_policy_unless_stopped() { + let js = JsRestartPolicy { + type_: "unless_stopped".into(), + max_retries: None, + }; + let policy = RestartPolicy::try_from(js).unwrap(); + assert_eq!(policy, RestartPolicy::UnlessStopped); + } + + #[test] + fn restart_policy_invalid_type() { + let js = JsRestartPolicy { + type_: "invalid".into(), + max_retries: None, + }; + let err = RestartPolicy::try_from(js).unwrap_err(); + assert!(err.to_string().contains("invalid")); + } + + #[test] + fn box_options_from_js_restart_policy() { + let js = JsBoxOptions { + image: Some("alpine:latest".into()), + rootfs_path: None, + cpus: None, + memory_mib: None, + disk_size_gb: None, + working_dir: None, + env: None, + volumes: None, + network: None, + ports: None, + auto_remove: None, + auto_pause: None, + auto_delete: None, + auto_resume: None, + detach: None, + entrypoint: None, + cmd: None, + user: None, + security: None, + health_check: None, + restart_policy: Some(JsRestartPolicy { + type_: "always".into(), + max_retries: None, + }), + secrets: None, + }; + + let opts = BoxOptions::try_from(js).unwrap(); + assert_eq!(opts.advanced.restart_policy, Some(RestartPolicy::Always)); + } + #[test] fn disabled_network_rejects_allow_net() { let err = NetworkSpec::try_from(JsNetworkSpec { diff --git a/sdks/python/boxlite/__init__.py b/sdks/python/boxlite/__init__.py index 786809ca9..3c02f3cfe 100644 --- a/sdks/python/boxlite/__init__.py +++ b/sdks/python/boxlite/__init__.py @@ -11,6 +11,7 @@ from .boxlite import ( AccessToken, ApiKeyCredential, + AdvancedBoxOptions, Box, BoxInfo, Boxlite, @@ -34,6 +35,7 @@ NetworkHandle, NetworkSpec, Options, + RestartPolicy, RuntimeMetrics, Secret, SecurityOptions, @@ -49,6 +51,7 @@ "Options", "ImageRegistry", "BoxOptions", + "AdvancedBoxOptions", "BoxliteRestOptions", "ApiKeyCredential", "AccessToken", @@ -70,6 +73,7 @@ "BoxMetrics", "CopyOptions", "HealthCheckOptions", + "RestartPolicy", "SecurityOptions", "Secret", "SnapshotHandle", diff --git a/sdks/python/src/advanced_options.rs b/sdks/python/src/advanced_options.rs index acaa025d7..63d7ca1d8 100644 --- a/sdks/python/src/advanced_options.rs +++ b/sdks/python/src/advanced_options.rs @@ -1,4 +1,6 @@ -use boxlite::runtime::advanced_options::{HealthCheckOptions, ResourceLimits, SecurityOptions}; +use boxlite::runtime::advanced_options::{ + HealthCheckOptions, ResourceLimits, RestartPolicy, SecurityOptions, +}; use pyo3::prelude::*; // ============================================================================ @@ -249,6 +251,97 @@ impl From for SecurityOptions { } } +// ============================================================================ +// Restart Policy +// ============================================================================ + +/// Restart policy for automatic restart on crash. +/// +/// Similar to Docker's restart policy. Controls what happens when a box's +/// shim process crashes. +/// +/// # Variants +/// - `RestartPolicy.no()` - Never restart (default) +/// - `RestartPolicy.always()` - Always restart regardless of exit status +/// - `RestartPolicy.on_failure(max_retries)` - Restart only on non-zero exit code +/// - `RestartPolicy.unless_stopped()` - Always restart unless user explicitly called stop() +/// +/// # Example +/// ```python +/// from boxlite import RestartPolicy, AdvancedBoxOptions +/// +/// # Always restart +/// policy = RestartPolicy.always() +/// +/// # Restart on failure, max 3 retries +/// policy = RestartPolicy.on_failure(max_retries=3) +/// +/// opts = AdvancedBoxOptions(restart_policy=policy) +/// ``` +#[pyclass(name = "RestartPolicy")] +#[derive(Clone, Debug)] +pub struct PyRestartPolicy { + inner: RestartPolicy, +} + +#[pymethods] +impl PyRestartPolicy { + /// Never restart (default). + #[staticmethod] + fn no() -> Self { + Self { + inner: RestartPolicy::No, + } + } + + /// Always restart regardless of exit status. + /// Unlimited retries with exponential backoff. + #[staticmethod] + fn always() -> Self { + Self { + inner: RestartPolicy::Always, + } + } + + /// Restart only on non-zero exit code. + /// + /// Args: + /// max_retries: Maximum consecutive restart attempts before giving up + #[staticmethod] + #[pyo3(signature = (max_retries))] + fn on_failure(max_retries: u32) -> Self { + Self { + inner: RestartPolicy::OnFailure { max_retries }, + } + } + + /// Always restart unless user explicitly called stop(). + /// Unlimited retries. + #[staticmethod] + fn unless_stopped() -> Self { + Self { + inner: RestartPolicy::UnlessStopped, + } + } + + fn __repr__(&self) -> String { + match &self.inner { + RestartPolicy::No => "RestartPolicy.no()".to_string(), + RestartPolicy::Always => "RestartPolicy.always()".to_string(), + RestartPolicy::OnFailure { max_retries } => { + format!("RestartPolicy.on_failure(max_retries={})", max_retries) + } + RestartPolicy::UnlessStopped => "RestartPolicy.unless_stopped()".to_string(), + } + } +} + +impl From for RestartPolicy { + fn from(py_policy: PyRestartPolicy) -> Self { + py_policy.inner + } +} + // ============================================================================ // Advanced Options // ============================================================================ @@ -266,19 +359,25 @@ pub struct PyAdvancedBoxOptions { /// Health check options. #[pyo3(get, set)] pub health_check: Option, + + /// Restart policy for automatic restart on crash. + #[pyo3(get, set)] + pub restart_policy: Option, } #[pymethods] impl PyAdvancedBoxOptions { #[new] - #[pyo3(signature = (security=None, health_check=None))] + #[pyo3(signature = (security=None, health_check=None, restart_policy=None))] fn new( security: Option, health_check: Option, + restart_policy: Option, ) -> Self { Self { security, health_check, + restart_policy, } } } diff --git a/sdks/python/src/lib.rs b/sdks/python/src/lib.rs index 6ff730b17..665bf96b1 100644 --- a/sdks/python/src/lib.rs +++ b/sdks/python/src/lib.rs @@ -14,7 +14,9 @@ mod snapshots; mod util; mod volumes; -use crate::advanced_options::{PyAdvancedBoxOptions, PyHealthCheckOptions, PySecurityOptions}; +use crate::advanced_options::{ + PyAdvancedBoxOptions, PyHealthCheckOptions, PyRestartPolicy, PySecurityOptions, +}; use crate::box_handle::PyBox; use crate::exec::{PyExecStderr, PyExecStdin, PyExecStdout, PyExecution}; use crate::images::{PyImageHandle, PyImageInfo, PyImagePullResult}; @@ -39,6 +41,7 @@ fn boxlite_python(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/sdks/python/src/options.rs b/sdks/python/src/options.rs index 9518650c6..c80b0eca3 100644 --- a/sdks/python/src/options.rs +++ b/sdks/python/src/options.rs @@ -564,6 +564,9 @@ impl TryFrom for BoxOptions { if let Some(health_check) = advanced.health_check { opts.advanced.health_check = Some(HealthCheckOptions::from(health_check)); } + if let Some(restart_policy) = advanced.restart_policy { + opts.advanced.restart_policy = Some(restart_policy.into()); + } } // Convert Python secrets to Rust secrets From 19a42edd2e5b7d13f165e48aaaa8c39bf216a7da Mon Sep 17 00:00:00 2001 From: Wenyu Huang Date: Mon, 11 May 2026 16:42:01 +0000 Subject: [PATCH 6/6] docs(restart): document runtime restart policies Document the runtime crash-restart architecture, policy behavior, state model, backoff behavior, and SDK configuration examples. The document explicitly scopes this phase to in-process runtime crash handling. Startup-time recovery of persisted crashed boxes is deferred. Signed-off-by: Wenyu Huang --- docs/development/restart.md | 269 ++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 docs/development/restart.md diff --git a/docs/development/restart.md b/docs/development/restart.md new file mode 100644 index 000000000..898a1e0da --- /dev/null +++ b/docs/development/restart.md @@ -0,0 +1,269 @@ +# Box Restart Policies + +BoxLite supports restart policies for automatic recovery when a running Box VM +crashes while the embedding process is alive. BoxLite is an embedded library, not +a daemon, so crash monitoring runs inside the user process and stops when that +process exits. + +This document describes the current in-process crash-restart path coordinated by +the runtime crash coordinator. Startup-time auto-restart of persisted crashed +boxes is intentionally out of scope for this phase. + +## Architecture + +Restart policy is implemented as an in-process crash-recovery pipeline. Health +checks detect shim process death and report it to the runtime crash coordinator; +the coordinator owns crash state updates, restart-policy evaluation, backoff, +and VM rebuilds. + +```text +BoxImpl health check task + | + | shim process died + v +mpsc::Sender + | + v +Runtime crash coordinator + | + | dedupe by BoxID + | record Crashed state and exit metadata + | evaluate RestartPolicy + | wait with backoff + v +RuntimeImpl::restart(expected_epoch) + | + v +fresh BoxImpl swapped into stable BoxHandle + | + | start() + v +fresh BoxImpl reaches Running + | + v +coordinator rechecks lifecycle epoch and Running status + | + v +live StopInfo reset and the same snapshot persisted +``` + +## Runtime Flow + +1. A Box starts with a health-check task when health checks are configured or + auto-enabled by a restart policy. +2. The health-check task periodically pings the guest. +3. If the ping fails, the health-check task checks whether the shim process is + still alive. +4. If the shim is alive, the task records health-check failure state. Guest + unresponsiveness alone does not trigger restart policy. +5. If the shim process died, the task sends the Box ID to the runtime crash + coordinator and exits. +6. The coordinator deduplicates notifications by Box ID. +7. The coordinator records the Box as `Crashed`, stores exit metadata, and + evaluates the configured restart policy. +8. If restart is denied, the coordinator marks the Box `Stopped` with the + appropriate `StopCause`. +9. If restart is allowed, the coordinator waits with exponential backoff and + calls `RuntimeImpl::restart()` with the expected lifecycle epoch. +10. `restart()` transitions the Box through `Restarting`, swaps a fresh + `BoxImpl` into the stable `BoxHandle`, and then starts it. +11. After `restart()` returns successfully, the coordinator rechecks the + lifecycle epoch and the swapped-in `BoxImpl`'s `Running` status. It resets + the live stop info and persists the same state snapshot. + +Existing `LiteBox` values continue to work after a successful restart because +they point to the stable handle rather than directly to the old VM implementation. + +## Detached Boxes + +`detach=true` changes the Box lifetime, not the monitoring model. A detached +Box is skipped by runtime shutdown and can keep running after the embedding +process exits. The health-check task and crash coordinator still live inside the +embedding process. + +After a runtime restart, startup recovery reads the PID file and can mark a live +detached Box as `Running`. It does not reconnect to the guest or start a new +health-check task at that point. Monitoring and restart policy resume after a +control-plane operation reattaches to the Box and initializes `LiveState`, such +as `exec()`. + +This means `detach=true` plus a restart policy does not create daemon-style +self-healing while no BoxLite runtime is alive. It only restarts detected +crashes while a runtime is attached and monitoring the Box. + +## Crash Coordinator + +The crash coordinator is one background task per runtime. It owns: + +- `mpsc::Receiver` for crash notifications. +- `HashSet` for per-box de-duplication. +- `JoinSet` for supervised crash/restart tasks. +- A task-ID map so completion, cancellation, and panic all release per-box + de-duplication state. +- `Weak` plus cooperative-shutdown and forced-cancellation tokens. + +Different boxes can still be handled concurrently. The coordinator spawns and +supervises each crash handler, then removes a Box ID from the pending set when +the task completes, is cancelled, or panics. Tasks are never detached from the +coordinator. + +The coordinator task continues polling for new crash notifications while a box's +restart task waits in backoff. Crash handlers perform their synchronous database +and small per-box artifact operations inside the spawned task. Tasks hold only a +weak runtime reference and temporarily upgrade it when they need to read state, +write state, or call `restart()`. + +Shutdown cancels the runtime token and stops accepting new crash notifications. +For a finite shutdown deadline, the coordinator gets up to five seconds to drain +cooperatively, bounded by the remaining shutdown time. If that grace period +expires, shutdown requests forced cancellation and stops waiting. The +coordinator aborts its supervised crash tasks when it next runs. Its handle +remains tracked so a later shutdown call can reap it. `shutdown(Some(-1))` waits +indefinitely and does not request forced cancellation. Synchronous work already +executing cannot be interrupted until it returns control to Tokio. + +## Restart Policy Semantics + +| Policy | Restart condition | Retry limit | +|--------|-------------------|-------------| +| `No` | Never restart after a crash. | N/A | +| `Always` | Restart after detected crashes. Manual stop is respected. | Unlimited | +| `OnFailure { max_retries }` | Restart when the exit code is non-zero or unknown, while the current retry count is below `max_retries`. | `max_retries` | +| `UnlessStopped` | Restart after detected crashes. Manual stop is respected because stale crash work cannot commit after lifecycle epoch changes. | Unlimited | + +When a restart policy is set without a health check, BoxLite enables a default +health check so shim process death can be detected: + +| Field | Default | +|-------|---------| +| `interval` | 5s | +| `timeout` | 10s | +| `retries` | 3 | +| `start_period` | 60s | + +## State Model + +Restart adds two runtime statuses: + +- `Crashed`: the shim process died and the runtime has recorded crash metadata. +- `Restarting`: the runtime is rebuilding the VM after a crash. + +```text +[Configured] --start()--> [Running] --stop()--> [Stopped] + | | ^ + | | shim died | + | v | + | [Crashed]--denied-----+ + | | + | restart allowed + | v + +------------------[Restarting]--success--> [Running] + | + | cooperative shutdown / max retries / + | restart failed + v + [Stopped] +``` + +`StopInfo` stores the stop cause, exit code, exit time, restart count, and last +successful restart time. `last_restart_error` stores the most recent failed +restart attempt, if any. + +Forced cancellation after the shutdown deadline can interrupt a crash task +before its final state commit. In that case, the database keeps the last state +that the task committed, such as `Crashed`, `Restarting`, or `Running`. + +| Scenario | Final status | Stop cause | +|----------|--------------|------------| +| No policy or `RestartPolicy::No` | `Stopped` | `CrashedNoPolicy` | +| `OnFailure` with exit code `0` | `Stopped` | `Normal` | +| `OnFailure` retries exhausted | `Stopped` | `MaxRetriesExceeded` | +| Restart attempt failed but more retries remain | `Crashed` / `Restarting` | `RestartFailed` | +| Cooperative runtime shutdown during backoff | `Stopped` | `Normal` | +| Forced cancellation after the shutdown deadline | Last committed status | Last committed value | +| Successful restart | `Running` | stop info reset, `restarted_at` set | + +## Backoff And Stale Restart Protection + +Restart attempts use exponential backoff: + +```text +100ms, 200ms, 400ms, 800ms, 1.6s, ... capped at 30s +``` + +Before committing an automatic restart, the crash path re-reads state. If a user +manually stopped, removed, or restarted the Box during backoff, the stale crash +work exits instead of overwriting the user's newer lifecycle operation. + +Crash handling records the expected `lifecycle_epoch` when it first observes the +crash. Under the per-Box lifecycle lock, `restart()` rechecks the database and +starts a new VM only if the Box remains `Crashed` or `Restarting` at that epoch. +After the new `BoxImpl` reaches `Running`, the coordinator takes the same lock +and rechecks the live state at the same epoch before it resets `StopInfo`. It +updates the swapped-in `BoxImpl` first and persists the same snapshot so +existing `LiteBox` handles do not retain stale stop metadata. + +## Startup Recovery Scope + +`RuntimeImpl::new()` runs `recover_boxes()` to make persisted state consistent +before the runtime accepts new operations. This path cleans up stale process +state, reclaims per-box locks, recovers interrupted local snapshot operations, +and marks boxes whose verified or legacy shim PID is still alive as `Running`. +It does not reconnect to the guest or initialize `LiveState`. + +Startup recovery does not evaluate restart policy or queue automatic restarts +for boxes that crashed while the embedding process was down. If no live shim +exists, recovery converts an interrupted `Restarting` state to `Stopped` with +`RestartFailed`; a valid crash report can instead produce `Failed`. A persisted +`Crashed` state is not startable until an explicit `stop()` acknowledges the +crash and moves the Box to `Stopped`. + +## API Examples + +Rust: + +```rust +use boxlite::runtime::advanced_options::{AdvancedBoxOptions, RestartPolicy}; +use boxlite::runtime::options::BoxOptions; + +let options = BoxOptions { + advanced: AdvancedBoxOptions { + restart_policy: Some(RestartPolicy::OnFailure { max_retries: 3 }), + ..Default::default() + }, + ..Default::default() +}; +``` + +Python: + +```python +from boxlite import AdvancedBoxOptions, BoxOptions, RestartPolicy + +options = BoxOptions( + image="alpine:latest", + advanced=AdvancedBoxOptions( + restart_policy=RestartPolicy.on_failure(max_retries=3), + ), +) +``` + +Node: + +```ts +const box = await runtime.create({ + image: "alpine:latest", + restartPolicy: { type: "on_failure", maxRetries: 3 }, +}); +``` + +## Current Limits + +- Restart detection is in-process. If the embedding process exits, health checks + and the crash coordinator stop. +- Startup-time evaluation of persisted crashed boxes is not included in this + phase. +- Guest health-check failure only marks health state. It does not trigger + restart policy unless the shim process is dead. +- Manual `start()` starts a stopped Box directly and does not evaluate restart + policy.