feat(pool): decay warm-process refill target after idle TTL

config/default.toml

@@ -298,6 +298,10 @@ enabled = true
 startup_prewarm = true
 # Maximum concurrent process creations in one refill batch.
 fill_concurrency = 4
+# Seconds without warm-process acquisitions before the pool decays its refill
+# target back to the low watermark and drains the excess warm processes
+# (0 = never decay).
+idle_ttl_secs = 600
 
 [memory_snapshot]
 # Generated OverlayBD config dedicated to memory snapshot devices.

docs/src/configuration/reference.md

@@ -294,6 +294,7 @@ Component sections:
 | `[pool.firecracker]` | `maintenance_enabled` | boolean | `true` | Enable the background Firecracker process maintenance worker |
 | `[pool.firecracker]` | `startup_prewarm` | boolean | `true` | Spawn warm Firecracker entries up to the low watermark during server startup |
 | `[pool.firecracker]` | `fill_concurrency` | integer | `4` | Maximum number of warm Firecracker processes created concurrently by one maintenance refill batch |
+| `[pool.firecracker]` | `idle_ttl_secs` | integer | `600` | Seconds without warm-process acquisitions before the refill target decays back to the low watermark and excess warm processes are drained (`0` disables decay) |
 
 Validation rules:
 

src/cfg.rs

@@ -1,5 +1,6 @@
 use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
+use std::time::Duration;
 
 pub(crate) mod image;
 pub(crate) mod network;
@@ -215,6 +216,12 @@ pub struct FirecrackerProcessPoolConfig {
     pub startup_prewarm: bool,
     #[config(default = 4usize)]
     pub fill_concurrency: usize,
+    /// Seconds without pool acquisitions before the geometric fill target
+    /// decays back to the low watermark and excess warm processes are drained.
+    /// 0 disables decay and keeps the fill target ratcheted for the process
+    /// lifetime.
+    #[config(default = 600u64)]
+    pub idle_ttl_secs: u64,
 }
 
 #[derive(Debug, Clone)]
@@ -764,6 +771,9 @@ impl AppConfig {
             high_watermark: self.pool.high_watermark,
             maintenance_enabled: pool.enabled && pool.maintenance_enabled,
             startup_prewarm: pool.startup_prewarm,
+            // The network slot pool holds no processes; keep its historical
+            // ratchet behavior without idle decay.
+            idle_ttl: None,
         }
     }
 
@@ -781,6 +791,7 @@ impl AppConfig {
             // reusable device shape is image/size dependent.
             maintenance_enabled: false,
             startup_prewarm: pool.startup_prewarm,
+            idle_ttl: None,
         })
     }
 
@@ -797,6 +808,7 @@ impl AppConfig {
                 high_watermark: self.pool.high_watermark,
                 maintenance_enabled: pool.maintenance_enabled,
                 startup_prewarm: pool.startup_prewarm,
+                idle_ttl: (pool.idle_ttl_secs > 0).then(|| Duration::from_secs(pool.idle_ttl_secs)),
             },
             fill_concurrency: pool.fill_concurrency,
         })

src/sandbox/firecracker/pool.rs

@@ -228,13 +228,16 @@ impl FirecrackerPool {
     }
 
     fn run_maintenance_cycle(&self) -> Result<()> {
-        match self.pool.compute_maintenance_action(self.pool.len()) {
+        match self.pool.compute_maintenance_action() {
             PoolMaintenanceAction::Fill(to_fill) => {
                 self.runtime.block_on(self.fill_warm_entries(to_fill))?;
             }
             PoolMaintenanceAction::Drain(to_drain) => {
                 for _ in 0..to_drain {
-                    let Some(warm) = self.pool.try_drain_one() else {
+                    // Re-validate each removal at execution time: the computed
+                    // drain count can be stale if an acquisition cancelled an
+                    // in-progress decay or the pool length changed.
+                    let Some(warm) = self.pool.try_drain_one_for_maintenance() else {
                         break;
                     };
                     self.cleanup_warm_blocking(warm, false)?;

src/sandbox/network/manager.rs

@@ -115,6 +115,7 @@ impl NetworkManager {
                 high_watermark,
                 maintenance_enabled,
                 startup_prewarm: true,
+                idle_ttl: None,
             },
             address_plan: NetworkAddressPlan::default(),
             netns_dir: std::env::temp_dir().join("aenv-network-tests/netns"),
@@ -287,12 +288,12 @@ impl NetworkManager {
     }
 
     #[cfg(test)]
-    fn compute_maintenance_action(&self, pool_len: usize) -> PoolMaintenanceAction {
-        self.pool.compute_maintenance_action(pool_len)
+    fn compute_maintenance_action(&self) -> PoolMaintenanceAction {
+        self.pool.compute_maintenance_action()
     }
 
     fn run_pool_maintenance_cycle(&self) -> Result<()> {
-        let action = self.pool.compute_maintenance_action(self.pool.len());
+        let action = self.pool.compute_maintenance_action();
 
         match action {
             PoolMaintenanceAction::Fill(to_fill) => {
@@ -326,7 +327,10 @@ impl NetworkManager {
             }
             PoolMaintenanceAction::Drain(to_drain) => {
                 for _ in 0..to_drain {
-                    let maybe_slot = self.pool.try_drain_one();
+                    // Re-validate each removal at execution time: the computed
+                    // drain count can be stale if an acquisition cancelled an
+                    // in-progress decay or the pool length changed.
+                    let maybe_slot = self.pool.try_drain_one_for_maintenance();
                     let Some(slot) = maybe_slot else {
                         break;
                     };
@@ -897,25 +901,44 @@ mod tests {
     #[test]
     fn maintenance_action_fills_to_current_target() {
         let manager = NetworkManager::new(true, 4, 10);
+        // Two warm slots below the low watermark: fill the missing two.
+        for idx in 1..=2u32 {
+            manager.allocated.insert(idx as usize);
+            manager.pool.release(test_slot(idx)).unwrap();
+        }
         assert_eq!(
-            manager.compute_maintenance_action(2),
+            manager.compute_maintenance_action(),
             PoolMaintenanceAction::Fill(2)
         );
+        // Seven warm slots within the watermarks: no action.
+        for idx in 3..=7u32 {
+            manager.allocated.insert(idx as usize);
+            manager.pool.release(test_slot(idx)).unwrap();
+        }
         assert_eq!(
-            manager.compute_maintenance_action(7),
+            manager.compute_maintenance_action(),
             PoolMaintenanceAction::Idle
         );
     }
 
     #[test]
     fn maintenance_action_drains_above_high_watermark() {
         let manager = NetworkManager::new(true, 2, 4);
+        // Eight warm slots above the high watermark: drain the excess.
+        for idx in 1..=8u32 {
+            manager.allocated.insert(idx as usize);
+            manager.pool.release(test_slot(idx)).unwrap();
+        }
         assert_eq!(
-            manager.compute_maintenance_action(8),
+            manager.compute_maintenance_action(),
             PoolMaintenanceAction::Drain(4)
         );
+        // At the high watermark: no action.
+        for _ in 0..4 {
+            assert!(manager.pool.try_drain_one().is_some());
+        }
         assert_eq!(
-            manager.compute_maintenance_action(4),
+            manager.compute_maintenance_action(),
             PoolMaintenanceAction::Idle
         );
     }

storage/ublk-daemon/src/main.rs

@@ -141,6 +141,7 @@ fn load_pool_config(
             .startup_prewarm
             .or_else(|| pool.and_then(|pool| pool.startup_prewarm))
             .unwrap_or(true),
+        idle_ttl: None,
     }))
 }
 
@@ -232,6 +233,7 @@ fn default_pool_config() -> warm_pool::PoolConfig {
         high_watermark: 64,
         maintenance_enabled: false,
         startup_prewarm: true,
+        idle_ttl: None,
     }
 }
 

storage/ublk-daemon/src/server.rs

@@ -1442,7 +1442,7 @@ fn schedule_idle_pool_refill(
         pool.refill_inflight.store(false, Ordering::Release);
 
         if matches!(
-            pool.idle.compute_maintenance_action(pool.idle.len()),
+            pool.idle.compute_maintenance_action(),
             PoolMaintenanceAction::Fill(_)
         ) {
             schedule_idle_pool_refill(pool, ctrl_ring, virtual_size);
@@ -1455,9 +1455,7 @@ async fn refill_idle_pool(
     ctrl_ring: IoRingHandle<io_uring::squeue::Entry128>,
     virtual_size: u64,
 ) -> Result<()> {
-    let PoolMaintenanceAction::Fill(to_create) =
-        pool.idle.compute_maintenance_action(pool.idle.len())
-    else {
+    let PoolMaintenanceAction::Fill(to_create) = pool.idle.compute_maintenance_action() else {
         return Ok(());
     };
 
@@ -1658,6 +1656,7 @@ mod tests {
             high_watermark: 1,
             maintenance_enabled: false,
             startup_prewarm: false,
+            idle_ttl: None,
         }
     }