From c14f48f0841580dfd2cce81960218918da23ebbe Mon Sep 17 00:00:00 2001 From: Mathieu JESER <6737167+bobdivx@users.noreply.github.com> Date: Wed, 21 Oct 2026 21:43:38 +0200 Subject: [PATCH] feat(admin): automatic Docker cleanup on every node - Scheduler on the leader (default: daily 03:00 Europe/Paris), emergency cleanup when a node's Docker disk exceeds the alert threshold (85%). - Build cache: buildx prune per builder + buildctl prune inside running buildx_buildkit_* containers, older than the retention (24h). - Images: unused images older than the retention, old deployment images beyond the N kept per project (3). Never images used by a container (running or stopped), never df-*:latest, never volumes/containers, never image rm -f. - Each successful deploy tags df-:r so rollback images survive. - Admin page: tiles (auto switch, clean now with dry-run + confirmation, history, log rotation, per-node disk), details in modals. - Optional --log-opt rotation for DevForge-launched containers; daemon.json never edited (snippet shown). - Retires the broken legacy /etc/cron.d/devforge-disk-guard when the host is visible; DOCKER_CONFIG fallback for read-only ~/.docker. --- apps/server/src/docker_gc.rs | 716 +++++++++++++++ apps/server/src/docker_gc_routes.rs | 616 +++++++++++++ apps/server/src/docker_gc_tests.rs | 348 ++++++++ apps/server/src/main.rs | 4 + apps/web/src/components/AdminPage.tsx | 44 +- .../web/src/components/DockerGcPanel.test.tsx | 12 + apps/web/src/components/DockerGcPanel.tsx | 835 ++++++++++++++++++ apps/web/src/components/ui/HubTile.tsx | 17 + apps/web/src/lib/api.ts | 111 +++ crates/deploy/src/docker.rs | 42 +- crates/deploy/src/lib.rs | 44 +- docs/DOCKER_GC.md | 42 + 12 files changed, 2826 insertions(+), 5 deletions(-) create mode 100644 apps/server/src/docker_gc.rs create mode 100644 apps/server/src/docker_gc_routes.rs create mode 100644 apps/server/src/docker_gc_tests.rs create mode 100644 apps/web/src/components/DockerGcPanel.test.tsx create mode 100644 apps/web/src/components/DockerGcPanel.tsx create mode 100644 docs/DOCKER_GC.md diff --git a/apps/server/src/docker_gc.rs b/apps/server/src/docker_gc.rs new file mode 100644 index 0000000000..3ecb80a157 --- /dev/null +++ b/apps/server/src/docker_gc.rs @@ -0,0 +1,716 @@ +//! Nettoyage Docker automatique (cache de build, images inutilisées, anciennes +//! images de déploiement) sur chaque nœud du cluster. +//! +//! Ce module ne contient que la logique pure (réglages, scripts shell, parsing, +//! sélection) ; l'exécution, la planification et l'API sont dans +//! `docker_gc_routes.rs`. +//! +//! Garanties (testées) : +//! - jamais de `docker volume …`, `container prune` ni `system prune` ; +//! - jamais `docker image rm -f` : Docker refuse donc de supprimer une image +//! utilisée par un conteneur, même arrêté (filet de sécurité en plus de la +//! sélection) ; +//! - une image utilisée par un conteneur (en cours ou arrêté) n'est jamais +//! sélectionnée, ni `df-…:latest` (image du déploiement courant) ; +//! - par projet DevForge (`df-`), les N images les plus récentes sont gardées +//! pour un rollback. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_TIMEZONE: &str = "Europe/Paris"; + +/// Réglages (stockés en JSON dans `docker_gc_settings`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct GcSettings { + pub enabled: bool, + /// Fréquence en heures (diviseur de 24, ou multiple de 24 jusqu'à 168). + pub frequency_hours: u32, + /// Heure locale du premier passage (0–23). + pub hour: u32, + /// Expression cron (5 champs) : remplace fréquence + heure si renseignée. + pub cron: Option, + pub timezone: String, + /// Âge minimal du cache de build et des images inutilisées supprimés. + pub retention_hours: u32, + /// Images gardées par projet pour un rollback. + pub keep_per_project: u32, + /// Seuil d'occupation disque (%) qui déclenche un nettoyage d'urgence. + pub disk_alert_percent: u32, + /// Rotation des logs des conteneurs lancés par DevForge (`--log-opt`). + pub log_rotation: bool, + pub log_max_size_mb: u32, + pub log_max_files: u32, +} + +impl Default for GcSettings { + fn default() -> Self { + Self { + enabled: true, + frequency_hours: 24, + hour: 3, + cron: None, + timezone: DEFAULT_TIMEZONE.into(), + retention_hours: 24, + keep_per_project: 3, + disk_alert_percent: 85, + log_rotation: false, + log_max_size_mb: 10, + log_max_files: 3, + } + } +} + +impl GcSettings { + pub fn validate(&self) -> Result<(), String> { + let f = self.frequency_hours; + if f == 0 || f > 168 || (24 % f != 0 && f % 24 != 0) { + return Err( + "Fréquence : 1, 2, 3, 4, 6, 8, 12 h ou un multiple de 24 h (max 168 h).".into(), + ); + } + if self.hour > 23 { + return Err("Heure : entre 0 et 23.".into()); + } + if !devforge_cron::timezone_is_known(&self.timezone) { + return Err(format!("Fuseau inconnu : {}", self.timezone)); + } + if let Some(c) = self + .cron + .as_deref() + .map(str::trim) + .filter(|c| !c.is_empty()) + { + if !devforge_cron::is_five_fields(c) { + return Err("Cron : 5 champs (minute heure jour mois jour-semaine).".into()); + } + devforge_cron::validate_cron_expression(c)?; + if devforge_cron::min_interval_minutes(c).is_some_and(|m| m < 60) { + return Err("Cron : au plus un passage par heure.".into()); + } + } + if !(1..=720).contains(&self.retention_hours) { + return Err("Rétention : entre 1 et 720 h.".into()); + } + if !(1..=20).contains(&self.keep_per_project) { + return Err("Images gardées par projet : entre 1 et 20.".into()); + } + if !(50..=99).contains(&self.disk_alert_percent) { + return Err("Seuil d'alerte disque : entre 50 et 99 %.".into()); + } + if !(1..=1024).contains(&self.log_max_size_mb) || !(1..=20).contains(&self.log_max_files) { + return Err("Logs : taille 1–1024 Mo, 1–20 fichiers.".into()); + } + Ok(()) + } + + /// Expression cron effective (5 champs, fuseau [`Self::timezone`]). + pub fn cron_expression(&self) -> String { + if let Some(c) = self + .cron + .as_deref() + .map(str::trim) + .filter(|c| !c.is_empty()) + { + return c.to_string(); + } + let h = self.hour.min(23); + let f = self.frequency_hours.clamp(1, 168); + if f < 24 && 24 % f == 0 { + let mut hours: Vec = (0..24 / f).map(|k| (h + k * f) % 24).collect(); + hours.sort_unstable(); + let list: Vec = hours.iter().map(u32::to_string).collect(); + return format!("0 {} * * *", list.join(",")); + } + match f / 24 { + 0 | 1 => format!("0 {h} * * *"), + 7 => format!("0 {h} * * 1"), + d => format!("0 {h} */{d} * *"), + } + } + + pub fn next_run(&self) -> Option> { + devforge_cron::next_run_time(&self.cron_expression(), Some(&self.timezone)) + } +} + +// ───────────────────────── Inventaire ───────────────────────── + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ImageInfo { + pub id: String, + pub created: DateTime, + pub size: u64, + pub tags: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct ContainerInfo { + pub id: String, + pub image_id: String, + pub state: String, + pub name: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize)] +pub struct DiskInfo { + pub root_dir: String, + pub logging_driver: String, + pub docker_version: String, + pub total_bytes: u64, + pub used_bytes: u64, + pub avail_bytes: u64, + pub percent: f64, + /// Ancien cron `devforge-disk-guard` : present / absent / container. + pub legacy_guard: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize)] +pub struct Inventory { + pub disk: DiskInfo, + pub images: Vec, + pub containers: Vec, + pub builders: Vec, + /// Conteneurs BuildKit en cours (`buildx_buildkit_*`), nettoyés via `buildctl`. + pub buildkit: Vec, + /// Lignes `docker system df` : (type, taille, récupérable). + pub system_df: Vec<(String, String, String)>, +} + +/// `DOCKER_CONFIG` inscriptible (ZimaOS : `/root/.docker` en lecture seule) +/// puis racine Docker (`$R`). +const ROOT_DETECT: &str = "DC=\"${DOCKER_CONFIG:-$HOME/.docker}\"; \ + if ! mkdir -p \"$DC\" 2>/dev/null || [ ! -w \"$DC\" ]; then \ + export DOCKER_CONFIG=/tmp/devforge-docker-gc; mkdir -p \"$DOCKER_CONFIG\"; fi; \ + R=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null); [ -n \"$R\" ] || R=/var/lib/docker"; + +/// Ancien garde-fou cron (installé à la main avant la v2) : cassé sous dcron +/// (champ `root` + lignes de variables refusés) et il supprimait les +/// conteneurs arrêtés. Remplacé par le planificateur intégré. +pub const LEGACY_GUARD_CRON: &str = "/etc/cron.d/devforge-disk-guard"; + +/// État de l'ancien cron : `container` (DevForge conteneurisé, hôte non +/// visible), `present` ou `absent`. +const LEGACY_DETECT: &str = "if [ -f /.dockerenv ]; then echo container; \ + elif [ -f /etc/cron.d/devforge-disk-guard ]; then echo present; else echo absent; fi"; + +/// Désactive l'ancien cron s'il est visible : déplacé (réversible) vers +/// `/DATA/.devforge/state/` (persistant sur ZimaOS), sinon `/var/tmp`. +pub fn retire_legacy_guard_command() -> String { + format!( + "f={LEGACY_GUARD_CRON}; if [ ! -f /.dockerenv ] && [ -f \"$f\" ] && grep -q 'disk-guard.sh' \"$f\"; then \ + d=/DATA/.devforge/state; mkdir -p \"$d\" 2>/dev/null || d=/var/tmp; \ + mv \"$f\" \"$d/devforge-disk-guard.cron.disabled\" && echo retired || echo failed; else echo noop; fi" + ) +} + +/// Commande légère : disque de la racine Docker + driver de logs. +pub fn disk_command() -> String { + format!( + "{ROOT_DETECT}; echo '@@info'; docker info -f '{{{{.DockerRootDir}}}}|{{{{.LoggingDriver}}}}|{{{{.ServerVersion}}}}' 2>/dev/null; \ + echo '@@df'; df -Pk \"$R\" 2>/dev/null | tail -n 1; echo '@@legacy'; {LEGACY_DETECT}; echo '@@end'" + ) +} + +/// Inventaire en lecture seule (aucune suppression). +pub fn inventory_command() -> String { + format!( + "{disk}; echo '@@images'; I=$(docker image ls -q --no-trunc 2>/dev/null | sort -u); \ + [ -z \"$I\" ] || docker image inspect --format '{{{{.Id}}}}|{{{{.Created}}}}|{{{{.Size}}}}|{{{{join .RepoTags \",\"}}}}' $I 2>/dev/null; \ + echo '@@containers'; C=$(docker ps -aq --no-trunc 2>/dev/null); \ + [ -z \"$C\" ] || docker inspect --type container --format '{{{{.Id}}}}|{{{{.Image}}}}|{{{{.State.Status}}}}|{{{{.Name}}}}' $C 2>/dev/null; \ + echo '@@builders'; docker buildx ls 2>/dev/null; \ + echo '@@buildkit'; docker ps --filter name=buildx_buildkit_ --format '{{{{.Names}}}}' 2>/dev/null; \ + echo '@@sysdf'; docker system df --format '{{{{.Type}}}}|{{{{.Size}}}}|{{{{.Reclaimable}}}}' 2>/dev/null; echo '@@end'", + disk = disk_command().trim_end_matches("; echo '@@end'") + ) +} + +/// Cache de build récupérable : par builder buildx (plus vieux que +/// `retention_hours`) et par conteneur BuildKit (`buildctl du`, sans filtre +/// d'âge disponible : total récupérable). +pub fn cache_du_command( + builders: &[String], + buildkit: &[String], + retention_hours: Option, +) -> String { + let filter = retention_hours + .map(|h| format!(" --filter until={h}h")) + .unwrap_or_default(); + let mut cmd = format!("{ROOT_DETECT}; echo '@@cache'"); + for c in buildkit.iter().filter(|c| is_safe_name(c)) { + cmd.push_str(&format!( + "; printf '%s|%s\\n' 'buildkit:{c}' \"$(docker exec '{c}' buildctl du 2>/dev/null | awk '/^Reclaimable:/{{print $2}}' | tail -n 1)\"" + )); + } + for b in builders.iter().filter(|b| is_safe_name(b)) { + cmd.push_str(&format!( + "; printf '%s|%s\\n' '{b}' \"$(docker buildx du --builder '{b}'{filter} 2>/dev/null | awk '/^Reclaimable:/{{print $2}}' | tail -n 1)\"" + )); + } + cmd.push_str("; echo '@@end'"); + cmd +} + +fn is_safe_name(s: &str) -> bool { + !s.is_empty() + && s.len() <= 128 + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c)) +} + +/// Référence d'image passable à `docker image rm` (tag ou id), sans +/// caractère shell. +fn is_safe_ref(s: &str) -> bool { + !s.is_empty() + && s.len() <= 256 + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.:/@".contains(c)) +} + +fn sections(output: &str) -> HashMap> { + let mut out: HashMap> = HashMap::new(); + let mut current: Option = None; + for line in output.lines() { + let t = line.trim_end(); + if let Some(name) = t.trim().strip_prefix("@@") { + let name = name.trim().to_string(); + current = if name == "end" { + None + } else { + Some(name.clone()) + }; + if let Some(n) = ¤t { + out.entry(n.clone()).or_default(); + } + continue; + } + if let Some(n) = ¤t { + if !t.trim().is_empty() { + out.entry(n.clone()).or_default().push(t.to_string()); + } + } + } + out +} + +pub fn parse_disk(output: &str) -> DiskInfo { + let s = sections(output); + let mut d = DiskInfo::default(); + if let Some(line) = s.get("info").and_then(|v| v.first()) { + let mut p = line.split('|'); + d.root_dir = p.next().unwrap_or("").trim().to_string(); + d.logging_driver = p.next().unwrap_or("").trim().to_string(); + d.docker_version = p.next().unwrap_or("").trim().to_string(); + } + if let Some(line) = s.get("legacy").and_then(|v| v.first()) { + d.legacy_guard = line.trim().to_string(); + } + if let Some(line) = s.get("df").and_then(|v| v.last()) { + let cols: Vec<&str> = line.split_whitespace().collect(); + // Filesystem 1024-blocks Used Available Capacity Mounted + if cols.len() >= 4 { + let total = cols[1].parse::().unwrap_or(0) * 1024; + let used = cols[2].parse::().unwrap_or(0) * 1024; + let avail = cols[3].parse::().unwrap_or(0) * 1024; + d.total_bytes = total; + d.used_bytes = used; + d.avail_bytes = avail; + if used + avail > 0 { + d.percent = ((used as f64) * 1000.0 / ((used + avail) as f64)).round() / 10.0; + } + } + } + d +} + +/// Noms des builders `docker buildx ls` (formats ancien et nouveau). +pub fn parse_builders(lines: &[String]) -> Vec { + let mut out = Vec::new(); + for line in lines { + if line.starts_with(char::is_whitespace) || line.starts_with('\\') { + continue; + } + let Some(first) = line.split_whitespace().next() else { + continue; + }; + if first == "NAME/NODE" || first == "NAME" { + continue; + } + let name = first.trim_end_matches('*').trim(); + if is_safe_name(name) && !out.iter().any(|n| n == name) { + out.push(name.to_string()); + } + } + out +} + +pub fn parse_inventory(output: &str) -> Inventory { + let s = sections(output); + let mut inv = Inventory { + disk: parse_disk(output), + ..Default::default() + }; + let mut seen = HashSet::new(); + for line in s.get("images").into_iter().flatten() { + let p: Vec<&str> = line.splitn(4, '|').collect(); + if p.len() < 3 { + continue; + } + let Ok(created) = DateTime::parse_from_rfc3339(p[1].trim()) else { + continue; + }; + let id = p[0].trim().to_string(); + if !seen.insert(id.clone()) { + continue; + } + let tags = p + .get(3) + .map(|t| { + t.split(',') + .map(str::trim) + .filter(|t| !t.is_empty() && *t != ":") + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + inv.images.push(ImageInfo { + id, + created: created.with_timezone(&Utc), + size: p[2].trim().parse().unwrap_or(0), + tags, + }); + } + for line in s.get("containers").into_iter().flatten() { + let p: Vec<&str> = line.splitn(4, '|').collect(); + if p.len() < 3 { + continue; + } + inv.containers.push(ContainerInfo { + id: p[0].trim().to_string(), + image_id: p[1].trim().to_string(), + state: p[2].trim().to_string(), + name: p + .get(3) + .unwrap_or(&"") + .trim() + .trim_start_matches('/') + .to_string(), + }); + } + inv.builders = parse_builders(s.get("builders").map(Vec::as_slice).unwrap_or(&[])); + for line in s.get("buildkit").into_iter().flatten() { + let n = line.trim(); + if n.starts_with("buildx_buildkit_") + && is_safe_name(n) + && !inv.buildkit.iter().any(|b| b == n) + { + inv.buildkit.push(n.to_string()); + } + } + for line in s.get("sysdf").into_iter().flatten() { + let p: Vec<&str> = line.splitn(3, '|').collect(); + if p.len() == 3 { + inv.system_df + .push((p[0].trim().into(), p[1].trim().into(), p[2].trim().into())); + } + } + inv +} + +/// `builder|1.2GB` → (builder, octets). +pub fn parse_cache_du(output: &str) -> Vec<(String, u64)> { + sections(output) + .get("cache") + .into_iter() + .flatten() + .filter_map(|l| { + let (b, size) = l.split_once('|')?; + Some((b.trim().to_string(), parse_size(size).unwrap_or(0))) + }) + .collect() +} + +/// Taille Docker (`1.23GB`, `512MB`, `12.3kB`, `0B`, `1.5GiB`) → octets. +pub fn parse_size(s: &str) -> Option { + let t = s.trim().split_whitespace().next()?.trim(); + // `docker system df` : « 1.2GB (45%) » → on ne garde que la taille. + let t = t.split('(').next()?.trim(); + if t.is_empty() { + return None; + } + let idx = t + .find(|c: char| !(c.is_ascii_digit() || c == '.')) + .unwrap_or(t.len()); + let (num, unit) = t.split_at(idx); + let n: f64 = num.parse().ok()?; + let mult: f64 = match unit.trim().to_ascii_lowercase().as_str() { + "" | "b" => 1.0, + "kb" => 1e3, + "mb" => 1e6, + "gb" => 1e9, + "tb" => 1e12, + "kib" => 1024.0, + "mib" => 1024f64.powi(2), + "gib" => 1024f64.powi(3), + "tib" => 1024f64.powi(4), + _ => return None, + }; + Some((n * mult).round() as u64) +} + +// ───────────────────────── Sélection ───────────────────────── + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Candidate { + pub id: String, + /// Références à passer à `docker image rm` (tags, sinon l'id). + pub refs: Vec, + pub size: u64, + pub created: DateTime, + pub project: Option, + pub reason: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Kept { + pub id: String, + pub tags: Vec, + pub reason: String, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize)] +pub struct ImagePlan { + pub remove: Vec, + pub kept: Vec, +} + +impl ImagePlan { + pub fn bytes(&self) -> u64 { + self.remove.iter().map(|c| c.size).sum() + } +} + +/// `docker.io/library/df-abc:r2026…` → (`df-abc`, `r2026…`). +fn split_tag(tag: &str) -> (String, String) { + let t = tag + .trim_start_matches("docker.io/") + .trim_start_matches("library/"); + match t.rfind(':') { + Some(i) if !t[i + 1..].contains('/') => (t[..i].to_string(), t[i + 1..].to_string()), + _ => (t.to_string(), "latest".to_string()), + } +} + +/// Projet DevForge d'une image (`df-` ou preview `df-pv…`), sinon `None`. +fn project_of(image: &ImageInfo) -> Option { + image.tags.iter().find_map(|t| { + let (repo, _) = split_tag(t); + let rest = repo.strip_prefix("df-")?; + (!rest.is_empty() && rest.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')) + .then_some(repo) + }) +} + +fn is_current_deploy(image: &ImageInfo) -> bool { + image.tags.iter().any(|t| { + let (repo, tag) = split_tag(t); + repo.starts_with("df-") && tag == "latest" + }) +} + +/// Choisit les images à supprimer. Ne regarde ni les volumes ni les +/// conteneurs : un conteneur (quel que soit son état) protège son image. +pub fn select_images( + images: &[ImageInfo], + containers: &[ContainerInfo], + now: DateTime, + retention_hours: u32, + keep_per_project: u32, +) -> ImagePlan { + let running: HashSet<&str> = containers + .iter() + .filter(|c| c.state == "running" || c.state == "restarting" || c.state == "paused") + .map(|c| c.image_id.as_str()) + .collect(); + let used: HashSet<&str> = containers.iter().map(|c| c.image_id.as_str()).collect(); + let cutoff = now - Duration::hours(i64::from(retention_hours)); + let keep_n = keep_per_project.max(1) as usize; + + let mut plan = ImagePlan::default(); + let mut by_project: BTreeMap> = BTreeMap::new(); + for img in images { + if let Some(p) = project_of(img) { + by_project.entry(p).or_default().push(img); + } + } + let mut rollback_kept: HashSet<&str> = HashSet::new(); + for list in by_project.values_mut() { + list.sort_by(|a, b| b.created.cmp(&a.created)); + for img in list.iter().take(keep_n) { + rollback_kept.insert(img.id.as_str()); + } + } + + for img in images { + let kept = |reason: &str| Kept { + id: img.id.clone(), + tags: img.tags.clone(), + reason: reason.into(), + }; + if running.contains(img.id.as_str()) { + plan.kept.push(kept("utilisée par un conteneur en cours")); + continue; + } + if used.contains(img.id.as_str()) { + plan.kept + .push(kept("utilisée par un conteneur arrêté (jamais touché)")); + continue; + } + if is_current_deploy(img) { + plan.kept.push(kept("image du déploiement courant")); + continue; + } + let project = project_of(img); + let reason = if project.is_some() { + if rollback_kept.contains(img.id.as_str()) { + plan.kept + .push(kept(&format!("gardée pour rollback ({keep_n} par projet)"))); + continue; + } + format!("ancien déploiement (au-delà des {keep_n} gardées)") + } else if img.created <= cutoff { + if img.tags.is_empty() { + format!("image orpheline de plus de {retention_hours} h") + } else { + format!("inutilisée, créée il y a plus de {retention_hours} h") + } + } else { + plan.kept + .push(kept(&format!("plus récente que {retention_hours} h"))); + continue; + }; + let refs: Vec = if img.tags.is_empty() { + vec![img.id.clone()] + } else { + img.tags.clone() + }; + if !refs.iter().all(|r| is_safe_ref(r)) { + plan.kept.push(kept("référence non reconnue (ignorée)")); + continue; + } + plan.remove.push(Candidate { + id: img.id.clone(), + refs, + size: img.size, + created: img.created, + project, + reason, + }); + } + plan.remove.sort_by(|a, b| b.size.cmp(&a.size)); + plan +} + +/// Script de nettoyage d'un nœud. `cache_until_hours = None` (urgence) vide +/// tout le cache de build inutilisé. N'utilise jamais `-f` sur `image rm`. +pub fn cleanup_command( + builders: &[String], + buildkit: &[String], + cache_until_hours: Option, + plan: &ImagePlan, +) -> String { + let filter = cache_until_hours + .map(|h| format!(" --filter until={h}h")) + .unwrap_or_default(); + let avail = "$(df -Pk \"$R\" 2>/dev/null | tail -n 1 | awk '{print $4}')"; + let mut cmd = format!("{ROOT_DETECT}; A0={avail}"); + let safe: Vec<&String> = builders.iter().filter(|b| is_safe_name(b)).collect(); + if safe.is_empty() { + cmd.push_str(&format!( + "; echo '@@cache default'; docker builder prune -f{filter} 2>&1 | tail -n 2" + )); + } else { + for b in safe { + cmd.push_str(&format!( + "; echo '@@cache {b}'; docker buildx prune --builder '{b}' -f{filter} 2>&1 | tail -n 2" + )); + } + } + // Builders buildx non enregistrés pour cet utilisateur (ZimaOS) : prune + // directement dans chaque conteneur BuildKit en cours. + let keep = cache_until_hours + .map(|h| format!(" --keep-duration {h}h")) + .unwrap_or_default(); + for c in buildkit.iter().filter(|c| is_safe_name(c)) { + cmd.push_str(&format!( + "; echo '@@cache {c}'; docker exec '{c}' buildctl prune{keep} 2>&1 | tail -n 2" + )); + } + for c in &plan.remove { + for r in c.refs.iter().filter(|r| is_safe_ref(r)) { + cmd.push_str(&format!( + "; echo '@@rmi {r}'; docker image rm '{r}' 2>&1 | tail -n 2" + )); + } + } + cmd.push_str(&format!("; echo \"@@avail $A0 {avail}\"; echo '@@end'")); + cmd +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize)] +pub struct CleanupOutcome { + pub freed_bytes: u64, + pub images_removed: u32, + pub errors: Vec, +} + +/// Lit la sortie de [`cleanup_command`] : espace libéré (df avant/après), +/// images supprimées et erreurs. +pub fn parse_cleanup(output: &str) -> CleanupOutcome { + let mut out = CleanupOutcome::default(); + let mut current: Option = None; + let mut rmi_ok = false; + let finish = |cur: &Option, ok: bool, out: &mut CleanupOutcome| { + if ok && cur.as_deref().is_some_and(|c| c.starts_with("rmi ")) { + out.images_removed += 1; + } + }; + for line in output.lines() { + let t = line.trim(); + if let Some(rest) = t.strip_prefix("@@") { + finish(¤t, rmi_ok, &mut out); + rmi_ok = false; + if let Some(nums) = rest.strip_prefix("avail ") { + let v: Vec = nums + .split_whitespace() + .filter_map(|n| n.parse().ok()) + .collect(); + if v.len() == 2 { + out.freed_bytes = v[1].saturating_sub(v[0]) * 1024; + } + current = None; + continue; + } + current = (rest != "end").then(|| rest.to_string()); + continue; + } + let Some(cur) = ¤t else { continue }; + if cur.starts_with("rmi ") { + // « Deleted: » n'apparaît qu'à la suppression effective de l'image + // (les tags précédents ne donnent que « Untagged: »). + if t.starts_with("Deleted:") { + rmi_ok = true; + } else if t.to_ascii_lowercase().contains("error") { + out.errors.push(format!("{}: {}", cur, t)); + } + } else if cur.starts_with("cache ") && t.to_ascii_lowercase().contains("error") { + out.errors.push(format!("{}: {}", cur, t)); + } + } + finish(¤t, rmi_ok, &mut out); + out +} diff --git a/apps/server/src/docker_gc_routes.rs b/apps/server/src/docker_gc_routes.rs new file mode 100644 index 0000000000..4506c727a8 --- /dev/null +++ b/apps/server/src/docker_gc_routes.rs @@ -0,0 +1,616 @@ +//! API + planification du nettoyage Docker (voir `docker_gc.rs`). +//! +//! - `GET/PUT /api/v1/admin/docker-gc` : réglages, prochain passage, état ; +//! - `GET /api/v1/admin/docker-gc/nodes` : disque de chaque nœud ; +//! - `POST /api/v1/admin/docker-gc/preview` : simulation (lecture seule) ; +//! - `POST /api/v1/admin/docker-gc/run` `{confirm: true}` : nettoyage maintenant ; +//! - `GET /api/v1/admin/docker-gc/runs` : historique. +//! +//! Le planificateur tourne sur le leader : passage selon le cron (défaut +//! 03:00 Europe/Paris chaque jour) et nettoyage d'urgence d'un nœud dont le +//! disque Docker dépasse le seuil (au plus une fois par heure et par nœud). + +use std::collections::HashMap; +use std::sync::{Mutex as StdMutex, OnceLock}; +use std::time::Instant; + +use axum::{ + extract::{Query, State}, + http::{HeaderMap, StatusCode}, + routing::{get, post}, + Json, Router, +}; +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use serde_json::{json, Value}; + +use crate::docker_gc::{self, GcSettings}; +use crate::state::AppState; +use devforge_cluster::NodeRole; + +type ApiErr = (StatusCode, Json); + +fn bad(status: StatusCode, msg: impl Into) -> ApiErr { + (status, Json(json!({ "error": msg.into() }))) +} + +/// Un seul nettoyage à la fois (manuel, planifié ou urgence). +static RUN_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); +/// Dernier nettoyage d'urgence par nœud. +fn emergency_marks() -> &'static StdMutex> { + static M: OnceLock>> = OnceLock::new(); + M.get_or_init(|| StdMutex::new(HashMap::new())) +} + +pub fn router() -> Router { + Router::new() + .route( + "/api/v1/admin/docker-gc", + get(get_settings).put(put_settings), + ) + .route("/api/v1/admin/docker-gc/nodes", get(nodes_disk)) + .route("/api/v1/admin/docker-gc/preview", post(preview)) + .route("/api/v1/admin/docker-gc/run", post(run_now)) + .route("/api/v1/admin/docker-gc/runs", get(list_runs)) +} + +async fn require_admin(state: &AppState, headers: &HeaderMap) -> Result<(), ApiErr> { + let (user, _) = crate::auth_routes::current_workspace(state, headers).await?; + if user.role != "instance_admin" { + return Err(bad( + StatusCode::FORBIDDEN, + "Réservé à l’administrateur d’instance", + )); + } + Ok(()) +} + +// ───────────────────────── Stockage ───────────────────────── + +pub async fn ensure_tables(pool: &sqlx::PgPool) { + for sql in [ + r#"CREATE TABLE IF NOT EXISTS docker_gc_settings ( + id INTEGER PRIMARY KEY, config TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now())"#, + r#"CREATE TABLE IF NOT EXISTS docker_gc_runs ( + id BIGSERIAL PRIMARY KEY, started_at TIMESTAMPTZ NOT NULL, + finished_at TIMESTAMPTZ NOT NULL, trigger TEXT NOT NULL, + node_id TEXT NOT NULL, node_name TEXT NOT NULL, + ok BOOLEAN NOT NULL, freed_bytes BIGINT NOT NULL DEFAULT 0, + images_removed INTEGER NOT NULL DEFAULT 0, duration_ms BIGINT NOT NULL DEFAULT 0, + disk_percent_before DOUBLE PRECISION, disk_percent_after DOUBLE PRECISION, + errors TEXT NOT NULL DEFAULT '')"#, + ] { + if let Err(e) = sqlx::query(sql).execute(pool).await { + tracing::warn!(error = %e, "docker gc: création des tables"); + } + } +} + +pub async fn load_settings(pool: &sqlx::PgPool) -> GcSettings { + let row: Option<(String,)> = + sqlx::query_as("SELECT config FROM docker_gc_settings WHERE id = 1") + .fetch_optional(pool) + .await + .ok() + .flatten(); + row.and_then(|(c,)| serde_json::from_str(&c).ok()) + .unwrap_or_default() +} + +async fn save_settings(pool: &sqlx::PgPool, s: &GcSettings) -> Result<(), String> { + let cfg = serde_json::to_string(s).map_err(|e| e.to_string())?; + sqlx::query( + "INSERT INTO docker_gc_settings (id, config, updated_at) VALUES (1, $1, now()) + ON CONFLICT (id) DO UPDATE SET config = excluded.config, updated_at = now()", + ) + .bind(cfg) + .execute(pool) + .await + .map_err(|e| e.to_string())?; + Ok(()) +} + +fn apply_log_rotation(s: &GcSettings) { + devforge_deploy::docker::set_container_log_rotation( + s.log_rotation + .then_some((s.log_max_size_mb, s.log_max_files)), + ); +} + +// ───────────────────────── Nœuds ───────────────────────── + +#[derive(Clone, Debug)] +struct GcNode { + /// Id passé à l'executor (`default` pour le leader). + exec_id: String, + id: String, + name: String, + online: bool, +} + +async fn gc_nodes(state: &AppState) -> Vec { + let nodes = state.cluster.list_nodes().await.unwrap_or_default(); + if nodes.is_empty() { + return vec![GcNode { + exec_id: "default".into(), + id: "default".into(), + name: "Ce serveur".into(), + online: true, + }]; + } + nodes + .into_iter() + .map(|n| { + let leader = n.role == NodeRole::Leader; + GcNode { + exec_id: if leader { + "default".into() + } else { + n.id.clone() + }, + online: leader || n.status.as_str() == "online", + id: n.id, + name: n.name, + } + }) + .collect() +} + +async fn exec(state: &AppState, node: &GcNode, cmd: &str, timeout: u64) -> Result { + if !node.online { + return Err("nœud hors ligne".into()); + } + state + .deploy + .exec(&node.exec_id, "", cmd, timeout) + .await + .map(|r| r.output) + .map_err(|e| e.to_string()) +} + +async fn node_disk(state: &AppState, node: &GcNode) -> Value { + match exec(state, node, &docker_gc::disk_command(), 30).await { + Ok(out) => { + let d = docker_gc::parse_disk(&out); + json!({ + "node_id": node.id, "name": node.name, "ok": d.total_bytes > 0, + "disk": d, + "log_rotation_supported": log_driver_supports_rotation(&d.logging_driver), + }) + } + Err(e) => json!({ "node_id": node.id, "name": node.name, "ok": false, "error": e }), + } +} + +fn log_driver_supports_rotation(driver: &str) -> bool { + matches!(driver.trim(), "json-file" | "local" | "") +} + +// ───────────────────────── Handlers ───────────────────────── + +async fn get_settings( + State(state): State, + headers: HeaderMap, +) -> Result, ApiErr> { + require_admin(&state, &headers).await?; + let s = load_settings(&state.pool).await; + let last: Option<(DateTime, bool, i64, String)> = sqlx::query_as( + "SELECT finished_at, ok, freed_bytes, trigger FROM docker_gc_runs ORDER BY id DESC LIMIT 1", + ) + .fetch_optional(&state.pool) + .await + .ok() + .flatten(); + Ok(Json(json!({ + "settings": s, + "cron_expression": s.cron_expression(), + "next_run_at": s.enabled.then(|| s.next_run()).flatten(), + "running": RUN_LOCK.try_lock().is_err(), + "log_rotation_active": devforge_deploy::docker::container_log_rotation().is_some(), + "last_run": last.map(|(at, ok, freed, trigger)| json!({ + "finished_at": at, "ok": ok, "freed_bytes": freed, "trigger": trigger, + })), + }))) +} + +async fn put_settings( + State(state): State, + headers: HeaderMap, + Json(body): Json, +) -> Result, ApiErr> { + require_admin(&state, &headers).await?; + let mut s = body; + s.cron = s + .cron + .map(|c| c.trim().to_string()) + .filter(|c| !c.is_empty()); + s.validate().map_err(|e| bad(StatusCode::BAD_REQUEST, e))?; + if s.log_rotation { + // `--log-opt max-size` n'existe que pour json-file / local : un autre + // driver ferait échouer chaque `docker run`. + for node in gc_nodes(&state).await.iter().filter(|n| n.online) { + if let Ok(out) = exec(&state, node, &docker_gc::disk_command(), 30).await { + let d = docker_gc::parse_disk(&out); + if !log_driver_supports_rotation(&d.logging_driver) { + return Err(bad( + StatusCode::BAD_REQUEST, + format!( + "Rotation impossible : le nœud {} utilise le driver de logs « {} » (json-file ou local requis).", + node.name, d.logging_driver + ), + )); + } + } + } + } + save_settings(&state.pool, &s) + .await + .map_err(|e| bad(StatusCode::INTERNAL_SERVER_ERROR, e))?; + apply_log_rotation(&s); + Ok(Json(json!({ + "ok": true, + "settings": s, + "cron_expression": s.cron_expression(), + "next_run_at": s.enabled.then(|| s.next_run()).flatten(), + }))) +} + +async fn nodes_disk( + State(state): State, + headers: HeaderMap, +) -> Result, ApiErr> { + require_admin(&state, &headers).await?; + let s = load_settings(&state.pool).await; + let mut out = Vec::new(); + for node in gc_nodes(&state).await { + let mut v = node_disk(&state, &node).await; + let pct = v.pointer("/disk/percent").and_then(Value::as_f64); + v["alert"] = json!(pct.is_some_and(|p| p >= f64::from(s.disk_alert_percent))); + out.push(v); + } + Ok(Json( + json!({ "nodes": out, "alert_percent": s.disk_alert_percent }), + )) +} + +/// Plan d'un nœud : inventaire + images sélectionnées + cache récupérable. +struct NodePlan { + inventory: docker_gc::Inventory, + plan: docker_gc::ImagePlan, + cache: Vec<(String, u64)>, +} + +async fn plan_node( + state: &AppState, + node: &GcNode, + s: &GcSettings, + cache_hours: Option, + measure_cache: bool, +) -> Result { + let out = exec(state, node, &docker_gc::inventory_command(), 120).await?; + let inventory = docker_gc::parse_inventory(&out); + if inventory.disk.total_bytes == 0 && inventory.images.is_empty() { + return Err("Docker injoignable sur ce nœud".into()); + } + let plan = docker_gc::select_images( + &inventory.images, + &inventory.containers, + Utc::now(), + s.retention_hours, + s.keep_per_project, + ); + let cache = + if measure_cache && (!inventory.builders.is_empty() || !inventory.buildkit.is_empty()) { + exec( + state, + node, + &docker_gc::cache_du_command(&inventory.builders, &inventory.buildkit, cache_hours), + 180, + ) + .await + .map(|o| docker_gc::parse_cache_du(&o)) + .unwrap_or_default() + } else { + Vec::new() + }; + Ok(NodePlan { + inventory, + plan, + cache, + }) +} + +async fn preview(State(state): State, headers: HeaderMap) -> Result, ApiErr> { + require_admin(&state, &headers).await?; + let s = load_settings(&state.pool).await; + let mut nodes = Vec::new(); + for node in gc_nodes(&state).await { + match plan_node(&state, &node, &s, Some(s.retention_hours), true).await { + Ok(p) => { + let cache_bytes: u64 = p.cache.iter().map(|(_, b)| *b).sum(); + let build_cache_df = p + .inventory + .system_df + .iter() + .find(|(t, _, _)| t.eq_ignore_ascii_case("Build Cache")) + .map(|(_, size, rec)| json!({ "size": size, "reclaimable": rec })); + nodes.push(json!({ + "node_id": node.id, "name": node.name, "ok": true, + "disk": p.inventory.disk, + "images": p.plan.remove.iter().map(|c| json!({ + "id": c.id.trim_start_matches("sha256:").chars().take(12).collect::(), + "refs": c.refs.iter().filter(|r| !r.starts_with("sha256:")).collect::>(), + "size": c.size, "created": c.created, "project": c.project, "reason": c.reason, + })).collect::>(), + "images_bytes": p.plan.bytes(), + "kept_count": p.plan.kept.len(), + "protected_running": p.plan.kept.iter().filter(|k| k.reason.contains("en cours")).count(), + "containers_untouched": p.inventory.containers.len(), + "builders": p.inventory.builders, + "buildkit_containers": p.inventory.buildkit, + "cache": p.cache.iter().map(|(b, n)| json!({ "builder": b, "bytes": n })).collect::>(), + "cache_bytes": cache_bytes, + "build_cache_total": build_cache_df, + "total_bytes": cache_bytes + p.plan.bytes(), + })); + } + Err(e) => nodes.push(json!({ + "node_id": node.id, "name": node.name, "ok": false, "error": e, + })), + } + } + Ok(Json(json!({ + "dry_run": true, + "retention_hours": s.retention_hours, + "keep_per_project": s.keep_per_project, + "nodes": nodes, + }))) +} + +#[cfg(test)] +#[path = "docker_gc_tests.rs"] +mod tests; + +#[derive(Deserialize)] +struct RunBody { + #[serde(default)] + confirm: bool, +} + +async fn run_now( + State(state): State, + headers: HeaderMap, + Json(body): Json, +) -> Result<(StatusCode, Json), ApiErr> { + require_admin(&state, &headers).await?; + if !body.confirm { + return Err(bad( + StatusCode::BAD_REQUEST, + "Confirmation requise (confirm: true).", + )); + } + let Ok(guard) = RUN_LOCK.try_lock() else { + return Err(bad(StatusCode::CONFLICT, "Un nettoyage est déjà en cours.")); + }; + drop(guard); + let st = state.clone(); + tokio::spawn(async move { + run_all(&st, "manual", None).await; + }); + Ok(( + StatusCode::ACCEPTED, + Json(json!({ "ok": true, "started": true })), + )) +} + +#[derive(Deserialize)] +struct RunsQuery { + limit: Option, +} + +async fn list_runs( + State(state): State, + headers: HeaderMap, + Query(q): Query, +) -> Result, ApiErr> { + require_admin(&state, &headers).await?; + let limit = q.limit.unwrap_or(50).clamp(1, 200); + #[allow(clippy::type_complexity)] + let rows: Vec<( + i64, + DateTime, + DateTime, + String, + String, + String, + bool, + i64, + i32, + i64, + Option, + Option, + String, + )> = sqlx::query_as( + "SELECT id, started_at, finished_at, trigger, node_id, node_name, ok, freed_bytes, + images_removed, duration_ms, disk_percent_before, disk_percent_after, errors + FROM docker_gc_runs ORDER BY id DESC LIMIT $1", + ) + .bind(limit) + .fetch_all(&state.pool) + .await + .map_err(|e| bad(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + let runs: Vec = rows + .into_iter() + .map(|r| { + json!({ + "id": r.0, "started_at": r.1, "finished_at": r.2, "trigger": r.3, + "node_id": r.4, "node_name": r.5, "ok": r.6, "freed_bytes": r.7, + "images_removed": r.8, "duration_ms": r.9, + "disk_percent_before": r.10, "disk_percent_after": r.11, + "errors": r.12.lines().filter(|l| !l.is_empty()).collect::>(), + }) + }) + .collect(); + Ok(Json(json!({ "runs": runs }))) +} + +// ───────────────────────── Exécution ───────────────────────── + +/// Nettoie tous les nœuds (ou `only` : un nœud). `trigger` : manual, +/// schedule, emergency. L'urgence vide tout le cache de build inutilisé. +async fn run_all(state: &AppState, trigger: &str, only: Option<&str>) { + let _guard = RUN_LOCK.lock().await; + let s = load_settings(&state.pool).await; + for node in gc_nodes(state).await { + if only.is_some_and(|id| id != node.id) { + continue; + } + run_node(state, &node, &s, trigger).await; + } +} + +async fn run_node(state: &AppState, node: &GcNode, s: &GcSettings, trigger: &str) { + let started = Utc::now(); + let t0 = Instant::now(); + let cache_hours = (trigger != "emergency").then_some(s.retention_hours); + let mut errors: Vec = Vec::new(); + let mut ok = false; + let mut freed = 0u64; + let mut removed = 0u32; + let mut before = None; + let mut after = None; + match plan_node(state, node, s, cache_hours, false).await { + Ok(p) => { + before = Some(p.inventory.disk.percent); + let cmd = docker_gc::cleanup_command( + &p.inventory.builders, + &p.inventory.buildkit, + cache_hours, + &p.plan, + ); + match exec(state, node, &cmd, 1800).await { + Ok(out) => { + let o = docker_gc::parse_cleanup(&out); + freed = o.freed_bytes; + removed = o.images_removed; + ok = o.errors.is_empty(); + errors.extend(o.errors); + } + Err(e) => errors.push(e), + } + if let Ok(out) = exec(state, node, &docker_gc::disk_command(), 30).await { + after = Some(docker_gc::parse_disk(&out).percent); + } + } + Err(e) => errors.push(e), + } + let errors: Vec = errors.into_iter().take(30).collect(); + tracing::info!( + node = %node.name, trigger, ok, freed_bytes = freed, images_removed = removed, + "docker gc terminé" + ); + let _ = sqlx::query( + "INSERT INTO docker_gc_runs (started_at, finished_at, trigger, node_id, node_name, ok, + freed_bytes, images_removed, duration_ms, disk_percent_before, disk_percent_after, errors) + VALUES ($1, now(), $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind(started) + .bind(trigger) + .bind(&node.id) + .bind(&node.name) + .bind(ok) + .bind(freed.min(i64::MAX as u64) as i64) + .bind(removed as i32) + .bind(t0.elapsed().as_millis().min(i64::MAX as u128) as i64) + .bind(before) + .bind(after) + .bind(errors.join("\n")) + .execute(&state.pool) + .await; + let _ = sqlx::query( + "DELETE FROM docker_gc_runs WHERE id NOT IN (SELECT id FROM docker_gc_runs ORDER BY id DESC LIMIT 500)", + ) + .execute(&state.pool) + .await; +} + +// ───────────────────────── Planificateur ───────────────────────── + +/// Urgence due pour un nœud ? (seuil atteint, pas d'urgence depuis 1 h). +pub fn emergency_due(percent: f64, threshold: u32, last: Option, now: Instant) -> bool { + let cooled = match last { + None => true, + Some(t) => now.duration_since(t) >= std::time::Duration::from_secs(3600), + }; + percent >= f64::from(threshold) && cooled +} + +/// Désactive l'ancien cron `devforge-disk-guard` sur les nœuds où il est +/// visible (DevForge sur l'hôte). Conteneurisé : signalé dans l'admin. +async fn retire_legacy_guards(state: &AppState) { + for node in gc_nodes(state).await.iter().filter(|n| n.online) { + match exec(state, node, &docker_gc::retire_legacy_guard_command(), 20).await { + Ok(out) if out.contains("retired") => tracing::warn!( + node = %node.name, + "ancien cron {} désactivé (remplacé par le nettoyage Docker intégré)", + docker_gc::LEGACY_GUARD_CRON + ), + Ok(out) if out.contains("failed") => tracing::warn!( + node = %node.name, + "ancien cron {} présent mais non déplaçable", + docker_gc::LEGACY_GUARD_CRON + ), + _ => {} + } + } +} + +pub fn spawn_loop(state: AppState) { + tokio::spawn(async move { + ensure_tables(&state.pool).await; + retire_legacy_guards(&state).await; + let mut s = load_settings(&state.pool).await; + apply_log_rotation(&s); + let mut next = s.next_run(); + let mut last_disk_check = Instant::now() - std::time::Duration::from_secs(3600); + let mut tick = tokio::time::interval(std::time::Duration::from_secs(60)); + loop { + tick.tick().await; + let fresh = load_settings(&state.pool).await; + if fresh != s { + s = fresh; + apply_log_rotation(&s); + next = s.next_run(); + } + if !s.enabled { + continue; + } + if next.is_some_and(|t| Utc::now() >= t) { + run_all(&state, "schedule", None).await; + next = s.next_run(); + continue; + } + if last_disk_check.elapsed() >= std::time::Duration::from_secs(300) { + last_disk_check = Instant::now(); + for node in gc_nodes(&state).await.iter().filter(|n| n.online) { + let Ok(out) = exec(&state, node, &docker_gc::disk_command(), 30).await else { + continue; + }; + let pct = docker_gc::parse_disk(&out).percent; + let last = emergency_marks() + .lock() + .ok() + .and_then(|m| m.get(&node.id).copied()); + if emergency_due(pct, s.disk_alert_percent, last, Instant::now()) { + tracing::warn!(node = %node.name, percent = pct, "disque Docker au-dessus du seuil : nettoyage d'urgence"); + if let Ok(mut m) = emergency_marks().lock() { + m.insert(node.id.clone(), Instant::now()); + } + run_all(&state, "emergency", Some(&node.id)).await; + } + } + } + } + }); +} diff --git a/apps/server/src/docker_gc_tests.rs b/apps/server/src/docker_gc_tests.rs new file mode 100644 index 0000000000..bcdbcc4e8c --- /dev/null +++ b/apps/server/src/docker_gc_tests.rs @@ -0,0 +1,348 @@ +use super::emergency_due; +use crate::docker_gc::*; +use chrono::{Duration, TimeZone, Utc}; +use std::time::Instant; + +fn now() -> chrono::DateTime { + Utc.with_ymd_and_hms(2026, 10, 21, 3, 0, 0).unwrap() +} + +fn img(id: &str, hours_old: i64, size: u64, tags: &[&str]) -> ImageInfo { + ImageInfo { + id: format!("sha256:{id}"), + created: now() - Duration::hours(hours_old), + size, + tags: tags.iter().map(|t| t.to_string()).collect(), + } +} + +fn ctr(image: &str, state: &str) -> ContainerInfo { + ContainerInfo { + id: format!("c-{image}"), + image_id: format!("sha256:{image}"), + state: state.into(), + name: format!("df-{image}"), + } +} + +fn bk() -> Vec { + vec!["buildx_buildkit_builder-x0".to_string()] +} + +fn removed(plan: &ImagePlan) -> Vec { + let mut v: Vec = plan + .remove + .iter() + .map(|c| c.id.trim_start_matches("sha256:").to_string()) + .collect(); + v.sort(); + v +} + +#[test] +fn defaults_match_spec() { + let s = GcSettings::default(); + assert!(s.enabled); + assert_eq!(s.frequency_hours, 24); + assert_eq!(s.hour, 3); + assert_eq!(s.timezone, "Europe/Paris"); + assert_eq!(s.retention_hours, 24); + assert_eq!(s.keep_per_project, 3); + assert_eq!(s.disk_alert_percent, 85); + assert!(!s.log_rotation); + assert_eq!(s.cron_expression(), "0 3 * * *"); + assert!(s.validate().is_ok()); + assert!(s.next_run().is_some()); +} + +#[test] +fn schedule_expressions_and_validation() { + let mut s = GcSettings { + frequency_hours: 6, + hour: 3, + ..Default::default() + }; + assert_eq!(s.cron_expression(), "0 3,9,15,21 * * *"); + s.frequency_hours = 12; + s.hour = 20; + assert_eq!(s.cron_expression(), "0 8,20 * * *"); + s.frequency_hours = 48; + assert_eq!(s.cron_expression(), "0 20 */2 * *"); + s.frequency_hours = 168; + assert_eq!(s.cron_expression(), "0 20 * * 1"); + s.cron = Some("30 4 * * *".into()); + assert_eq!(s.cron_expression(), "30 4 * * *"); + assert!(s.validate().is_ok()); + s.cron = Some("* * * * *".into()); + assert!(s.validate().is_err(), "plus d'un passage par heure refusé"); + let bad = GcSettings { + frequency_hours: 5, + ..Default::default() + }; + assert!(bad.validate().is_err()); + let bad = GcSettings { + disk_alert_percent: 100, + ..Default::default() + }; + assert!(bad.validate().is_err()); + let bad = GcSettings { + keep_per_project: 0, + ..Default::default() + }; + assert!(bad.validate().is_err()); +} + +#[test] +fn settings_json_is_backward_compatible() { + let s: GcSettings = serde_json::from_str(r#"{"enabled":false}"#).unwrap(); + assert!(!s.enabled); + assert_eq!(s.keep_per_project, 3); +} + +#[test] +fn retention_applies_to_unused_non_project_images() { + let images = vec![ + img("old", 48, 100, &["node:24-bookworm"]), + img("fresh", 2, 100, &["node:22"]), + img("dangling-old", 30, 50, &[]), + img("dangling-new", 1, 50, &[]), + ]; + let plan = select_images(&images, &[], now(), 24, 3); + assert_eq!(removed(&plan), vec!["dangling-old", "old"]); + let dangling = plan + .remove + .iter() + .find(|c| c.id.ends_with("dangling-old")) + .unwrap(); + assert_eq!(dangling.refs, vec!["sha256:dangling-old".to_string()]); + assert_eq!(plan.bytes(), 150); +} + +#[test] +fn keeps_n_most_recent_images_per_project() { + let images = vec![ + img("p1", 1, 10, &["df-abc123def456:latest"]), + img("p2", 10, 10, &["df-abc123def456:r2"]), + img("p3", 20, 10, &["df-abc123def456:r3"]), + img("p4", 30, 10, &["df-abc123def456:r4"]), + img("p5", 2, 10, &["df-abc123def456:r5"]), + img("q1", 100, 10, &["df-zzz:r1"]), + ]; + let plan = select_images(&images, &[ctr("p1", "running")], now(), 24, 3); + // Récentes gardées : p1 (courante), p5, p2. p3/p4 au-delà de 3 — même si < 24 h. + assert_eq!(removed(&plan), vec!["p3", "p4"]); + let c = plan.remove.iter().find(|c| c.id.ends_with("p3")).unwrap(); + assert_eq!(c.project.as_deref(), Some("df-abc123def456")); + assert_eq!(c.refs, vec!["df-abc123def456:r3".to_string()]); + // Seule image de l'autre projet : gardée même vieille. + assert!(plan.kept.iter().any(|k| k.id.ends_with("q1"))); +} + +#[test] +fn running_and_stopped_container_images_are_never_selected() { + let images = vec![ + img("run", 500, 10, &["postgres:16"]), + img("stop", 500, 10, &["redis:7"]), + img("old1", 500, 10, &["df-abc:r1"]), + img("old2", 400, 10, &["df-abc:r2"]), + img("old3", 300, 10, &["df-abc:r3"]), + img("old4", 200, 10, &["df-abc:r4"]), + ]; + let containers = vec![ + ctr("run", "running"), + ctr("stop", "exited"), + // conteneur arrêté sur la plus vieille image du projet + ctr("old1", "exited"), + ]; + let plan = select_images(&images, &containers, now(), 1, 1); + let r = removed(&plan); + assert!(!r.contains(&"run".to_string())); + assert!(!r.contains(&"stop".to_string())); + assert!(!r.contains(&"old1".to_string())); + assert_eq!(r, vec!["old2", "old3"]); + assert!(plan + .kept + .iter() + .any(|k| k.id.ends_with("stop") && k.reason.contains("arrêté"))); +} + +#[test] +fn current_deploy_latest_is_never_selected() { + // Même hors des N et sans conteneur : `df-…:latest` est l'image courante. + let images = vec![ + img("new1", 1, 10, &["df-abc:r9"]), + img("cur", 50, 10, &["df-abc:latest"]), + ]; + let plan = select_images(&images, &[], now(), 24, 1); + assert!(plan.remove.iter().all(|c| !c.id.ends_with("cur"))); +} + +#[test] +fn unsafe_refs_are_skipped() { + let images = vec![img("evil", 100, 10, &["bad;rm -rf /:x"])]; + let plan = select_images(&images, &[], now(), 24, 3); + assert!(plan.remove.is_empty()); +} + +#[test] +fn cleanup_script_never_touches_volumes_or_containers() { + let images = vec![ + img("old", 48, 100, &["node:24"]), + img("dangling", 48, 100, &[]), + ]; + let plan = select_images(&images, &[], now(), 24, 3); + for cmd in [ + cleanup_command(&["default".into(), "multi".into()], &bk(), Some(24), &plan), + cleanup_command(&[], &[], None, &plan), + inventory_command(), + disk_command(), + cache_du_command(&["default".into()], &bk(), Some(24)), + retire_legacy_guard_command(), + ] { + let lower = cmd.to_ascii_lowercase(); + assert!(!lower.contains("volume"), "{cmd}"); + assert!(!lower.contains("container prune"), "{cmd}"); + assert!(!lower.contains("system prune"), "{cmd}"); + assert!(!lower.contains("docker rm "), "{cmd}"); + assert!(!lower.contains(" rm -f"), "{cmd}"); + assert!(!lower.contains("rmi -f"), "{cmd}"); + assert!(!lower.contains("--force"), "{cmd}"); + assert!(!lower.contains("image prune"), "{cmd}"); + } + let cmd = cleanup_command(&["default".into(), "multi".into()], &bk(), Some(24), &plan); + assert!( + cmd.contains("docker exec 'buildx_buildkit_builder-x0' buildctl prune --keep-duration 24h") + ); + assert!(cmd.contains("export DOCKER_CONFIG=/tmp/devforge-docker-gc")); + assert!(cmd.contains("docker buildx prune --builder 'default' -f --filter until=24h")); + assert!(cmd.contains("docker buildx prune --builder 'multi' -f --filter until=24h")); + assert!(cmd.contains("docker image rm 'node:24'")); + assert!(cmd.contains("docker image rm 'sha256:dangling'")); + // Urgence : tout le cache inutilisé, sans filtre d'âge. + let urgent = cleanup_command(&[], &bk(), None, &plan); + assert!(urgent.contains("buildctl prune 2>&1")); + assert!( + urgent.contains("docker builder prune -f;") + || urgent.contains("docker builder prune -f 2>&1") + ); + // Builder au nom douteux ignoré. + let cmd = cleanup_command( + &["x;rm".into()], + &["y;rm".into()], + Some(24), + &ImagePlan::default(), + ); + assert!(!cmd.contains("x;rm")); + assert!(!cmd.contains("y;rm")); +} + +#[test] +fn inventory_is_parsed() { + let out = "\ +@@info +/media/Docker/docker|json-file|27.5.1 +@@df +/dev/sda1 465000000 395250000 69750000 85% /media/Docker +@@legacy +present +@@images +sha256:aaa|2026-10-20T01:00:00.123456789Z|1000|df-abc:latest,df-abc:r1 +sha256:bbb|2026-10-01T01:00:00+02:00|2000| +sha256:aaa|2026-10-20T01:00:00Z|1000|df-abc:latest +garbage line +@@containers +c1|sha256:aaa|running|/df-abc +c2|sha256:bbb|exited|/old +@@builders +NAME/NODE DRIVER/ENDPOINT STATUS BUILDKIT PLATFORMS +default* docker + \\_ default \\_ default running v0.13.2 linux/amd64 +multi docker-container + multi0 unix:///var/run/docker.sock running v0.12 linux/amd64 +@@buildkit +buildx_buildkit_builder-x0 +buildx_buildkit_bad;name +@@sysdf +Images|40GB|12GB (30%) +Build Cache|380GB|300GB +@@end +"; + let inv = parse_inventory(out); + assert_eq!(inv.disk.root_dir, "/media/Docker/docker"); + assert_eq!(inv.disk.logging_driver, "json-file"); + assert_eq!(inv.disk.total_bytes, 465000000 * 1024); + assert!((inv.disk.percent - 85.0).abs() < 0.1); + assert_eq!(inv.images.len(), 2, "doublons et lignes invalides ignorés"); + assert_eq!(inv.images[0].tags, vec!["df-abc:latest", "df-abc:r1"]); + assert!(inv.images[1].tags.is_empty()); + assert_eq!(inv.containers.len(), 2); + assert_eq!(inv.containers[1].state, "exited"); + assert_eq!(inv.containers[0].name, "df-abc"); + assert_eq!(inv.builders, vec!["default", "multi"]); + assert_eq!(inv.buildkit, vec!["buildx_buildkit_builder-x0"]); + assert_eq!(inv.disk.legacy_guard, "present"); + assert_eq!(inv.system_df.len(), 2); +} + +#[test] +fn sizes_and_cache_du_are_parsed() { + assert_eq!(parse_size("0B"), Some(0)); + assert_eq!(parse_size("12.5kB"), Some(12_500)); + assert_eq!(parse_size("1.5GB"), Some(1_500_000_000)); + assert_eq!(parse_size("2GiB"), Some(2 * 1024 * 1024 * 1024)); + assert_eq!(parse_size("300GB (45%)"), Some(300_000_000_000)); + assert_eq!(parse_size(""), None); + assert_eq!(parse_size("abc"), None); + let out = "@@cache\ndefault|1.2GB\nmulti|\n@@end\n"; + assert_eq!( + parse_cache_du(out), + vec![ + ("default".to_string(), 1_200_000_000), + ("multi".to_string(), 0) + ] + ); +} + +#[test] +fn cleanup_output_is_parsed() { + let out = "\ +@@cache default +Total: 3.2GB +@@rmi df-abc:r1 +Untagged: df-abc:r1 +@@rmi df-abc:r2 +Untagged: df-abc:r2 +Deleted: sha256:111 +Deleted: sha256:222 +@@rmi node:24 +Error response from daemon: conflict: unable to remove repository reference \"node:24\" (must force) - container 123 is using its referenced image +@@avail 1000 5000 +@@end +"; + let o = parse_cleanup(out); + assert_eq!(o.freed_bytes, 4000 * 1024); + assert_eq!(o.images_removed, 1); + assert_eq!(o.errors.len(), 1); + assert!(o.errors[0].starts_with("rmi node:24")); +} + +#[test] +fn emergency_threshold_and_cooldown() { + let t = Instant::now(); + assert!(emergency_due(90.0, 85, None, t)); + assert!(!emergency_due(80.0, 85, None, t)); + assert!(!emergency_due(99.0, 85, Some(t), t)); + let later = t + std::time::Duration::from_secs(3601); + assert!(emergency_due(99.0, 85, Some(t), later)); +} + +#[test] +fn legacy_guard_is_moved_not_deleted_and_never_touches_other_crons() { + let cmd = retire_legacy_guard_command(); + assert!(cmd.contains("/etc/cron.d/devforge-disk-guard")); + assert!(cmd.contains("mv ")); + assert!(!cmd.contains("rm ")); + assert!(!cmd.contains("docker-cleanup-interim")); + assert!(cmd.contains("[ ! -f /.dockerenv ]")); + assert!(disk_command().contains("@@legacy")); +} diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index 327a18cfe6..9a274b1a4e 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -24,6 +24,8 @@ mod detect_svc; mod dev_env; mod dns; mod dns_failover; +mod docker_gc; +mod docker_gc_routes; mod domain_catalog; mod draft; mod draft_origin; @@ -252,6 +254,7 @@ async fn main() -> Result<(), Box> { crate::dns::spawn_dns_loop(state.clone()); crate::dns_failover::spawn_loop(state.clone()); + crate::docker_gc_routes::spawn_loop(state.clone()); routes::resume_deploy_queue(state.clone()); run_routes::install_launcher(&state); routes::resume_agent_runs(state.clone()); @@ -378,6 +381,7 @@ async fn main() -> Result<(), Box> { .merge(backup_routes::router()) .merge(sso_routes::router()) .merge(acme_routes::router()) + .merge(docker_gc_routes::router()) .merge(project_oidc_routes::router()) .merge(dev_env::router()) .merge(project_pg_routes::router()) diff --git a/apps/web/src/components/AdminPage.tsx b/apps/web/src/components/AdminPage.tsx index a5f61b8f44..e704562381 100644 --- a/apps/web/src/components/AdminPage.tsx +++ b/apps/web/src/components/AdminPage.tsx @@ -10,6 +10,7 @@ import { PostgresAdminPanel } from './PostgresAdminPanel'; import { SsoSettingsPanel } from './SsoSettingsPanel'; import { UpdateSettingsPanel } from './UpdateSettingsPanel'; import { CursorWorkerSettingsPanel } from './CursorWorkerSettingsPanel'; +import { DockerGcPanel, formatBytes } from './DockerGcPanel'; import { Alert, Badge, @@ -72,7 +73,8 @@ type AdminSection = | 'update' | 'domaine' | 'cluster' - | 'cursor-worker'; + | 'cursor-worker' + | 'docker'; const SECTION_TITLES: Record, string> = { workspaces: 'Workspaces', @@ -87,6 +89,7 @@ const SECTION_TITLES: Record, string> = { domaine: 'Domaine principal', cluster: 'Cluster', 'cursor-worker': 'Worker Cursor', + docker: 'Nettoyage Docker', }; const ADMIN_TABS: AdminSection[] = [ @@ -102,6 +105,7 @@ const ADMIN_TABS: AdminSection[] = [ 'domaine', 'cluster', 'cursor-worker', + 'docker', ]; type BetaFeatures = { @@ -216,6 +220,11 @@ function AdminHub() { eligible: number; cli: boolean; } | null>(null); + const [dockerGc, setDockerGc] = useState<{ + enabled: boolean; + next?: string | null; + lastFreed?: number; + } | null>(null); const [loading, setLoading] = useState(true); useEffect(() => { @@ -238,6 +247,7 @@ function AdminHub() { api.dnsSettings(), api.bootstrap(), api.cursorWorkerStatus(), + api.dockerGcGet(), ]); const [proxyRes, healthRes, adminRes] = await core; if (cancelled) return; @@ -258,6 +268,7 @@ function AdminHub() { dnsRes, bootRes, cursorRes, + gcRes, ] = await rest; if (cancelled) return; if (clusterRes.status === 'fulfilled') { @@ -267,6 +278,13 @@ function AdminHub() { interim: !!clusterRes.value.acting_leader, }); } + if (gcRes.status === 'fulfilled') { + setDockerGc({ + enabled: !!gcRes.value.settings?.enabled, + next: gcRes.value.next_run_at, + lastFreed: gcRes.value.last_run?.freed_bytes, + }); + } if (cursorRes.status === 'fulfilled') { setCursorWorker({ running: !!cursorRes.value.running, @@ -487,6 +505,18 @@ function AdminHub() { : tileStatus('Off', 'neutral', `${cursorWorker.eligible} repo(s)`) : null; + const gcTile = !dockerGc + ? null + : dockerGc.enabled + ? tileStatus( + 'Auto', + 'ok', + dockerGc.lastFreed != null + ? `${formatBytes(dockerGc.lastFreed)} libérés au dernier passage` + : 'Chaque jour à 03:00', + ) + : tileStatus('Off', 'warn', 'Nettoyage désactivé'); + return ( - + } + badge={gcTile?.badge} + subtitle={gcTile?.subtitle} + /> + ); } @@ -1238,6 +1277,7 @@ function AdminPageInner() { {section === 'update' && } {section === 'cluster' && } {section === 'cursor-worker' && } + {section === 'docker' && } ); diff --git a/apps/web/src/components/DockerGcPanel.test.tsx b/apps/web/src/components/DockerGcPanel.test.tsx new file mode 100644 index 0000000000..37d042e0b3 --- /dev/null +++ b/apps/web/src/components/DockerGcPanel.test.tsx @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest'; +import { formatBytes } from './DockerGcPanel'; + +describe('formatBytes', () => { + it('formats decimal units like Docker', () => { + expect(formatBytes(0)).toBe('0 o'); + expect(formatBytes(999)).toBe('999 o'); + expect(formatBytes(1_500_000_000)).toBe('1,5 Go'); + expect(formatBytes(425_000_000_000)).toBe('425 Go'); + expect(formatBytes(null)).toBe('0 o'); + }); +}); diff --git a/apps/web/src/components/DockerGcPanel.tsx b/apps/web/src/components/DockerGcPanel.tsx new file mode 100644 index 0000000000..8648b07b75 --- /dev/null +++ b/apps/web/src/components/DockerGcPanel.tsx @@ -0,0 +1,835 @@ +import type { ComponentChildren } from 'preact'; +import { useEffect, useState } from 'preact/hooks'; +import { + api, + type DockerGcNodeDisk, + type DockerGcPreview, + type DockerGcRun, + type DockerGcSettings, + type DockerGcState, +} from '../lib/api'; +import { + Alert, + Badge, + Button, + HubGrid, + HubIcon, + HubTile, + Input, + Modal, + Skeleton, + Spinner, + Switch, + useToast, +} from './ui'; + +const TZ = 'Europe/Paris'; + +export function formatBytes(n: number | null | undefined): string { + const v = Number(n || 0); + if (v < 1000) return `${v} o`; + const units = ['ko', 'Mo', 'Go', 'To']; + let x = v; + let i = -1; + while (x >= 1000 && i < units.length - 1) { + x /= 1000; + i += 1; + } + return `${x.toLocaleString('fr-FR', { maximumFractionDigits: x < 10 ? 1 : 0 })} ${units[i]}`; +} + +function formatWhen(iso: string | null | undefined): string { + if (!iso) return '—'; + try { + return new Date(iso).toLocaleString('fr-FR', { + timeZone: TZ, + weekday: 'short', + day: 'numeric', + month: 'short', + hour: '2-digit', + minute: '2-digit', + }); + } catch { + return iso; + } +} + +const TRIGGER_LABEL: Record = { + manual: 'Manuel', + schedule: 'Planifié', + emergency: 'Urgence disque', +}; + +type Tone = 'ok' | 'warn' | 'danger' | 'neutral' | 'accent'; + +const PILL: Record = { + ok: 'bg-emerald-500 text-zinc-950', + warn: 'bg-amber-500 text-zinc-950', + danger: 'bg-red-500 text-white', + accent: 'bg-[var(--color-accent)] text-black', + neutral: 'bg-white/20 text-white', +}; + +const TEXT: Record = { + ok: 'text-[var(--color-ok)]', + warn: 'text-[var(--color-warn)]', + danger: 'text-[var(--color-danger)]', + accent: 'text-[var(--color-accent)]', + neutral: 'text-[var(--color-ink-muted)]', +}; + +function status(label: string, tone: Tone, detail?: string) { + return { + badge: ( + + {label} + + ), + subtitle: detail ? ( +
+ {detail} +
+ ) : undefined, + }; +} + +function diskTone(pct: number, alert: number): Tone { + if (pct >= alert) return 'danger'; + if (pct >= alert - 10) return 'warn'; + return 'ok'; +} + +function DiskBar({ pct, alert }: { pct: number; alert: number }) { + const tone = diskTone(pct, alert); + const color = + tone === 'danger' ? 'bg-red-500' : tone === 'warn' ? 'bg-amber-500' : 'bg-emerald-500'; + return ( +
+
+
+
+ ); +} + +function Row({ label, value }: { label: string; value: ComponentChildren }) { + return ( +
+ {label} + {value} +
+ ); +} + +const FREQUENCIES = [ + { h: 6, label: 'Toutes les 6 h' }, + { h: 12, label: 'Toutes les 12 h' }, + { h: 24, label: 'Chaque jour' }, + { h: 48, label: 'Tous les 2 jours' }, + { h: 168, label: 'Chaque semaine (lundi)' }, +]; + +const SELECT_CLASS = + 'h-11 w-full min-w-0 cursor-pointer rounded-xl border border-[var(--color-line)] bg-[var(--color-surface)] px-3 text-base text-[var(--color-ink)] outline-none transition-[border-color,box-shadow] duration-200 focus:border-[var(--color-accent)]/50 focus:ring-2 focus:ring-[var(--color-accent-soft)] sm:text-sm lg:h-10'; + +type ModalKind = 'settings' | 'clean' | 'history' | 'logs' | { node: string } | null; + +/** + * Admin « Nettoyage Docker » : tuiles (statut + au plus un switch), détails et + * actions dans des modales. Le nettoyage manuel passe par une simulation puis une + * confirmation explicite. + */ +export function DockerGcPanel() { + const toast = useToast(); + const [state, setState] = useState(null); + const [nodes, setNodes] = useState(null); + const [alertPct, setAlertPct] = useState(85); + const [runs, setRuns] = useState(null); + const [denied, setDenied] = useState(false); + const [modal, setModal] = useState(null); + const [busy, setBusy] = useState(false); + + async function loadState() { + try { + setState(await api.dockerGcGet()); + } catch (e) { + const msg = String((e as Error).message || e); + if (/admin|réservé|403/i.test(msg)) setDenied(true); + } + } + async function loadNodes() { + try { + const r = await api.dockerGcNodes(); + setNodes(r.nodes); + setAlertPct(r.alert_percent); + } catch { + setNodes([]); + } + } + async function loadRuns() { + try { + setRuns((await api.dockerGcRuns(50)).runs); + } catch { + setRuns([]); + } + } + + useEffect(() => { + void loadState(); + void loadNodes(); + void loadRuns(); + const t = setInterval(() => { + void loadState(); + void loadRuns(); + }, 15000); + return () => clearInterval(t); + }, []); + + async function save(next: DockerGcSettings, okTitle = 'Réglages enregistrés') { + setBusy(true); + try { + const r = await api.dockerGcSave(next); + setState((s) => ({ ...(s as DockerGcState), ...r })); + toast.push({ title: okTitle, tone: 'ok' }); + void loadState(); + return true; + } catch (e) { + toast.push({ title: 'Échec', detail: String((e as Error).message || e), tone: 'danger' }); + return false; + } finally { + setBusy(false); + } + } + + if (denied) return Réservé à l’administrateur d’instance.; + if (!state) { + return ( +
+ + + +
+ ); + } + + const s = state.settings; + const auto = s.enabled + ? status('On', 'ok', `Prochain : ${formatWhen(state.next_run_at)}`) + : status('Off', 'warn', 'Désactivé'); + const last = state.last_run; + const clean = state.running + ? status('En cours', 'accent', 'Nettoyage en cours…') + : last + ? status( + last.ok ? 'OK' : 'Erreur', + last.ok ? 'ok' : 'warn', + `${formatBytes(last.freed_bytes)} libérés · ${formatWhen(last.finished_at)}`, + ) + : status('Prêt', 'neutral', 'Simulation avant suppression'); + const errorsRecent = (runs ?? []).slice(0, 10).filter((r) => !r.ok).length; + const history = !runs + ? undefined + : runs.length === 0 + ? status('0', 'neutral', 'Aucun passage') + : status( + String(runs.length), + errorsRecent ? 'warn' : 'ok', + errorsRecent + ? `${errorsRecent} en erreur récemment` + : `${formatBytes(runs.reduce((a, r) => a + r.freed_bytes, 0))} libérés au total`, + ); + const logs = state.log_rotation_active + ? status('On', 'ok', `${s.log_max_size_mb} Mo × ${s.log_max_files}`) + : status('Off', 'warn', 'Recommandé'); + + const nodeModal = typeof modal === 'object' && modal ? nodes?.find((n) => n.node_id === modal.node) : null; + + return ( +
+ + } + badge={auto.badge} + subtitle={auto.subtitle} + onClick={() => setModal('settings')} + footer={ + + void save( + { ...s, enabled: !s.enabled }, + s.enabled ? 'Nettoyage automatique désactivé' : 'Nettoyage automatique activé', + ) + } + /> + } + /> + } + badge={clean.badge} + subtitle={clean.subtitle} + onClick={() => setModal('clean')} + /> + } + badge={history?.badge} + subtitle={history?.subtitle} + onClick={() => { + void loadRuns(); + setModal('history'); + }} + /> + } + badge={logs.badge} + subtitle={logs.subtitle} + onClick={() => setModal('logs')} + /> + {nodes === null ? ( + + ) : ( + nodes.map((n, i) => { + const pct = n.disk?.percent ?? 0; + const st = n.ok + ? status( + `${Math.round(pct)} %`, + diskTone(pct, alertPct), + `${formatBytes(n.disk?.avail_bytes)} libres`, + ) + : status('KO', 'danger', n.error || 'Injoignable'); + return ( + } + badge={st.badge} + subtitle={st.subtitle} + onClick={() => setModal({ node: n.node_id })} + /> + ); + }) + )} + + + setModal(null)} + onSave={async (next) => { + if (await save(next)) setModal(null); + }} + /> + setModal(null)} + onStarted={() => { + setModal(null); + void loadState(); + setTimeout(() => { + void loadState(); + void loadRuns(); + void loadNodes(); + }, 5000); + }} + /> + setModal(null)} /> + setModal(null)} + onSave={async (next) => { + if (await save(next)) setModal(null); + }} + /> + setModal(null)} + title={nodeModal?.name || 'Nœud'} + description="Disque de la racine Docker" + size="sm" + > + {nodeModal?.ok && nodeModal.disk ? ( +
+ +
+ + + + + {nodeModal.disk.root_dir || '—'}} /> + + +
+ {nodeModal.disk.legacy_guard === 'present' && ( + + Ancien cron /etc/cron.d/devforge-disk-guard détecté : il + sera désactivé automatiquement (remplacé par ce nettoyage). + + )} + {nodeModal.disk.legacy_guard === 'container' && ( + + DevForge tourne en conteneur sur ce nœud : l’ancien cron{' '} + /etc/cron.d/devforge-disk-guard de l’hôte n’est pas + visible d’ici. S’il existe, supprime-le sur l’hôte. + + )} +
+ ) : ( + {nodeModal?.error || 'Nœud injoignable'} + )} + +
+ ); +} + +function SettingsModal({ + open, + settings, + cron, + busy, + onClose, + onSave, +}: { + open: boolean; + settings: DockerGcSettings; + cron: string; + busy: boolean; + onClose: () => void; + onSave: (s: DockerGcSettings) => void; +}) { + const [draft, setDraft] = useState(settings); + const [useCron, setUseCron] = useState(!!settings.cron); + useEffect(() => { + if (open) { + setDraft(settings); + setUseCron(!!settings.cron); + } + }, [open]); + + const num = (k: keyof DockerGcSettings) => (e: Event) => + setDraft((d) => ({ ...d, [k]: Number((e.target as HTMLInputElement).value) })); + + return ( + + + +
+ } + > +
+
+ Expression cron personnalisée + setUseCron((v) => !v)} /> +
+ {useCron ? ( + setDraft((d) => ({ ...d, cron: (e.target as HTMLInputElement).value }))} + /> + ) : ( +
+ + +
+ )} +
+ + + +
+ + Jamais touchés : volumes, conteneurs (même arrêtés) et leurs images, image du + déploiement en cours. Une urgence vide aussi tout le cache de build inutilisé. + +
+ + ); +} + +function CleanModal({ + open, + running, + onClose, + onStarted, +}: { + open: boolean; + running: boolean; + onClose: () => void; + onStarted: () => void; +}) { + const toast = useToast(); + const [preview, setPreview] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [confirm, setConfirm] = useState(false); + const [busy, setBusy] = useState(false); + + useEffect(() => { + if (!open) return; + setConfirm(false); + setPreview(null); + setError(null); + setLoading(true); + api + .dockerGcPreview() + .then(setPreview) + .catch((e) => setError(String((e as Error).message || e))) + .finally(() => setLoading(false)); + }, [open]); + + const total = (preview?.nodes ?? []).reduce((a, n) => a + (n.total_bytes ?? 0), 0); + const count = (preview?.nodes ?? []).reduce((a, n) => a + (n.images?.length ?? 0), 0); + + async function run() { + setBusy(true); + try { + await api.dockerGcRun(); + toast.push({ title: 'Nettoyage lancé', detail: 'Suivi dans l’historique', tone: 'ok' }); + onStarted(); + } catch (e) { + toast.push({ title: 'Échec', detail: String((e as Error).message || e), tone: 'danger' }); + } finally { + setBusy(false); + } + } + + return ( + !busy && onClose()} + title={confirm ? 'Confirmer le nettoyage ?' : 'Nettoyer maintenant'} + description={confirm ? undefined : 'Simulation : rien n’est supprimé tant que tu ne confirmes pas.'} + size="lg" + fullOnMobile + footer={ +
+ {confirm ? ( + <> + + + + ) : ( + <> + + + + )} +
+ } + > + {confirm ? ( +
+

+ Sur {preview?.nodes.filter((n) => n.ok).length ?? 0} nœud(s) : {count} image(s) et le + cache de build de plus de {preview?.retention_hours} h, soit environ{' '} + {formatBytes(total)}. +

+ + Les builds suivants seront plus longs (cache vidé) et les images supprimées ne + pourront plus servir de rollback. Volumes et conteneurs ne sont jamais touchés. + +
+ ) : loading ? ( +
+ Inventaire des nœuds… +
+ ) : error ? ( + {error} + ) : ( +
+

+ Estimation : {formatBytes(total)}{' '} + ({count} image(s) + cache de build). Gardé : {preview?.keep_per_project} image(s) par + projet, rétention {preview?.retention_hours} h. +

+ {(preview?.nodes ?? []).map((n) => ( +
+
+ {n.name} + {n.ok ? ( + {formatBytes(n.total_bytes)} + ) : ( + KO + )} +
+ {!n.ok ? ( +

{n.error}

+ ) : ( +
+
+ Cache de build : {formatBytes(n.cache_bytes)} + {n.build_cache_total ? ` (total ${n.build_cache_total.size})` : ''} + {(n.cache ?? []).length > 0 && + ` · ${(n.cache ?? []).map((c) => `${c.builder.replace('buildkit:', '')} ${formatBytes(c.bytes)}`).join(', ')}`} +
+
+ Protégé : {n.kept_count} image(s) gardée(s), {n.containers_untouched}{' '} + conteneur(s) non touché(s). +
+ {(n.images ?? []).length === 0 ? ( +
Aucune image à supprimer.
+ ) : ( +
    + {(n.images ?? []).map((img) => ( +
  • +
    +
    + {img.refs.length ? img.refs.join(', ') : ` ${img.id}`} +
    +
    {img.reason}
    +
    + {formatBytes(img.size)} +
    • + ))} +
+ )} +
+ )} +
+ ))} +
+ )} + + ); +} + +function HistoryModal({ + open, + runs, + onClose, +}: { + open: boolean; + runs: DockerGcRun[] | null; + onClose: () => void; +}) { + return ( + + {!runs ? ( + + ) : runs.length === 0 ? ( +

Aucun nettoyage pour l’instant.

+ ) : ( +
    + {runs.map((r) => ( +
  • +
    + + {formatWhen(r.finished_at)} · {r.node_name} + +
    + + {TRIGGER_LABEL[r.trigger] || r.trigger} + + {r.ok ? 'OK' : 'Erreur'} +
    +
    +
    + {formatBytes(r.freed_bytes)} libérés · {r.images_removed} image(s) ·{' '} + {(r.duration_ms / 1000).toFixed(1)} s + {r.disk_percent_before != null && r.disk_percent_after != null + ? ` · disque ${r.disk_percent_before} % → ${r.disk_percent_after} %` + : ''} +
    + {r.errors.length > 0 && ( +
      + {r.errors.slice(0, 5).map((e, i) => ( +
    • + {e} +
      • + ))} +
    + )} +
    • + ))} +
+ )} + + ); +} + +function LogsModal({ + open, + settings, + active, + nodes, + busy, + onClose, + onSave, +}: { + open: boolean; + settings: DockerGcSettings; + active: boolean; + nodes: DockerGcNodeDisk[]; + busy: boolean; + onClose: () => void; + onSave: (s: DockerGcSettings) => void; +}) { + const [draft, setDraft] = useState(settings); + useEffect(() => { + if (open) setDraft(settings); + }, [open]); + const unsupported = nodes.filter((n) => n.ok && n.log_rotation_supported === false); + const snippet = `{\n "log-driver": "json-file",\n "log-opts": { "max-size": "${draft.log_max_size_mb}m", "max-file": "${draft.log_max_files}" }\n}`; + + return ( + + + +
+ } + > +
+
+
+
Limiter les logs
+
+ {active ? 'Actif' : 'Inactif'} · ajoute --log-opt au lancement +
+
+ setDraft((d) => ({ ...d, log_rotation: !d.log_rotation }))} + /> +
+
+ setDraft((d) => ({ ...d, log_max_size_mb: Number((e.target as HTMLInputElement).value) }))} + /> + setDraft((d) => ({ ...d, log_max_files: Number((e.target as HTMLInputElement).value) }))} + /> +
+ {unsupported.length > 0 && ( + + Driver de logs incompatible sur : {unsupported.map((n) => `${n.name} (${n.disk?.logging_driver})`).join(', ')}. + + )} +
+ + Pour tous les conteneurs (optionnel, manuel) + +

+ DevForge ne modifie pas /etc/docker/daemon.json. Pour l’appliquer à tous les + conteneurs de l’hôte, ajoute ceci puis redémarre Docker (les conteneurs existants + gardent leur réglage jusqu’à leur recréation) : +

+ 
{snippet}
+
+
+ + ); +} diff --git a/apps/web/src/components/ui/HubTile.tsx b/apps/web/src/components/ui/HubTile.tsx index b1ad29f201..5c2ff2bf31 100644 --- a/apps/web/src/components/ui/HubTile.tsx +++ b/apps/web/src/components/ui/HubTile.tsx @@ -305,6 +305,23 @@ export function HubIcon({ name, size = 24 }: { name: string; size?: number }) { ), + trash: ( + + + + ), + disk: ( + + + + + ), + clock: ( + + + + + ), cpu: ( diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index 9bda91b986..991784d3ce 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -2006,6 +2006,24 @@ export const api = { body: JSON.stringify(body), }), + dockerGcGet: () => request('/admin/docker-gc'), + dockerGcSave: (settings: DockerGcSettings) => + request('/admin/docker-gc', { + method: 'PUT', + body: JSON.stringify(settings), + }), + dockerGcNodes: () => + request<{ nodes: DockerGcNodeDisk[]; alert_percent: number }>('/admin/docker-gc/nodes'), + dockerGcPreview: () => + request('/admin/docker-gc/preview', { method: 'POST', body: '{}' }), + dockerGcRun: () => + request<{ ok: boolean; started: boolean }>('/admin/docker-gc/run', { + method: 'POST', + body: JSON.stringify({ confirm: true }), + }), + dockerGcRuns: (limit = 50) => + request<{ runs: DockerGcRun[] }>(`/admin/docker-gc/runs?limit=${limit}`), + acmeSettings: () => request('/settings/acme'), saveAcmeSettings: (acme_email: string) => @@ -2919,3 +2937,96 @@ export type AppReposResponse = { refresh_error?: string | null; }; export type ProjectGithubLink = { ok: boolean; linked: boolean; lost: boolean; installation_id?: number; repo_id?: number | null }; + +export type DockerGcSettings = { + enabled: boolean; + frequency_hours: number; + hour: number; + cron: string | null; + timezone: string; + retention_hours: number; + keep_per_project: number; + disk_alert_percent: number; + log_rotation: boolean; + log_max_size_mb: number; + log_max_files: number; +}; + +export type DockerGcState = { + settings: DockerGcSettings; + cron_expression: string; + next_run_at: string | null; + running?: boolean; + log_rotation_active?: boolean; + last_run?: { finished_at: string; ok: boolean; freed_bytes: number; trigger: string } | null; +}; + +export type DockerGcDisk = { + root_dir: string; + logging_driver: string; + docker_version: string; + total_bytes: number; + used_bytes: number; + avail_bytes: number; + percent: number; + /** Ancien cron devforge-disk-guard : present / absent / container. */ + legacy_guard?: string; +}; + +export type DockerGcNodeDisk = { + node_id: string; + name: string; + ok: boolean; + error?: string; + alert?: boolean; + disk?: DockerGcDisk; + log_rotation_supported?: boolean; +}; + +export type DockerGcPreviewNode = { + node_id: string; + name: string; + ok: boolean; + error?: string; + disk?: DockerGcDisk; + images?: { + id: string; + refs: string[]; + size: number; + created: string; + project: string | null; + reason: string; + }[]; + images_bytes?: number; + kept_count?: number; + protected_running?: number; + containers_untouched?: number; + builders?: string[]; + cache?: { builder: string; bytes: number }[]; + cache_bytes?: number; + build_cache_total?: { size: string; reclaimable: string } | null; + total_bytes?: number; +}; + +export type DockerGcPreview = { + dry_run: boolean; + retention_hours: number; + keep_per_project: number; + nodes: DockerGcPreviewNode[]; +}; + +export type DockerGcRun = { + id: number; + started_at: string; + finished_at: string; + trigger: 'manual' | 'schedule' | 'emergency' | string; + node_id: string; + node_name: string; + ok: boolean; + freed_bytes: number; + images_removed: number; + duration_ms: number; + disk_percent_before: number | null; + disk_percent_after: number | null; + errors: string[]; +}; diff --git a/crates/deploy/src/docker.rs b/crates/deploy/src/docker.rs index 9515a345ca..cdb67c832b 100644 --- a/crates/deploy/src/docker.rs +++ b/crates/deploy/src/docker.rs @@ -2,6 +2,34 @@ use serde_json::{Map, Value}; +/// Rotation des logs des conteneurs lancés par DevForge : `(max-size Mo, max-file)`. +/// `None` = options du démon Docker inchangées. Réglée depuis l'admin +/// (Nettoyage Docker) ; appliquée aux prochains `docker run`. +static CONTAINER_LOG_ROTATION: std::sync::RwLock> = std::sync::RwLock::new(None); + +pub fn set_container_log_rotation(rotation: Option<(u32, u32)>) { + if let Ok(mut g) = CONTAINER_LOG_ROTATION.write() { + *g = rotation.filter(|(size, files)| *size > 0 && *files > 0); + } +} + +pub fn container_log_rotation() -> Option<(u32, u32)> { + CONTAINER_LOG_ROTATION.read().ok().and_then(|g| *g) +} + +/// Arguments `--log-opt` (drivers `json-file` / `local`). +pub fn log_rotation_args(rotation: Option<(u32, u32)>) -> Vec { + match rotation { + Some((size, files)) if size > 0 && files > 0 => vec![ + "--log-opt".into(), + format!("max-size={size}m"), + "--log-opt".into(), + format!("max-file={files}"), + ], + _ => Vec::new(), + } +} + pub fn docker_build(workdir: &str, image: &str, dockerfile: &str) -> String { format!( "docker build -f {} -t {} {}", @@ -142,6 +170,7 @@ pub fn docker_run_ex( "--restart".into(), "unless-stopped".into(), ]; + args.extend(log_rotation_args(container_log_rotation())); if gpu_nvidia { args.push("--gpus".into()); args.push("all".into()); @@ -1439,7 +1468,7 @@ pub fn classify_ready_probe(out: &str) -> ReadyProbe { } } -fn shell_escape(s: &str) -> String { +pub(crate) fn shell_escape(s: &str) -> String { if s.chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/' | ':' | '=')) { @@ -1451,6 +1480,17 @@ fn shell_escape(s: &str) -> String { #[cfg(test)] mod tests { + + #[test] + fn log_rotation_args_only_when_configured() { + assert!(super::log_rotation_args(None).is_empty()); + assert!(super::log_rotation_args(Some((0, 3))).is_empty()); + assert_eq!( + super::log_rotation_args(Some((10, 3))), + vec!["--log-opt", "max-size=10m", "--log-opt", "max-file=3"] + ); + } + #[test] fn acme_email_validation_and_traefik_command() { assert!(is_valid_acme_email("bobdivx@gmail.com")); diff --git a/crates/deploy/src/lib.rs b/crates/deploy/src/lib.rs index 626e8fa134..c2d511ef89 100644 --- a/crates/deploy/src/lib.rs +++ b/crates/deploy/src/lib.rs @@ -688,7 +688,7 @@ impl DeployFacade { format!("dfside-{}", app_container.trim_start_matches("df-")) } - fn image_name(project_uuid: &str) -> String { + pub fn image_name(project_uuid: &str) -> String { format!( "df-{}:latest", project_uuid.chars().take(12).collect::() @@ -1340,6 +1340,17 @@ impl DeployFacade { } }; + if build_ok && !cfg!(windows) { + // Tag de rollback : `:latest` est réécrit à chaque build, ce tag garde + // l'image de ce déploiement (le nettoyage Docker en conserve N par projet). + let cmd = rollback_tag_command(&image, unix_now_secs()); + if let Ok(r) = self.executor.exec(server, "", &cmd, 30).await { + if r.ok && !r.output.trim().is_empty() { + logs.push_str(&format!("[image] {}\n", r.output.trim())); + } + } + } + if !build_ok { logs.push_str( "[fallback] image Docker non démarrée — pas de processus local (il ne crée pas de route publique)\n", @@ -1370,7 +1381,9 @@ impl DeployFacade { logs: &mut LogBuffer, label: &str, ) -> Result { - let fut = self.executor.exec(server_id, workdir, command, timeout_secs); + let fut = self + .executor + .exec(server_id, workdir, command, timeout_secs); tokio::pin!(fut); let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -2306,8 +2319,35 @@ if (-not $candidates) { Write-Error 'docker missing'; exit 1 } } } +fn unix_now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// `df-abc:latest` → tag `df-abc:r` si l'image existe (sinon rien). +pub fn rollback_tag_command(image: &str, secs: u64) -> String { + let repo = image.split(':').next().unwrap_or(image); + let target = format!("{repo}:r{secs}"); + format!( + "docker image inspect {img} >/dev/null 2>&1 && docker tag {img} {t} && echo 'tag rollback {t}' || true", + img = docker::shell_escape(image), + t = docker::shell_escape(&target), + ) +} + #[cfg(test)] mod tests { + + #[test] + fn rollback_tag_targets_same_repo() { + let cmd = rollback_tag_command("df-abc123def456:latest", 1_760_000_000); + assert!(cmd.contains("docker tag df-abc123def456:latest df-abc123def456:r1760000000")); + assert!(cmd.starts_with("docker image inspect df-abc123def456:latest")); + assert!(!cmd.contains(" rm")); + } + use super::*; #[test] diff --git a/docs/DOCKER_GC.md b/docs/DOCKER_GC.md new file mode 100644 index 0000000000..095cd04df7 --- /dev/null +++ b/docs/DOCKER_GC.md @@ -0,0 +1,42 @@ +# Nettoyage Docker + +Admin → **Nettoyage Docker** (`/app/admin?tab=docker`). Exécuté par le leader sur +chaque nœud via l'executor du cluster (local ou `POST /internal/exec` des workers). + +## Défauts + +| Réglage | Défaut | +|---------|--------| +| Activé | oui | +| Planification | chaque jour à 03:00, heure de Paris (calculée explicitement en `Europe/Paris`, quel que soit le fuseau de l'hôte) | +| Rétention cache de build + images inutilisées | 24 h | +| Images gardées par projet (rollback) | 3 | +| Seuil d'alerte disque | 85 % → nettoyage d'urgence du nœud (au plus 1×/h) | +| Rotation des logs (`--log-opt`) | désactivée (proposée) | + +## Ce qui est supprimé + +- cache BuildKit : `docker buildx prune --builder --filter until=` par builder + enregistré, et `buildctl prune --keep-duration ` dans chaque conteneur + `buildx_buildkit_*` en cours (builders non enregistrés pour l'utilisateur, cas ZimaOS) ; +- images inutilisées (aucun conteneur) créées il y a plus que la rétention ; +- images d'anciens déploiements d'un projet au-delà des N plus récentes (chaque déploiement + réussi pose un tag `df-:r` en plus de `:latest`). + +Une urgence vide tout le cache de build inutilisé (sans filtre d'âge) ; les règles des images +ne changent pas. + +## Jamais touché + +Volumes, conteneurs (même arrêtés) et les images qu'ils utilisent, `df-…:latest`. +Pas de `system prune`, `container prune`, `volume …` ni `image rm -f`. + +## Environnement + +- `DOCKER_CONFIG` non inscriptible (ZimaOS : `/root/.docker` en lecture seule) → + `/tmp/devforge-docker-gc`. +- Ancien cron `/etc/cron.d/devforge-disk-guard` (pré-v2, cassé sous dcron et qui supprimait + les conteneurs arrêtés) : déplacé vers `/DATA/.devforge/state/devforge-disk-guard.cron.disabled` + au démarrage du leader quand l'hôte est visible. Si DevForge tourne en conteneur, l'admin le + signale : à retirer à la main sur l'hôte. +- `daemon.json` n'est jamais modifié : l'admin affiche l'extrait à appliquer manuellement.