diff --git a/CHANGELOG.md b/CHANGELOG.md index b2692a2..d3de8dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- A failing `sinfo`, `sacctmgr` or `squeue` is reported instead of being read as a cluster that has no partitions, nodes or QoS. Startup now stops with the message `squeue` printed, and a probe that fails once the app is up says so in the flash bar rather than leaving the sidebar quietly empty. + ## [0.2.0] - 2026-08-28 ### Added diff --git a/src/backend/commands.rs b/src/backend/commands.rs index 8684fbe..efc6808 100644 --- a/src/backend/commands.rs +++ b/src/backend/commands.rs @@ -5,7 +5,8 @@ use std::collections::HashMap; pub fn check_slurm_available() -> Result<()> { use std::process::Command as StdCommand; match StdCommand::new("squeue").arg("--version").output() { - Ok(_) => Ok(()), + Ok(out) if out.status.success() => Ok(()), + Ok(out) => color_eyre::eyre::bail!(failure_message("squeue --version", &out)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => { color_eyre::eyre::bail!( "SLURM tools not found.\n\ @@ -24,6 +25,37 @@ pub async fn run_cmd(program: &str, args: Vec) -> Result { Ok(out) } +/// What a command printed when it exited non-zero, or its exit status when it +/// printed nothing. +fn failure_message(program: &str, out: &Output) -> String { + let stderr = String::from_utf8_lossy(&out.stderr); + let detail = stderr.trim(); + if detail.is_empty() { + format!("{} exited with {}", program, out.status) + } else { + format!("{}: {}", program, detail) + } +} + +/// Run a command and collect its non-empty output lines. A non-zero exit is an +/// error rather than an empty list, so an unreachable controller cannot be +/// mistaken for a cluster that has none of whatever was asked for. +async fn collect_lines(program: &str, args: Vec) -> Result> { + let out = run_cmd(program, args) + .await + .map_err(|e| color_eyre::eyre::eyre!("failed to run {}: {}", program, e))?; + + if !out.status.success() { + color_eyre::eyre::bail!(failure_message(program, &out)); + } + + Ok(String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty()) + .collect()) +} + /// Parsed result of `scontrol show job -o`. #[derive(Clone)] pub struct JobDetail { @@ -90,42 +122,23 @@ pub async fn cancel_jobs(job_ids: Vec) -> Result<()> { Ok(()) } -pub async fn list_partitions() -> Vec { - let out = match run_cmd("sinfo", vec!["-h".into(), "-o".into(), "%R".into()]).await { - Ok(o) => o, - Err(_) => return Vec::new(), - }; - - String::from_utf8_lossy(&out.stdout) - .lines() - .map(|l| l.trim().to_string()) - .filter(|l| !l.is_empty()) - .collect() +pub async fn list_partitions() -> Result> { + collect_lines("sinfo", vec!["-h".into(), "-o".into(), "%R".into()]).await } -pub async fn list_nodes() -> Vec { - let out = match run_cmd( +pub async fn list_nodes() -> Result> { + let mut nodes = collect_lines( "sinfo", vec!["-h".into(), "-N".into(), "-o".into(), "%N".into()], ) - .await - { - Ok(o) => o, - Err(_) => return Vec::new(), - }; - - let mut nodes: Vec = String::from_utf8_lossy(&out.stdout) - .lines() - .map(|l| l.trim().to_string()) - .filter(|l| !l.is_empty()) - .collect(); + .await?; nodes.sort(); nodes.dedup(); - nodes + Ok(nodes) } -pub async fn list_qos() -> Vec { - let out = match run_cmd( +pub async fn list_qos() -> Result> { + collect_lines( "sacctmgr", vec![ "-n".into(), @@ -135,16 +148,61 @@ pub async fn list_qos() -> Vec { ], ) .await - { - Ok(o) if o.status.success() => o, - // No accounting DB / QoS on this cluster: show an empty list rather - // than inventing site-specific names that don't exist here. - _ => return Vec::new(), - }; - - String::from_utf8_lossy(&out.stdout) - .lines() - .map(|l| l.trim().to_string()) - .filter(|l| !l.is_empty()) - .collect() +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + fn block_on(fut: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(fut) + } + + fn sh(script: &str) -> Result> { + block_on(collect_lines("sh", vec!["-c".into(), script.into()])) + } + + #[test] + fn collects_the_non_empty_output_lines() { + let lines = sh("printf 'gpu\\n\\n cpu \\n'").unwrap(); + assert_eq!(lines, vec!["gpu", "cpu"]); + } + + #[test] + fn a_non_zero_exit_is_an_error_not_an_empty_list() { + let err = + sh("echo 'slurm_load_partitions: Unable to contact slurm controller' >&2; exit 1") + .unwrap_err(); + assert!( + err.to_string() + .contains("Unable to contact slurm controller"), + "error was {}", + err + ); + } + + #[test] + fn a_silent_failure_reports_the_exit_status() { + let err = sh("exit 2").unwrap_err(); + assert!(err.to_string().contains("exited with"), "error was {}", err); + } + + #[test] + fn a_cluster_with_nothing_to_list_is_still_ok() { + assert_eq!(sh("true").unwrap(), Vec::::new()); + } + + #[test] + fn a_missing_command_is_an_error() { + let err = block_on(collect_lines("sqwatch-no-such-command", Vec::new())).unwrap_err(); + assert!( + err.to_string().contains("failed to run"), + "error was {}", + err + ); + } } diff --git a/src/dashboard.rs b/src/dashboard.rs index 46d1df0..f9abd08 100644 --- a/src/dashboard.rs +++ b/src/dashboard.rs @@ -108,9 +108,16 @@ impl Dashboard { params.name_pattern = None; } - let known_partitions = rt.block_on(list_partitions()); - let known_qos = rt.block_on(list_qos()); - let known_nodes = rt.block_on(list_nodes()); + // An empty sidebar section is a real answer on some clusters, so a + // failed probe has to say so rather than blend in with one. + let mut probe_errors = Vec::new(); + let known_partitions = Self::probe( + "partitions", + rt.block_on(list_partitions()), + &mut probe_errors, + ); + let known_qos = Self::probe("QoS", rt.block_on(list_qos()), &mut probe_errors); + let known_nodes = Self::probe("nodes", rt.block_on(list_nodes()), &mut probe_errors); let known_states = JobState::all_known(); let (visible_fields, sort_fields) = load_columns().unwrap_or_else(|| { @@ -135,7 +142,7 @@ impl Dashboard { .map(|(i, def)| CustomOutputWidget::new(i, def.title.clone(), def.filename.clone())) .collect(); - Ok(Self { + let mut dashboard = Self { alive: true, input: InputLoop::start(InputConfig::default()), table: JobTable::new(), @@ -166,7 +173,24 @@ impl Dashboard { job_detail_resolver: JobDetailResolver::new(), job_fetcher: JobFetcher::new(), pending_filter_apply: false, - }) + }; + + if !probe_errors.is_empty() { + dashboard.flash(probe_errors.join("; "), 10); + } + + Ok(dashboard) + } + + /// Keep a startup probe's list, or note why the sidebar section is empty. + fn probe(label: &str, result: Result>, errors: &mut Vec) -> Vec { + match result { + Ok(values) => values, + Err(e) => { + errors.push(format!("no {} ({})", label, e)); + Vec::new() + } + } } pub fn run(