diff --git a/tof/src/main.rs b/tof/src/main.rs index a2e7cc90..22cdc80f 100644 --- a/tof/src/main.rs +++ b/tof/src/main.rs @@ -259,6 +259,18 @@ fn quiet_period(hz: u8) -> Duration { Duration::from_secs_f64(1.0 / f64::from(hz.max(1))).saturating_sub(POLL_GUARD) } +/// How long the poll may be told "no frame" before the sensor is called stuck. +/// +/// This is the one failure `data_ready` cannot see: a sensor whose ranging engine has +/// wedged still ACKs the bus, so the register read succeeds and the answer is `false` +/// forever — without a clock on that answer the daemon streams silence for the rest of +/// the boot and never re-opens the sensor. Scaled to the frame period so `--hz 1` gets +/// the same margin in frames as the shipped 15 Hz gets in time. +fn frame_watchdog(hz: u8) -> Duration { + let period = Duration::from_secs_f64(1.0 / f64::from(hz.max(1))); + Duration::from_secs(2).max(period * 4) +} + /// Bring the sensor up and stream from it, forever, with a backoff between /// attempts. Never returns until shutdown. fn sensor_loop( @@ -292,6 +304,12 @@ fn sensor_loop( // actually arrived. let mut quiet_until: Option = None; + // The clock for the one failure the poll cannot see — see + // [`frame_watchdog`]. Armed at open, so a sensor that never + // produces a first frame is reopened on the same bound. + let mut last_frame = Instant::now(); + let watchdog = frame_watchdog(hz); + // Stream until the sensor stops answering, then fall through to // the backoff and try the whole bring-up again. while !shutdown.load(Ordering::Acquire) { @@ -308,6 +326,7 @@ fn sensor_loop( match sensor.data_ready() { Ok(true) => match sensor.read_frame() { Ok(frame) => { + last_frame = Instant::now(); quiet_until = Some(Instant::now() + quiet); seq += 1; // No subscribers is the normal state — nobody is @@ -328,7 +347,16 @@ fn sensor_loop( break; } }, - Ok(false) => std::thread::sleep(POLL), + Ok(false) => { + if last_frame.elapsed() >= watchdog { + tracing::warn!( + ?watchdog, + "no frames on a sensor that still answers; reopening it" + ); + break; + } + std::thread::sleep(POLL); + } Err(e) => { tracing::warn!(error = %e, "lost the sensor"); break; @@ -801,6 +829,17 @@ mod tests { assert!(RETRY_MIN < RETRY_MAX); } + /// The watchdog's two ends: at the shipped rate it tolerates a couple of seconds + /// of "no frame" — long enough that a mere late frame never trips it — and at the + /// slowest rate it scales with the period, so `--hz 1` is not called stuck one + /// second into a one-second frame. + #[test] + fn the_frame_watchdog_scales_with_the_period() { + assert_eq!(frame_watchdog(15), Duration::from_secs(2)); + assert_eq!(frame_watchdog(1), Duration::from_secs(4)); + assert_eq!(frame_watchdog(0), frame_watchdog(1), "hz 0 means 1"); + } + /// **The saving, as arithmetic rather than as a claim in a comment.** /// /// At the shipped rate a frame used to cost seven `data_ready` reads to