ci: read every sentence of the gpsd receiver logs against a recorded tally

.github/workflows/ci.yml

@@ -79,6 +79,34 @@ jobs:
       - run: cargo clippy -p kinavis-kernel -p kinavis --all-targets --no-default-features --features "libm serde"
       - run: cargo doc --workspace --no-deps --all-features
 
+  real-data:
+    name: Real receiver logs
+    runs-on: ubuntu-latest
+    env:
+      # gpsd's regression logs at a pinned commit, so the tally changes only
+      # with a change here.
+      GPSD_COMMIT: 43362cd24a43864ceac7b8c70f29be2053dd7b0d
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+      # Data only, `test/daemon` alone: about two hundred logs of real receivers.
+      - name: Fetch the gpsd logs
+        run: |
+          git init -q "$RUNNER_TEMP/gpsd"
+          cd "$RUNNER_TEMP/gpsd"
+          git remote add origin https://gitlab.com/gpsd/gpsd.git
+          git sparse-checkout set test/daemon
+          git fetch -q --depth 1 origin "$GPSD_COMMIT"
+          git checkout -q FETCH_HEAD
+      # Every sentence read as the navigation system reads it, against the
+      # tally in examples/tests/gpsd_corpus.expected.
+      - run: cargo test -p kinavis-examples --test gpsd_corpus -- --ignored
+        env:
+          GPSD_LOGS: ${{ runner.temp }}/gpsd/test/daemon
+
   no-std:
     name: Build for a bare-metal target
     runs-on: ubuntu-latest

README.md

@@ -37,9 +37,9 @@ collision assessment and a bridge alert out.
   the same plan: the `std` and the pure-Rust `libm` maths are held to agree
   within 1e-13, and the estimator is a pure function that replays a voyage
   step by step.
-- **Verified.** The parsers are fuzzed and tested on real receivers' output
-  from the gpsd and Signal K logs; the algorithms are checked against
-  published reference values — NOAA's WMM test points, Vincenty's test
+- **Verified.** The parsers are fuzzed, and CI reads every sentence of
+  gpsd's two hundred logs of real receivers against a recorded tally; the
+  algorithms are checked against published reference values — NOAA's WMM test points, Vincenty's test
   lines, PROJ's datum shifts; and the filters pass Monte Carlo consistency
   tests.
 

examples/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kinavis-examples"
-description = "Runnable demonstrations of the KINAVIS crates. Not published."
+description = "Runnable demonstrations of the KINAVIS crates, and their check against real receiver logs. Not published."
 publish = false
 version = "0.0.0"
 authors.workspace = true

examples/README.md

@@ -65,6 +65,30 @@ UTC       position                       COG    track    steer        off track
 09:47:37  52°51.009'N 005°18.817'E   230.5°T  235.7°T  235.7°T      0.19 M Port  8.82 M  off track: 0.19 M > 0.10 M
 ```
 
+## Every gpsd log, in CI
+
+`tests/gpsd_corpus.rs` reads every sentence of the gpsd regression logs —
+about two hundred logs of real receivers, fetched at a pinned gpsd commit —
+and counts what became of each: read as which sentence, refused for which
+error, a fix or not, which AIS message. CI fails when a count changes, so a
+parser change shows its effect on real receivers in the diff of
+[`tests/gpsd_corpus.expected`](tests/gpsd_corpus.expected):
+
+```text
+   2659 RMC: fix
+    180 RMC: no fix, NoPosition
+   3026 GGA: position
+    804 AIS: position report, type 1
+    388 refused: BadChecksum
+    483 refused: NoChecksum
+```
+
+To run it locally, with gpsd's `test/daemon` checked out:
+
+```sh
+GPSD_LOGS=path/to/gpsd/test/daemon cargo test -p kinavis-examples --test gpsd_corpus -- --ignored
+```
+
 ## Data
 
 | File | Source | Licence |

examples/lib.rs

@@ -1,6 +1,7 @@
 //! Runnable demonstrations of the KINAVIS crates, on recorded data.
 //!
-//! The library is empty; the demonstrations are the examples of this package.
-//! See `README.md` beside this file.
+//! The library is empty; the demonstrations are the examples of this package,
+//! and `tests/gpsd_corpus.rs` reads the gpsd receiver logs against a recorded
+//! tally. See `README.md` beside this file.
 
 #![no_std]

examples/tests/gpsd_corpus.expected

@@ -0,0 +1,23 @@
+# Tally of the gpsd regression logs (199 logs).
+    142 AIS: AidToNavigation
+    154 AIS: StaticAndVoyageData
+    147 AIS: StaticDataReport
+    346 AIS: Unsupported
+    179 AIS: fragment held
+    804 AIS: position report, type 1
+    228 AIS: position report, type 18
+      3 AIS: position report, type 19
+     12 AIS: position report, type 2
+    147 AIS: position report, type 3
+    513 GGA: no fix
+   3026 GGA: position
+   2375 GLL
+   2659 RMC: fix
+    180 RMC: no fix, NoPosition
+   2714 VTG
+    388 refused: BadChecksum
+     79 refused: BadField
+      8 refused: BadHeader
+    483 refused: NoChecksum
+    413 refused: TooLong
+  24088 unsupported sentence

examples/tests/gpsd_corpus.rs

@@ -0,0 +1,140 @@
+//! Every sentence of the gpsd regression logs, read as the navigation system
+//! would read it, against a recorded tally.
+//!
+//! The logs, about two hundred, are recorded from real receivers and are not
+//! in this repository: CI fetches `test/daemon/` of gpsd at a pinned commit
+//! and names the directory in `GPSD_LOGS`. Ignored by default; run with
+//!
+//! ```text
+//! GPSD_LOGS=path/to/gpsd/test/daemon cargo test -p kinavis-examples --test gpsd_corpus -- --ignored
+//! ```
+//!
+//! The tally counts sentences by what became of them: read as which sentence,
+//! refused for which error, translated into a fix or not, decoded as which AIS
+//! message. A changed count fails the test and prints the new tally; with
+//! `KINAVIS_BLESS=1` the new tally is written as the expected one, to be
+//! reviewed in the diff.
+
+#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
+
+use std::collections::BTreeMap;
+use std::fmt::{Debug, Write as _};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use kinavis::{GnssFix, Instant, Utc};
+use kinavis_ais::{Assembler, Message};
+use kinavis_nmea0183::{parse, Sentence};
+
+const EXPECTED: &str = "tests/gpsd_corpus.expected";
+
+#[test]
+#[ignore = "needs the gpsd logs; see the module documentation"]
+fn the_gpsd_logs_read_as_recorded() {
+    let dir = std::env::var_os("GPSD_LOGS").expect("GPSD_LOGS names gpsd's test/daemon directory");
+    let logs = logs(Path::new(&dir));
+    assert!(logs.len() > 100, "{} logs in {dir:?}", logs.len());
+
+    let mut tally = Tally::default();
+    for log in &logs {
+        read_log(&fs::read(log).unwrap(), &mut tally);
+    }
+    let actual = tally.to_string(logs.len());
+
+    let expected_path = Path::new(env!("CARGO_MANIFEST_DIR")).join(EXPECTED);
+    if std::env::var_os("KINAVIS_BLESS").is_some() {
+        fs::write(&expected_path, &actual).unwrap();
+        return;
+    }
+    let expected = fs::read_to_string(&expected_path).unwrap_or_default();
+    assert!(
+        actual == expected,
+        "the tally changed; review it and rerun with KINAVIS_BLESS=1 to record it\n\n{actual}"
+    );
+}
+
+/// `*.log` files of a directory, in name order.
+fn logs(dir: &Path) -> Vec<PathBuf> {
+    let mut logs: Vec<PathBuf> = fs::read_dir(dir)
+        .unwrap()
+        .map(|entry| entry.unwrap().path())
+        .filter(|path| path.extension().is_some_and(|extension| extension == "log"))
+        .collect();
+    logs.sort();
+    logs
+}
+
+/// Counts by outcome.
+#[derive(Default)]
+struct Tally(BTreeMap<String, u64>);
+
+impl Tally {
+    fn count(&mut self, outcome: String) {
+        *self.0.entry(outcome).or_default() += 1;
+    }
+
+    fn to_string(&self, logs: usize) -> String {
+        let mut out = format!("# Tally of the gpsd regression logs ({logs} logs).\n");
+        for (outcome, count) in &self.0 {
+            writeln!(out, "{count:>7} {outcome}").unwrap();
+        }
+        out
+    }
+}
+
+/// One log, line by line; only lines that start a sentence count. Binary logs
+/// and comments are skipped by that rule.
+fn read_log(bytes: &[u8], tally: &mut Tally) {
+    let mut assembler = Assembler::new();
+    let now = Instant::<Utc>::from_unix_seconds(0);
+    for line in bytes.split(|&byte| byte == b'\n') {
+        let line = line.strip_suffix(b"\r").unwrap_or(line);
+        if !matches!(line.first(), Some(b'$' | b'!')) {
+            continue;
+        }
+        match parse(line) {
+            Err(error) => tally.count(format!("refused: {}", kind(&error))),
+            Ok(Sentence::Rmc(rmc)) => match GnssFix::try_from(rmc) {
+                Ok(_) => tally.count("RMC: fix".to_owned()),
+                Err(error) => tally.count(format!("RMC: no fix, {}", kind(&error))),
+            },
+            Ok(Sentence::Gga(gga)) => {
+                let fix = if gga.fix_type.is_position_fix() {
+                    "position"
+                } else {
+                    "no fix"
+                };
+                tally.count(format!("GGA: {fix}"));
+            }
+            Ok(Sentence::Gll(_)) => tally.count("GLL".to_owned()),
+            Ok(Sentence::Vtg(_)) => tally.count("VTG".to_owned()),
+            Ok(Sentence::Vdm(vdm)) => match assembler.push(&vdm, now) {
+                Ok(None) => tally.count("AIS: fragment held".to_owned()),
+                Ok(Some(bits)) => match Message::decode(&bits) {
+                    Ok(message) => tally.count(format!("AIS: {}", message_kind(&message))),
+                    Err(error) => tally.count(format!("AIS: refused, {}", kind(&error))),
+                },
+                Err(error) => tally.count(format!("AIS: refused, {}", kind(&error))),
+            },
+            Ok(Sentence::Unsupported { .. }) => tally.count("unsupported sentence".to_owned()),
+            Ok(other) => tally.count(format!("other: {}", kind(&other))),
+        }
+    }
+}
+
+/// Variant name of an enum value: its `Debug` form up to the first field.
+fn kind(value: &impl Debug) -> String {
+    let debug = format!("{value:?}");
+    debug
+        .split([' ', '(', '{'])
+        .next()
+        .unwrap_or_default()
+        .to_owned()
+}
+
+fn message_kind(message: &Message) -> String {
+    match message {
+        Message::PositionReport(report) => format!("position report, type {}", report.kind),
+        other => kind(other),
+    }
+}