From 4956345bb59f7ab94d3463b10c5cf905dced34fe Mon Sep 17 00:00:00 2001 From: Ragnarok Date: Fri, 25 Sep 2026 10:49:52 +0300 Subject: [PATCH] ci: read every sentence of the gpsd receiver logs against a recorded tally - examples/tests/gpsd_corpus.rs: each sentence of gpsd's test/daemon logs counted by what became of it (sentence read, error refused, fix or not, AIS message decoded), compared with tests/gpsd_corpus.expected. Ignored by default; KINAVIS_BLESS=1 records a new tally. - CI job "Real receiver logs" fetches test/daemon at a pinned gpsd commit and runs it. - README and examples/README describe the check. --- .github/workflows/ci.yml | 28 ++++++ README.md | 6 +- examples/Cargo.toml | 2 +- examples/README.md | 24 +++++ examples/lib.rs | 5 +- examples/tests/gpsd_corpus.expected | 23 +++++ examples/tests/gpsd_corpus.rs | 140 ++++++++++++++++++++++++++++ 7 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 examples/tests/gpsd_corpus.expected create mode 100644 examples/tests/gpsd_corpus.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbc6016..e5ccf10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,34 @@ jobs: - run: cargo clippy -p kinavis-kernel -p kinavis --all-targets --no-default-features --features "libm serde" - run: cargo doc --workspace --no-deps --all-features + real-data: + name: Real receiver logs + runs-on: ubuntu-latest + env: + # gpsd's regression logs at a pinned commit, so the tally changes only + # with a change here. + GPSD_COMMIT: 43362cd24a43864ceac7b8c70f29be2053dd7b0d + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # v1 + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + # Data only, `test/daemon` alone: about two hundred logs of real receivers. + - name: Fetch the gpsd logs + run: | + git init -q "$RUNNER_TEMP/gpsd" + cd "$RUNNER_TEMP/gpsd" + git remote add origin https://gitlab.com/gpsd/gpsd.git + git sparse-checkout set test/daemon + git fetch -q --depth 1 origin "$GPSD_COMMIT" + git checkout -q FETCH_HEAD + # Every sentence read as the navigation system reads it, against the + # tally in examples/tests/gpsd_corpus.expected. + - run: cargo test -p kinavis-examples --test gpsd_corpus -- --ignored + env: + GPSD_LOGS: ${{ runner.temp }}/gpsd/test/daemon + no-std: name: Build for a bare-metal target runs-on: ubuntu-latest diff --git a/README.md b/README.md index 7e53bcb..ca6b040 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,9 @@ collision assessment and a bridge alert out. the same plan: the `std` and the pure-Rust `libm` maths are held to agree within 1e-13, and the estimator is a pure function that replays a voyage step by step. -- **Verified.** The parsers are fuzzed and tested on real receivers' output - from the gpsd and Signal K logs; the algorithms are checked against - published reference values — NOAA's WMM test points, Vincenty's test +- **Verified.** The parsers are fuzzed, and CI reads every sentence of + gpsd's two hundred logs of real receivers against a recorded tally; the + algorithms are checked against published reference values — NOAA's WMM test points, Vincenty's test lines, PROJ's datum shifts; and the filters pass Monte Carlo consistency tests. diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 82553df..5814111 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kinavis-examples" -description = "Runnable demonstrations of the KINAVIS crates. Not published." +description = "Runnable demonstrations of the KINAVIS crates, and their check against real receiver logs. Not published." publish = false version = "0.0.0" authors.workspace = true diff --git a/examples/README.md b/examples/README.md index 4a488da..5c5aa5a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -65,6 +65,30 @@ UTC position COG track steer off track 09:47:37 52°51.009'N 005°18.817'E 230.5°T 235.7°T 235.7°T 0.19 M Port 8.82 M off track: 0.19 M > 0.10 M ``` +## Every gpsd log, in CI + +`tests/gpsd_corpus.rs` reads every sentence of the gpsd regression logs — +about two hundred logs of real receivers, fetched at a pinned gpsd commit — +and counts what became of each: read as which sentence, refused for which +error, a fix or not, which AIS message. CI fails when a count changes, so a +parser change shows its effect on real receivers in the diff of +[`tests/gpsd_corpus.expected`](tests/gpsd_corpus.expected): + +```text + 2659 RMC: fix + 180 RMC: no fix, NoPosition + 3026 GGA: position + 804 AIS: position report, type 1 + 388 refused: BadChecksum + 483 refused: NoChecksum +``` + +To run it locally, with gpsd's `test/daemon` checked out: + +```sh +GPSD_LOGS=path/to/gpsd/test/daemon cargo test -p kinavis-examples --test gpsd_corpus -- --ignored +``` + ## Data | File | Source | Licence | diff --git a/examples/lib.rs b/examples/lib.rs index a80cadd..0742c3e 100644 --- a/examples/lib.rs +++ b/examples/lib.rs @@ -1,6 +1,7 @@ //! Runnable demonstrations of the KINAVIS crates, on recorded data. //! -//! The library is empty; the demonstrations are the examples of this package. -//! See `README.md` beside this file. +//! The library is empty; the demonstrations are the examples of this package, +//! and `tests/gpsd_corpus.rs` reads the gpsd receiver logs against a recorded +//! tally. See `README.md` beside this file. #![no_std] diff --git a/examples/tests/gpsd_corpus.expected b/examples/tests/gpsd_corpus.expected new file mode 100644 index 0000000..89be3c8 --- /dev/null +++ b/examples/tests/gpsd_corpus.expected @@ -0,0 +1,23 @@ +# Tally of the gpsd regression logs (199 logs). + 142 AIS: AidToNavigation + 154 AIS: StaticAndVoyageData + 147 AIS: StaticDataReport + 346 AIS: Unsupported + 179 AIS: fragment held + 804 AIS: position report, type 1 + 228 AIS: position report, type 18 + 3 AIS: position report, type 19 + 12 AIS: position report, type 2 + 147 AIS: position report, type 3 + 513 GGA: no fix + 3026 GGA: position + 2375 GLL + 2659 RMC: fix + 180 RMC: no fix, NoPosition + 2714 VTG + 388 refused: BadChecksum + 79 refused: BadField + 8 refused: BadHeader + 483 refused: NoChecksum + 413 refused: TooLong + 24088 unsupported sentence diff --git a/examples/tests/gpsd_corpus.rs b/examples/tests/gpsd_corpus.rs new file mode 100644 index 0000000..f42778e --- /dev/null +++ b/examples/tests/gpsd_corpus.rs @@ -0,0 +1,140 @@ +//! Every sentence of the gpsd regression logs, read as the navigation system +//! would read it, against a recorded tally. +//! +//! The logs, about two hundred, are recorded from real receivers and are not +//! in this repository: CI fetches `test/daemon/` of gpsd at a pinned commit +//! and names the directory in `GPSD_LOGS`. Ignored by default; run with +//! +//! ```text +//! GPSD_LOGS=path/to/gpsd/test/daemon cargo test -p kinavis-examples --test gpsd_corpus -- --ignored +//! ``` +//! +//! The tally counts sentences by what became of them: read as which sentence, +//! refused for which error, translated into a fix or not, decoded as which AIS +//! message. A changed count fails the test and prints the new tally; with +//! `KINAVIS_BLESS=1` the new tally is written as the expected one, to be +//! reviewed in the diff. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::BTreeMap; +use std::fmt::{Debug, Write as _}; +use std::fs; +use std::path::{Path, PathBuf}; + +use kinavis::{GnssFix, Instant, Utc}; +use kinavis_ais::{Assembler, Message}; +use kinavis_nmea0183::{parse, Sentence}; + +const EXPECTED: &str = "tests/gpsd_corpus.expected"; + +#[test] +#[ignore = "needs the gpsd logs; see the module documentation"] +fn the_gpsd_logs_read_as_recorded() { + let dir = std::env::var_os("GPSD_LOGS").expect("GPSD_LOGS names gpsd's test/daemon directory"); + let logs = logs(Path::new(&dir)); + assert!(logs.len() > 100, "{} logs in {dir:?}", logs.len()); + + let mut tally = Tally::default(); + for log in &logs { + read_log(&fs::read(log).unwrap(), &mut tally); + } + let actual = tally.to_string(logs.len()); + + let expected_path = Path::new(env!("CARGO_MANIFEST_DIR")).join(EXPECTED); + if std::env::var_os("KINAVIS_BLESS").is_some() { + fs::write(&expected_path, &actual).unwrap(); + return; + } + let expected = fs::read_to_string(&expected_path).unwrap_or_default(); + assert!( + actual == expected, + "the tally changed; review it and rerun with KINAVIS_BLESS=1 to record it\n\n{actual}" + ); +} + +/// `*.log` files of a directory, in name order. +fn logs(dir: &Path) -> Vec { + let mut logs: Vec = fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "log")) + .collect(); + logs.sort(); + logs +} + +/// Counts by outcome. +#[derive(Default)] +struct Tally(BTreeMap); + +impl Tally { + fn count(&mut self, outcome: String) { + *self.0.entry(outcome).or_default() += 1; + } + + fn to_string(&self, logs: usize) -> String { + let mut out = format!("# Tally of the gpsd regression logs ({logs} logs).\n"); + for (outcome, count) in &self.0 { + writeln!(out, "{count:>7} {outcome}").unwrap(); + } + out + } +} + +/// One log, line by line; only lines that start a sentence count. Binary logs +/// and comments are skipped by that rule. +fn read_log(bytes: &[u8], tally: &mut Tally) { + let mut assembler = Assembler::new(); + let now = Instant::::from_unix_seconds(0); + for line in bytes.split(|&byte| byte == b'\n') { + let line = line.strip_suffix(b"\r").unwrap_or(line); + if !matches!(line.first(), Some(b'$' | b'!')) { + continue; + } + match parse(line) { + Err(error) => tally.count(format!("refused: {}", kind(&error))), + Ok(Sentence::Rmc(rmc)) => match GnssFix::try_from(rmc) { + Ok(_) => tally.count("RMC: fix".to_owned()), + Err(error) => tally.count(format!("RMC: no fix, {}", kind(&error))), + }, + Ok(Sentence::Gga(gga)) => { + let fix = if gga.fix_type.is_position_fix() { + "position" + } else { + "no fix" + }; + tally.count(format!("GGA: {fix}")); + } + Ok(Sentence::Gll(_)) => tally.count("GLL".to_owned()), + Ok(Sentence::Vtg(_)) => tally.count("VTG".to_owned()), + Ok(Sentence::Vdm(vdm)) => match assembler.push(&vdm, now) { + Ok(None) => tally.count("AIS: fragment held".to_owned()), + Ok(Some(bits)) => match Message::decode(&bits) { + Ok(message) => tally.count(format!("AIS: {}", message_kind(&message))), + Err(error) => tally.count(format!("AIS: refused, {}", kind(&error))), + }, + Err(error) => tally.count(format!("AIS: refused, {}", kind(&error))), + }, + Ok(Sentence::Unsupported { .. }) => tally.count("unsupported sentence".to_owned()), + Ok(other) => tally.count(format!("other: {}", kind(&other))), + } + } +} + +/// Variant name of an enum value: its `Debug` form up to the first field. +fn kind(value: &impl Debug) -> String { + let debug = format!("{value:?}"); + debug + .split([' ', '(', '{']) + .next() + .unwrap_or_default() + .to_owned() +} + +fn message_kind(message: &Message) -> String { + match message { + Message::PositionReport(report) => format!("position report, type {}", report.kind), + other => kind(other), + } +}