diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 639cd04..3c7f026 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -1,10 +1,12 @@ name: matrix -# The full suites against real containers. Slow and Docker-bound, so this runs -# on a schedule and on demand rather than on every pull request. -# -# A matrix without a date is a claim about the past that reads as a claim about -# the present. The quarterly run is what keeps a published page honest. +# The full suites against real containers, on a schedule. A matrix without a +# date is a claim about the past that reads as a claim about the present, so +# this is what keeps a published page honest — and it opens a pull request +# rather than pushing straight to main, because "a maintainer bumps every +# image tag to the current release and re-confirms each adapter by hand" is a +# human step this workflow cannot do for them; it produces the page and lets +# a maintainer read it before it publishes. on: workflow_dispatch: schedule: @@ -21,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [otlp-logs, es-bulk] + suite: [otlp-logs, es-bulk, remote-write, otlp-metrics, loki-push] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable @@ -29,8 +31,11 @@ jobs: - name: run the suite against every column run: | case "${{ matrix.suite }}" in - otlp-logs) BACKENDS=parseable,openobserve,quickwit,victorialogs,loki ;; - es-bulk) BACKENDS=elasticsearch,quickwit,openobserve ;; + otlp-logs) BACKENDS=parseable,openobserve,quickwit,victorialogs,loki ;; + es-bulk) BACKENDS=elasticsearch,quickwit,openobserve ;; + remote-write) BACKENDS=prometheus,mimir,greptimedb,victoriametrics ;; + otlp-metrics) BACKENDS=prometheus,mimir,greptimedb,victoriametrics,parseable ;; + loki-push) BACKENDS=loki,victorialogs,openobserve ;; *) echo "unknown suite"; exit 1 ;; esac cargo run --release -- matrix --suite "${{ matrix.suite }}" --manage --backends "$BACKENDS" @@ -39,3 +44,67 @@ jobs: name: matrix-${{ matrix.suite }} path: results/ if-no-files-found: error + + # A backend whose adapter cannot be re-confirmed in a quarter is shown with + # its last date and a note, never silently carried forward — this is the + # note. It cannot know a maintainer skipped a re-confirmation this quarter + # on purpose, only that the comment's date is old; the pull request body is + # where a human reads that and decides whether it still matters. + confirmation-age: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - run: python3 tools/check_confirmation_age.py | tee confirmation-age.txt + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: confirmation-age + path: confirmation-age.txt + + publish: + needs: [run, confirmation-age] + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: matrix-* + path: results + merge-multiple: true + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: confirmation-age + - id: date + run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT" + - id: report + run: | + { + echo 'text<> "$GITHUB_OUTPUT" + - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + add-paths: results/ + commit-message: "results: quarterly matrix, ${{ steps.date.outputs.date }}" + branch: quarterly-matrix-${{ steps.date.outputs.date }} + title: "Quarterly matrix — ${{ steps.date.outputs.date }}" + body: | + The scheduled run of every suite this repository publishes, for review + before it goes out under the project's name. + + Before merging: bump every image tag to the current release and + re-confirm each adapter by hand against the checklist in + `docs/BACKENDS.md`, per `AGENTS.md`. That is the cost of a column, and + the reason the roster is capped — a matrix a maintainer did not + personally re-check this quarter is a claim about the past printed as + a claim about the present. + + `tools/check_confirmation_age.py`'s report for this run: + + ``` + ${{ steps.report.outputs.text }} + ``` + labels: quarterly-matrix diff --git a/AGENTS.md b/AGENTS.md index 9091798..1a1e1d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -247,6 +247,30 @@ Two `container:` fields exist for settings a CLI flag cannot reach: in the API says so — only `docker logs` names the bound address, and every ingest from outside answers a bare connection reset with no HTTP status. +## Quarterly reruns + +A matrix without a date is a claim about the past that reads as a claim about +the present. `.github/workflows/matrix.yml` runs every suite on the first day +of each quarter (or on demand) and opens a pull request adding +`results///matrix.{json,md,html}` — never pushed straight to +main, because bumping every image tag and re-confirming each adapter by hand +against `docs/BACKENDS.md` is a maintainer's own step, not something the +workflow can do for them. That is the cost of a column, and the reason the +roster is capped. + +`tools/check_confirmation_age.py` reads the "Confirmed by hand against +`` on ``" comment every adapter opens with, rather than adding a +second machine-only field that could drift from it, and reports how old each +one is. It never fails the job — a stale adapter is not a build failure, it +is something a maintainer needs to see and either re-confirm or explain — and +its report goes into the quarterly pull request body so a stale column is +never silently carried forward as current. + +`Matrix::to_html` compares each cell against the most recent prior run of the +same suite under `results/` and, where a verdict changed, links back to it — +the strongest evidence this project can offer that a finding is acted on is a +cell moving from `ALTER` to `PASS` with the version it changed at. + ## Reporting a divergence Findings are worth more filed than tabulated. diff --git a/src/main.rs b/src/main.rs index 9525469..40e2692 100644 --- a/src/main.rs +++ b/src/main.rs @@ -117,6 +117,40 @@ enum Command { }, } +/// The most recent prior run of `suite` under `results_dir`, strictly before +/// `current_generated`'s own date — so a rerun on the same day as a previous +/// one, or a re-render of an existing matrix.json, never compares against +/// itself. `results///` is the layout every `matrix` and +/// `render` invocation writes, so this reads back exactly what was written. +fn find_previous_run( + results_dir: &std::path::Path, + suite: &str, + current_generated: &str, +) -> Option<(String, matrix::Matrix)> { + let current_date = current_generated.get(..10)?; + let mut candidates: Vec<(String, PathBuf)> = std::fs::read_dir(results_dir) + .ok()? + .filter_map(|e| e.ok()) + .filter(|e| e.path().is_dir()) + .filter_map(|e| { + let date = e.file_name().to_string_lossy().into_owned(); + // Only a directory actually named as a date sorts and compares + // meaningfully against `current_date`. + if date.len() == 10 && date.as_str() < current_date { + let json = e.path().join(suite).join("matrix.json"); + json.exists().then_some((date, json)) + } else { + None + } + }) + .collect(); + candidates.sort_by(|a, b| a.0.cmp(&b.0)); + let (date, json_path) = candidates.pop()?; + let text = std::fs::read_to_string(&json_path).ok()?; + let previous: matrix::Matrix = serde_json::from_str(&text).ok()?; + Some((format!("../../{date}/{suite}/matrix.html"), previous)) +} + fn main() -> Result<()> { let cli = Cli::parse(); match cli.command { @@ -199,7 +233,12 @@ fn main() -> Result<()> { .ok() .filter(|o| o.status.success()) .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()); - let page = matrix.to_html(commit.as_deref()); + let previous = + find_previous_run(&PathBuf::from("results"), &matrix.suite, &matrix.generated); + let history = previous + .as_ref() + .map(|(link, m)| matrix::History { link: link.clone(), matrix: m }); + let page = matrix.to_html(commit.as_deref(), history.as_ref()); let out = out.unwrap_or_else(|| path.with_file_name("matrix.html")); std::fs::write(&out, page).with_context(|| format!("writing {}", out.display()))?; println!("wrote {}", out.display()); @@ -257,9 +296,84 @@ fn main() -> Result<()> { .ok() .filter(|o| o.status.success()) .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()); - std::fs::write(dir.join("matrix.html"), matrix.to_html(commit.as_deref()))?; + let previous = + find_previous_run(&PathBuf::from("results"), &matrix.suite, &matrix.generated); + let history = previous + .as_ref() + .map(|(link, m)| matrix::History { link: link.clone(), matrix: m }); + std::fs::write( + dir.join("matrix.html"), + matrix.to_html(commit.as_deref(), history.as_ref()), + )?; println!("wrote {}/matrix.{{json,md,html}}", dir.display()); Ok(()) } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir() + .join(format!("specmatrix-find-previous-{name}-{:x}", rand::random::())); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn write_matrix(results_dir: &std::path::Path, date: &str, suite: &str) { + let dir = results_dir.join(date).join(suite); + std::fs::create_dir_all(&dir).unwrap(); + let m = matrix::Matrix::new( + suite, + vec![], + chrono::DateTime::parse_from_rfc3339(&format!("{date}T00:00:00Z")) + .unwrap() + .with_timezone(&chrono::Utc), + ); + std::fs::write(dir.join("matrix.json"), serde_json::to_string(&m).unwrap()).unwrap(); + } + + /// Among several prior runs, the most recent one strictly before the + /// current date is chosen — not the first found, not the oldest. + #[test] + fn the_most_recent_prior_run_before_today_is_chosen() { + let dir = scratch_dir("several"); + write_matrix(&dir, "2026-01-01", "otlp-logs"); + write_matrix(&dir, "2026-04-01", "otlp-logs"); + write_matrix(&dir, "2026-07-01", "otlp-logs"); + let (link, previous) = find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00") + .expect("a previous run exists"); + assert_eq!(&previous.generated[..10], "2026-07-01"); + assert!(link.contains("2026-07-01"), "{link}"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A run on or after the current date is never picked as "previous" — + /// including a rerun on the same day, which must not compare against + /// itself. + #[test] + fn a_run_on_or_after_the_current_date_is_never_previous() { + let dir = scratch_dir("same-day"); + write_matrix(&dir, "2026-09-08", "otlp-logs"); + assert!(find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00").is_none()); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A previous run of a different suite is never mistaken for this one's + /// history. + #[test] + fn a_different_suites_run_is_not_history() { + let dir = scratch_dir("other-suite"); + write_matrix(&dir, "2026-07-01", "es-bulk"); + assert!(find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00").is_none()); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn no_results_directory_at_all_is_simply_no_history() { + let dir = std::env::temp_dir().join("specmatrix-find-previous-does-not-exist"); + assert!(find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00").is_none()); + } +} diff --git a/src/matrix.rs b/src/matrix.rs index 36b4df1..f701f1a 100644 --- a/src/matrix.rs +++ b/src/matrix.rs @@ -75,12 +75,26 @@ the backend and is never counted with the others.\n", } } +/// A previous run of the same suite, for the page to compare against. +/// +/// The strongest evidence this project can offer that a finding is acted on +/// is a cell changing from `ALTER` to `PASS` with the version it changed at +/// — which needs the previous verdict beside the new one, not just linked. +pub struct History<'a> { + /// Where the previous run's own page lives, relative to the page being + /// rendered now. + pub link: String, + pub matrix: &'a Matrix, +} + impl Matrix { /// A single static page, rendered from the JSON beside it. /// /// `corpus_commit` identifies the corpus the run used, so a reader can - /// check out exactly the checks that produced these cells. - pub fn to_html(&self, corpus_commit: Option<&str>) -> String { + /// check out exactly the checks that produced these cells. `history`, when + /// given, is the most recent prior run of this same suite: a cell whose + /// verdict changed says so and links back to it. + pub fn to_html(&self, corpus_commit: Option<&str>, history: Option<&History>) -> String { let mut out = String::new(); out.push_str(&format!( "\n\n\n{} — SpecMatrix\n\n\n", @@ -98,6 +112,13 @@ impl Matrix { escape_html(commit) )); } + if let Some(h) = history { + out.push_str(&format!( + "

Compared against the run of {}; a cell that changed says so.

\n", + escape_html(&h.link), + escape_html(&h.matrix.generated[..10]) + )); + } out.push_str("\n"); for outcome in &self.outcomes { @@ -124,10 +145,25 @@ impl Matrix { Verdict::Alter => "alter", Verdict::NotApplicable => "na", }; + let hist = history.and_then(|h| { + let prev_outcome = + h.matrix.outcomes.iter().find(|o| o.backend == outcome.backend)?; + let prev = prev_outcome.results.iter().find(|r| r.id == id)?; + if prev.verdict == result.verdict { + return None; + } + Some(format!( + "changed from {} on {}", + escape_html(&h.link), + label(prev.verdict), + escape_html(&h.matrix.generated[..10]) + )) + }); out.push_str(&format!( - "", + "", label(result.verdict), - escape_html(&result.detail) + escape_html(&result.detail), + hist.unwrap_or_default() )); } None => { @@ -165,6 +201,7 @@ td{max-width:22rem}\ .alter{background:#fdf0f0}.alter .v{color:#a01b1b}\ .na{background:#f4f4f7}.na .v{color:#5a5a70}\ .notrun{background:#fff}.notrun .v{color:#999}\ +.hist{display:block;font-size:10px;color:#555;margin-top:.2rem;text-decoration:underline}\ "; fn version_line(outcome: &Outcome) -> String { @@ -356,7 +393,7 @@ mod tests { /// The page must never carry a score, a percentage or an ordering. #[test] fn the_html_page_carries_no_score_or_ranking() { - let page = two().to_html(None).to_lowercase(); + let page = two().to_html(None, None).to_lowercase(); for banned in ["score", "rank", "%", " out of ", "best", "worst", "winner"] { assert!(!page.contains(banned), "page must not contain {banned:?}"); } @@ -364,7 +401,7 @@ mod tests { #[test] fn the_html_page_is_dated_and_versioned() { - let page = two().to_html(Some("abc1234")); + let page = two().to_html(Some("abc1234"), None); assert!(page.contains("2026-09-08"), "{page}"); assert!(page.contains("2.9.4"), "{page}"); assert!(page.contains("abc1234"), "{page}"); @@ -374,7 +411,7 @@ mod tests { /// it is not a verdict about the backend. #[test] fn each_verdict_gets_its_own_class() { - let page = two().to_html(None); + let page = two().to_html(None, None); for class in ["class=\"pass\"", "class=\"reject\"", "class=\"alter\""] { assert!(page.contains(class), "missing {class}"); } @@ -396,17 +433,74 @@ mod tests { )], when, ); - let page = matrix.to_html(None); + let page = matrix.to_html(None, None); assert!(!page.contains("
Check{}{}{}{}{}