Part I — Quarterly reruns: history on the page, PR instead of push, staleness report

.github/workflows/matrix.yml

@@ -1,10 +1,12 @@
 name: matrix
 
-# The full suites against real containers. Slow and Docker-bound, so this runs
-# on a schedule and on demand rather than on every pull request.
-#
-# A matrix without a date is a claim about the past that reads as a claim about
-# the present. The quarterly run is what keeps a published page honest.
+# The full suites against real containers, on a schedule. A matrix without a
+# date is a claim about the past that reads as a claim about the present, so
+# this is what keeps a published page honest — and it opens a pull request
+# rather than pushing straight to main, because "a maintainer bumps every
+# image tag to the current release and re-confirms each adapter by hand" is a
+# human step this workflow cannot do for them; it produces the page and lets
+# a maintainer read it before it publishes.
 on:
   workflow_dispatch:
   schedule:
@@ -21,16 +23,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [otlp-logs, es-bulk]
+        suite: [otlp-logs, es-bulk, remote-write, otlp-metrics, loki-push]
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       - name: run the suite against every column
         run: |
           case "${{ matrix.suite }}" in
-            otlp-logs) BACKENDS=parseable,openobserve,quickwit,victorialogs,loki ;;
-            es-bulk)   BACKENDS=elasticsearch,quickwit,openobserve ;;
+            otlp-logs)     BACKENDS=parseable,openobserve,quickwit,victorialogs,loki ;;
+            es-bulk)       BACKENDS=elasticsearch,quickwit,openobserve ;;
+            remote-write)  BACKENDS=prometheus,mimir,greptimedb,victoriametrics ;;
+            otlp-metrics)  BACKENDS=prometheus,mimir,greptimedb,victoriametrics,parseable ;;
+            loki-push)     BACKENDS=loki,victorialogs,openobserve ;;
             *) echo "unknown suite"; exit 1 ;;
           esac
           cargo run --release -- matrix --suite "${{ matrix.suite }}" --manage --backends "$BACKENDS"
@@ -39,3 +44,67 @@ jobs:
           name: matrix-${{ matrix.suite }}
           path: results/
           if-no-files-found: error
+
+  # A backend whose adapter cannot be re-confirmed in a quarter is shown with
+  # its last date and a note, never silently carried forward — this is the
+  # note. It cannot know a maintainer skipped a re-confirmation this quarter
+  # on purpose, only that the comment's date is old; the pull request body is
+  # where a human reads that and decides whether it still matters.
+  confirmation-age:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - run: python3 tools/check_confirmation_age.py | tee confirmation-age.txt
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: confirmation-age
+          path: confirmation-age.txt
+
+  publish:
+    needs: [run, confirmation-age]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: matrix-*
+          path: results
+          merge-multiple: true
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: confirmation-age
+      - id: date
+        run: echo "date=$(date -u +%F)" >> "$GITHUB_OUTPUT"
+      - id: report
+        run: |
+          {
+            echo 'text<<SPECMATRIX_EOF'
+            cat confirmation-age.txt
+            echo 'SPECMATRIX_EOF'
+          } >> "$GITHUB_OUTPUT"
+      - uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          add-paths: results/
+          commit-message: "results: quarterly matrix, ${{ steps.date.outputs.date }}"
+          branch: quarterly-matrix-${{ steps.date.outputs.date }}
+          title: "Quarterly matrix — ${{ steps.date.outputs.date }}"
+          body: |
+            The scheduled run of every suite this repository publishes, for review
+            before it goes out under the project's name.
+
+            Before merging: bump every image tag to the current release and
+            re-confirm each adapter by hand against the checklist in
+            `docs/BACKENDS.md`, per `AGENTS.md`. That is the cost of a column, and
+            the reason the roster is capped — a matrix a maintainer did not
+            personally re-check this quarter is a claim about the past printed as
+            a claim about the present.
+
+            `tools/check_confirmation_age.py`'s report for this run:
+
+            ```
+            ${{ steps.report.outputs.text }}
+            ```
+          labels: quarterly-matrix

AGENTS.md

@@ -247,6 +247,30 @@ Two `container:` fields exist for settings a CLI flag cannot reach:
   in the API says so — only `docker logs` names the bound address, and every
   ingest from outside answers a bare connection reset with no HTTP status.
 
+## Quarterly reruns
+
+A matrix without a date is a claim about the past that reads as a claim about
+the present. `.github/workflows/matrix.yml` runs every suite on the first day
+of each quarter (or on demand) and opens a pull request adding
+`results/<date>/<suite>/matrix.{json,md,html}` — never pushed straight to
+main, because bumping every image tag and re-confirming each adapter by hand
+against `docs/BACKENDS.md` is a maintainer's own step, not something the
+workflow can do for them. That is the cost of a column, and the reason the
+roster is capped.
+
+`tools/check_confirmation_age.py` reads the "Confirmed by hand against
+`<image>` on `<date>`" comment every adapter opens with, rather than adding a
+second machine-only field that could drift from it, and reports how old each
+one is. It never fails the job — a stale adapter is not a build failure, it
+is something a maintainer needs to see and either re-confirm or explain — and
+its report goes into the quarterly pull request body so a stale column is
+never silently carried forward as current.
+
+`Matrix::to_html` compares each cell against the most recent prior run of the
+same suite under `results/` and, where a verdict changed, links back to it —
+the strongest evidence this project can offer that a finding is acted on is a
+cell moving from `ALTER` to `PASS` with the version it changed at.
+
 ## Reporting a divergence
 
 Findings are worth more filed than tabulated.

src/main.rs

@@ -117,6 +117,40 @@ enum Command {
     },
 }
 
+/// The most recent prior run of `suite` under `results_dir`, strictly before
+/// `current_generated`'s own date — so a rerun on the same day as a previous
+/// one, or a re-render of an existing matrix.json, never compares against
+/// itself. `results/<date>/<suite>/` is the layout every `matrix` and
+/// `render` invocation writes, so this reads back exactly what was written.
+fn find_previous_run(
+    results_dir: &std::path::Path,
+    suite: &str,
+    current_generated: &str,
+) -> Option<(String, matrix::Matrix)> {
+    let current_date = current_generated.get(..10)?;
+    let mut candidates: Vec<(String, PathBuf)> = std::fs::read_dir(results_dir)
+        .ok()?
+        .filter_map(|e| e.ok())
+        .filter(|e| e.path().is_dir())
+        .filter_map(|e| {
+            let date = e.file_name().to_string_lossy().into_owned();
+            // Only a directory actually named as a date sorts and compares
+            // meaningfully against `current_date`.
+            if date.len() == 10 && date.as_str() < current_date {
+                let json = e.path().join(suite).join("matrix.json");
+                json.exists().then_some((date, json))
+            } else {
+                None
+            }
+        })
+        .collect();
+    candidates.sort_by(|a, b| a.0.cmp(&b.0));
+    let (date, json_path) = candidates.pop()?;
+    let text = std::fs::read_to_string(&json_path).ok()?;
+    let previous: matrix::Matrix = serde_json::from_str(&text).ok()?;
+    Some((format!("../../{date}/{suite}/matrix.html"), previous))
+}
+
 fn main() -> Result<()> {
     let cli = Cli::parse();
     match cli.command {
@@ -199,7 +233,12 @@ fn main() -> Result<()> {
                 .ok()
                 .filter(|o| o.status.success())
                 .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string());
-            let page = matrix.to_html(commit.as_deref());
+            let previous =
+                find_previous_run(&PathBuf::from("results"), &matrix.suite, &matrix.generated);
+            let history = previous
+                .as_ref()
+                .map(|(link, m)| matrix::History { link: link.clone(), matrix: m });
+            let page = matrix.to_html(commit.as_deref(), history.as_ref());
             let out = out.unwrap_or_else(|| path.with_file_name("matrix.html"));
             std::fs::write(&out, page).with_context(|| format!("writing {}", out.display()))?;
             println!("wrote {}", out.display());
@@ -257,9 +296,84 @@ fn main() -> Result<()> {
                 .ok()
                 .filter(|o| o.status.success())
                 .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string());
-            std::fs::write(dir.join("matrix.html"), matrix.to_html(commit.as_deref()))?;
+            let previous =
+                find_previous_run(&PathBuf::from("results"), &matrix.suite, &matrix.generated);
+            let history = previous
+                .as_ref()
+                .map(|(link, m)| matrix::History { link: link.clone(), matrix: m });
+            std::fs::write(
+                dir.join("matrix.html"),
+                matrix.to_html(commit.as_deref(), history.as_ref()),
+            )?;
             println!("wrote {}/matrix.{{json,md,html}}", dir.display());
             Ok(())
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn scratch_dir(name: &str) -> PathBuf {
+        let dir = std::env::temp_dir()
+            .join(format!("specmatrix-find-previous-{name}-{:x}", rand::random::<u64>()));
+        std::fs::create_dir_all(&dir).unwrap();
+        dir
+    }
+
+    fn write_matrix(results_dir: &std::path::Path, date: &str, suite: &str) {
+        let dir = results_dir.join(date).join(suite);
+        std::fs::create_dir_all(&dir).unwrap();
+        let m = matrix::Matrix::new(
+            suite,
+            vec![],
+            chrono::DateTime::parse_from_rfc3339(&format!("{date}T00:00:00Z"))
+                .unwrap()
+                .with_timezone(&chrono::Utc),
+        );
+        std::fs::write(dir.join("matrix.json"), serde_json::to_string(&m).unwrap()).unwrap();
+    }
+
+    /// Among several prior runs, the most recent one strictly before the
+    /// current date is chosen — not the first found, not the oldest.
+    #[test]
+    fn the_most_recent_prior_run_before_today_is_chosen() {
+        let dir = scratch_dir("several");
+        write_matrix(&dir, "2026-01-01", "otlp-logs");
+        write_matrix(&dir, "2026-04-01", "otlp-logs");
+        write_matrix(&dir, "2026-07-01", "otlp-logs");
+        let (link, previous) = find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00")
+            .expect("a previous run exists");
+        assert_eq!(&previous.generated[..10], "2026-07-01");
+        assert!(link.contains("2026-07-01"), "{link}");
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    /// A run on or after the current date is never picked as "previous" —
+    /// including a rerun on the same day, which must not compare against
+    /// itself.
+    #[test]
+    fn a_run_on_or_after_the_current_date_is_never_previous() {
+        let dir = scratch_dir("same-day");
+        write_matrix(&dir, "2026-09-08", "otlp-logs");
+        assert!(find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00").is_none());
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    /// A previous run of a different suite is never mistaken for this one's
+    /// history.
+    #[test]
+    fn a_different_suites_run_is_not_history() {
+        let dir = scratch_dir("other-suite");
+        write_matrix(&dir, "2026-07-01", "es-bulk");
+        assert!(find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00").is_none());
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn no_results_directory_at_all_is_simply_no_history() {
+        let dir = std::env::temp_dir().join("specmatrix-find-previous-does-not-exist");
+        assert!(find_previous_run(&dir, "otlp-logs", "2026-09-08T00:00:00+00:00").is_none());
+    }
+}