Part H — CI action: run the corpus against one backend without cloning

.github/workflows/dogfood.yml

@@ -0,0 +1,77 @@
+name: dogfood
+
+# Runs this repository's own `action.yml` against every declared (backend,
+# suite) pair, using the pull request's own binary and corpus rather than a
+# published release — so a case or adapter that is wrong goes red before it
+# reaches a tag, on the columns already committed here.
+#
+# Only for pull requests on this repository, not forks: the corpus-fetch step
+# inside the action asks for the pull request's SHA from this repository, and
+# a fork's commits do not exist here until the branch is pushed to it. A fork
+# PR still gets the fast static checks in ci.yml on every commit; this workflow
+# adds the containers.
+on:
+  pull_request:
+    paths:
+      - "cases/**"
+      - "backends/**"
+      - "action.yml"
+      - "tools/summarize.py"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: dogfood-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  matrix:
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    outputs:
+      pairs: ${{ steps.pairs.outputs.pairs }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - id: pairs
+        run: echo "pairs=$(python3 tools/list_backend_protocols.py)" >> "$GITHUB_OUTPUT"
+
+  run:
+    needs: matrix
+    if: needs.matrix.outputs.pairs != '[]'
+    runs-on: ubuntu-latest
+    # Twenty (backend, suite) pairs run as one matrix, all restoring and
+    # saving the same Rust build cache at once — measured contention pushed
+    # one leg past 20 minutes with nothing wrong, just queued behind others.
+    timeout-minutes: 35
+    strategy:
+      fail-fast: false
+      matrix:
+        pair: ${{ fromJson(needs.matrix.outputs.pairs) }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+      - run: cargo build --release
+
+      - name: start ${{ matrix.pair.backend }}
+        id: up
+        run: |
+          url="$(cargo run --release -q -- up --backend ${{ matrix.pair.backend }} \
+            | sed -n 's/^.* ready at //p')"
+          echo "url=$url" >> "$GITHUB_OUTPUT"
+
+      # The action itself, exercised the same way an external caller would
+      # use it — `ref`/`binary-path` are the two escape hatches that exist
+      # only for this job, so this run reads the pull request's own corpus
+      # and binary instead of the last published release.
+      - uses: ./
+        with:
+          backend: ${{ matrix.pair.backend }}
+          suite: ${{ matrix.pair.suite }}
+          url: ${{ steps.up.outputs.url }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          binary-path: target/release/specmatrix
+
+      - if: always()
+        run: cargo run --release -q -- down --backend ${{ matrix.pair.backend }}

.github/workflows/release.yml

@@ -0,0 +1,40 @@
+name: release
+
+# A maintainer running the CI action, or the corpus by hand, needs a binary
+# that is not "clone this repository and build it". This builds one for each
+# architecture the action supports and attaches it to the release the tag
+# names.
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu]
+    runs-on: ubuntu-latest
+    steps:
+      # Actions are pinned to a commit, not a tag, the same as every other
+      # workflow in this repository.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable
+        with:
+          targets: ${{ matrix.target }}
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+        with:
+          key: ${{ matrix.target }}
+      # Builds (cross-compiling for aarch64 via `cross`, which this action
+      # drives itself), archives, and uploads to the release the pushed tag
+      # names, creating it if this is the first artefact for it.
+      - uses: taiki-e/upload-rust-binary-action@f0d45ae91ee7b8ee928de7a9d04d893a08bcbec6 # v1.30.2
+        with:
+          bin: specmatrix
+          target: ${{ matrix.target }}
+          archive: specmatrix-$tag-$target
+          checksum: sha256
+          token: ${{ secrets.GITHUB_TOKEN }}

AGENTS.md

@@ -247,6 +247,43 @@ Two `container:` fields exist for settings a CLI flag cannot reach:
   in the API says so — only `docker logs` names the bound address, and every
   ingest from outside answers a bare connection reset with no HTTP status.
 
+`allow:` names case ids a maintainer has read and accepted, each with a
+one-line reason — for a backend maintainer running `action.yml` in their own
+CI, not for this repository's own adapters. It never changes a verdict: the
+runner decides `PASS`/`REJECT`/`ALTER` from the wire alone, exactly as it
+would without the entry. What it changes is only how the action's job summary
+presents the row, moving it to its own section instead of reading as an
+unreviewed failure — because the job already never fails on a verdict, only
+on a harness error. Silence is not acceptance; a case id absent from `allow:`
+is printed with the others.
+
+```yaml
+allow:
+  otlp-logs/body-invalid-utf8: "documented, tracked at our-org/our-store#123"
+```
+
+## Running the suite without cloning this repository
+
+`action.yml` at the repository root is a composite GitHub Action a backend
+maintainer adds to their own CI, so a commit that breaks conformance is
+visible without anyone cloning this repository by hand. It takes `backend`
+(a name from this repository's own `backends/`, or a `path/to/adapter.yaml`
+already checked out in the caller's own repository, for a backend not carried
+here), `suite`, `url` of the already-running backend, and `version` (a
+release tag of this project, default `latest`) — pinning `version` pins both
+the runner binary and the corpus it is paired with, so a result names an
+exact version of both rather than a mix. It fails the job only on a harness
+error; verdicts never fail it, which is what makes `allow:` above meaningful
+rather than a way to silence CI.
+
+`.github/workflows/release.yml` builds and attaches the binaries `action.yml`
+downloads, for `x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu`, on
+every `v*` tag. `.github/workflows/dogfood.yml` runs the action against every
+adapter this repository carries, on a pull request that touches `cases/` or
+`backends/` — using the pull request's own binary and corpus (`action.yml`'s
+`ref`/`binary-path` inputs, which exist only for that workflow) rather than
+the last published release, so a wrong case goes red before it ships.
+
 ## Quarterly reruns
 
 A matrix without a date is a claim about the past that reads as a claim about
@@ -296,6 +333,9 @@ than assert.
 - Every case cites a rule with a basis (`tools/check_corpus.py`)
 - Every adapter pins its image and can be started
 - `cargo audit`
+- A pull request touching `cases/` or `backends/` runs `action.yml` against
+  every adapter, using that pull request's own binary and corpus
+  (`.github/workflows/dogfood.yml`)
 
 ## Never
 

README.md

@@ -186,6 +186,21 @@ and deletes the stream or index each case uses before running it. Point it at a
 store you are willing to have written to, and read
 [`SECURITY.md`](SECURITY.md) first.
 
+## Running it in CI, without cloning this repository
+
+```yaml
+- uses: DeviousCardi/specmatrix@v1.0.0
+  with:
+    backend: loki      # or path/to/your-adapter.yaml for one not carried here
+    suite: otlp-logs
+    url: http://localhost:3100
+```
+
+Fails the job only on a harness error — a verdict never does, so a maintainer
+who has read and accepted a divergence names it in their adapter's `allow:`
+list rather than the job going red on a result they already know about. See
+[`AGENTS.md`](AGENTS.md#running-the-suite-without-cloning-this-repository).
+
 ## Backends
 
 | Backend | OTLP logs | Elasticsearch `_bulk` |

action.yml

@@ -0,0 +1,159 @@
+name: specmatrix
+description: >
+  Run a SpecMatrix conformance suite against one running backend, without
+  cloning this repository.
+author: DeviousCardi
+
+inputs:
+  backend:
+    description: >
+      Which adapter to run. Either a bare name resolved against this
+      repository's own `backends/<name>.yaml` at `version`, or a path (must
+      contain a `/`) to an adapter file already checked out in the caller's
+      own repository — for a backend not carried here.
+    required: true
+  suite:
+    description: Protocol suite, resolved to this repository's own cases/<suite>/*.yaml at `version`.
+    required: true
+  url:
+    description: Base URL of the already-running backend to test.
+    required: true
+  version:
+    description: >
+      Release tag of this project to run — pins both the runner binary and
+      the corpus (cases and built-in adapters) it is paired with, so a result
+      names an exact version of both. `latest` resolves to the newest
+      published release.
+    required: false
+    default: latest
+  ref:
+    description: >
+      Internal: overrides which commit the corpus (cases/, backends/) is
+      fetched at, independent of `version`. Used only by this repository's
+      own dogfood workflow, to run a pull request's own corpus rather than
+      the last tagged one — every other caller should leave this unset, which
+      fetches the corpus at `version` like the description above says.
+    required: false
+  binary-path:
+    description: >
+      Internal: a specmatrix binary already on disk to run instead of
+      downloading a release. Used only by this repository's own dogfood
+      workflow, which builds the pull request's own code; every other caller
+      should leave this unset.
+    required: false
+
+outputs:
+  verdict-summary:
+    description: One line — counts of pass/reject/alter/n/a — the same line the table prints.
+    value: ${{ steps.run.outputs.verdict-summary }}
+  matrix-json:
+    description: Path to the JSON result this run wrote.
+    value: ${{ steps.run.outputs.matrix-json }}
+
+runs:
+  using: composite
+  steps:
+    # Only actually needed to pick a corpus ref or a binary to download —
+    # both of which `ref`/`binary-path` can override. Skipped when both are
+    # given, so the dogfood workflow (which always gives both) never depends
+    # on a release existing at all, which matters before this project's own
+    # first tag is ever pushed.
+    - name: resolve version
+      id: version
+      if: inputs.ref == '' || inputs.binary-path == ''
+      shell: bash
+      run: |
+        set -euo pipefail
+        version="${{ inputs.version }}"
+        if [ "$version" = "latest" ]; then
+          version="$(curl --proto '=https' --tlsv1.2 --silent --show-error --fail \
+            https://api.github.com/repos/DeviousCardi/specmatrix/releases/latest \
+            | grep -m1 '"tag_name"' | sed -E 's/.*"tag_name": *"([^"]+)".*/\1/')"
+          if [ -z "$version" ]; then
+            echo "::error::could not resolve the latest specmatrix release" >&2
+            exit 1
+          fi
+        fi
+        echo "resolved to $version"
+        echo "version=$version" >> "$GITHUB_OUTPUT"
+
+    # The corpus (cases/, and the built-in backends/) is fetched at the same
+    # tag as the binary, into a directory the caller's own checkout cannot
+    # collide with. A result names one version of the runner and one version
+    # of the corpus it ran, never a mix.
+    - name: fetch the corpus at that version
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        repository: DeviousCardi/specmatrix
+        ref: ${{ inputs.ref || steps.version.outputs.version }}
+        path: .specmatrix-corpus
+        sparse-checkout: |
+          cases
+          backends
+        sparse-checkout-cone-mode: false
+
+    - name: download the runner binary
+      if: inputs.binary-path == ''
+      shell: bash
+      run: |
+        set -euo pipefail
+        version="${{ steps.version.outputs.version }}"
+        case "$(uname -m)" in
+          x86_64|amd64)  target=x86_64-unknown-linux-gnu ;;
+          aarch64|arm64) target=aarch64-unknown-linux-gnu ;;
+          *) echo "::error::no specmatrix release binary for $(uname -m)" >&2; exit 1 ;;
+        esac
+        archive="specmatrix-${version}-${target}.tar.gz"
+        url="https://github.com/DeviousCardi/specmatrix/releases/download/${version}/${archive}"
+        echo "downloading $url"
+        curl --proto '=https' --tlsv1.2 --location --silent --show-error --fail "$url" -o "$RUNNER_TEMP/${archive}"
+        tar -xzf "$RUNNER_TEMP/${archive}" -C "$RUNNER_TEMP"
+        chmod +x "$RUNNER_TEMP/specmatrix"
+        echo "$RUNNER_TEMP" >> "$GITHUB_PATH"
+
+    - name: use the given binary
+      if: inputs.binary-path != ''
+      shell: bash
+      run: |
+        set -euo pipefail
+        mkdir -p "$RUNNER_TEMP/specmatrix-bin"
+        cp "${{ inputs.binary-path }}" "$RUNNER_TEMP/specmatrix-bin/specmatrix"
+        chmod +x "$RUNNER_TEMP/specmatrix-bin/specmatrix"
+        echo "$RUNNER_TEMP/specmatrix-bin" >> "$GITHUB_PATH"
+
+    - name: run the suite
+      id: run
+      shell: bash
+      run: |
+        set -euo pipefail
+        corpus=".specmatrix-corpus"
+        backend_input="${{ inputs.backend }}"
+        if [[ "$backend_input" == */* ]]; then
+          # A path into the caller's own checkout: split it into the
+          # directory `--backends` reads and the bare name `--backend` names,
+          # since the CLI resolves the two together as <dir>/<name>.yaml.
+          backends_dir="$(dirname "$backend_input")"
+          backend_name="$(basename "$backend_input" .yaml)"
+        else
+          backends_dir="$corpus/backends"
+          backend_name="$backend_input"
+        fi
+
+        specmatrix run \
+          --backend "$backend_name" \
+          --backends "$backends_dir" \
+          --suite "${{ inputs.suite }}" \
+          --cases "$corpus/cases" \
+          --url "${{ inputs.url }}" \
+          --json > matrix.json
+        specmatrix run \
+          --backend "$backend_name" \
+          --backends "$backends_dir" \
+          --suite "${{ inputs.suite }}" \
+          --cases "$corpus/cases" \
+          --url "${{ inputs.url }}"
+
+        echo "matrix-json=matrix.json" >> "$GITHUB_OUTPUT"
+        python3 "${{ github.action_path }}/tools/summarize.py" matrix.json >> "$GITHUB_STEP_SUMMARY"
+        summary="$(python3 "${{ github.action_path }}/tools/summarize.py" matrix.json --line-only)"
+        echo "verdict-summary=$summary" >> "$GITHUB_OUTPUT"

src/backend.rs

@@ -53,6 +53,17 @@ pub struct Backend {
     /// difference between measuring the backend and measuring a race.
     pub setup_verify: Option<Verify>,
     pub teardown: Option<Request>,
+    /// Case ids a maintainer has read and accepted, each with a one-line
+    /// reason. Read by the CI action (`Part H` of the 1.0 plan), never by the
+    /// runner's own verdict logic: a verdict here is decided from the wire
+    /// alone, the same as every other column, so a badly behaved backend
+    /// cannot quietly turn its own `ALTER` into a `PASS` by adding an entry.
+    /// What `allow` changes is only how the action's job summary presents the
+    /// result — the allowed cases move to their own section instead of
+    /// reading as unreviewed failures — because the job already never fails
+    /// on a verdict, only on a harness error.
+    #[serde(default)]
+    pub allow: HashMap<String, String>,
 }
 
 #[derive(Debug, Deserialize, Clone)]

src/matrix.rs

@@ -269,6 +269,7 @@ mod tests {
                     title: String::new(),
                     verdict,
                     detail: detail.to_string(),
+                    allowed_reason: None,
                 })
                 .collect(),
         }