ci: build all five implementations and gate on unexplained failures and regressions

.github/workflows/conformance.yml

@@ -0,0 +1,130 @@
+name: conformance
+
+# Builds all five implementations from source and runs the full matrix.
+# Gate: `--gate known` fails on any failure no finding in findings.toml explains,
+# on a failure matching a finding marked `fixed` (a regression), and on any
+# driver that did not build or start.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  schedule:
+    - cron: "17 3 * * 1" # weekly, to catch upstream moves
+  workflow_dispatch:
+    inputs:
+      affinidi_tdk_rs_ref:
+        description: affinidi/affinidi-tdk-rs ref
+        default: main
+      vta_browser_plugin_ref:
+        description: OpenVTC/vta-browser-plugin ref (tsp-js)
+        default: feat/tsp-rev2-rev3-dual-handler
+      affinidi_tsp_go_ref:
+        description: affinidi/affinidi-tsp-go ref
+        default: main
+      affinidi_tsp_dart_ref:
+        description: affinidi/affinidi-tsp-dart ref
+        default: main
+
+permissions:
+  contents: read
+
+concurrency:
+  group: conformance-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  matrix:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      # The drivers build each implementation from a sibling checkout, so the
+      # workspace mirrors the layout in README.md.
+      - uses: actions/checkout@v5
+        with:
+          path: tsp-conformance
+      - uses: actions/checkout@v5
+        with:
+          repository: affinidi/affinidi-tdk-rs
+          ref: ${{ inputs.affinidi_tdk_rs_ref || 'main' }}
+          path: affinidi-tdk-rs
+      - uses: actions/checkout@v5
+        with:
+          repository: OpenVTC/vta-browser-plugin
+          ref: ${{ inputs.vta_browser_plugin_ref || 'feat/tsp-rev2-rev3-dual-handler' }}
+          path: pnm-browser-plugin
+      - uses: actions/checkout@v5
+        with:
+          repository: affinidi/affinidi-tsp-go
+          ref: ${{ inputs.affinidi_tsp_go_ref || 'main' }}
+          path: affinidi-tsp-go
+      - uses: actions/checkout@v5
+        with:
+          repository: affinidi/affinidi-tsp-dart
+          ref: ${{ inputs.affinidi_tsp_dart_ref || 'main' }}
+          path: affinidi-tsp-dart
+
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            tsp-conformance/runner -> target
+            tsp-conformance/drivers/affinidi-rust -> target
+            tsp-conformance/drivers/reference-rust -> target
+      - uses: actions/setup-node@v5
+        with:
+          node-version: 24
+          cache: npm
+          cache-dependency-path: pnm-browser-plugin/package-lock.json
+      - uses: actions/setup-go@v6
+        with:
+          go-version: "1.27"
+          cache-dependency-path: tsp-conformance/drivers/go/go.sum
+      - uses: dart-lang/setup-dart@v1
+        with:
+          sdk: stable
+
+      - name: Install tsp-js dependencies
+        working-directory: pnm-browser-plugin
+        run: npm ci --ignore-scripts --no-audit --no-fund
+
+      - name: Runner unit tests
+        working-directory: tsp-conformance
+        run: cargo +stable test --manifest-path runner/Cargo.toml
+
+      - name: Build runner
+        working-directory: tsp-conformance
+        run: cargo +stable build --release --manifest-path runner/Cargo.toml
+
+      - name: Build drivers and run the matrix
+        working-directory: tsp-conformance
+        run: |
+          sha() { git -C "../$1" rev-parse --short HEAD; }
+          note="CI run ${GITHUB_RUN_ID}: affinidi-tdk-rs $(sha affinidi-tdk-rs), vta-browser-plugin $(sha pnm-browser-plugin), affinidi-tsp-go $(sha affinidi-tsp-go), affinidi-tsp-dart $(sha affinidi-tsp-dart), tsp_sdk from crates.io"
+          runner/target/release/tsp-conformance \
+            --build \
+            --seed "${GITHUB_RUN_ID}" \
+            --gate known \
+            --note "$note" \
+            --json ci-report/report.json \
+            --markdown ci-report/report.md
+
+      - name: Summary
+        if: always()
+        working-directory: tsp-conformance
+        run: |
+          if [ -f ci-report/report.md ]; then
+            sed -n '1,/^## Capability matrix/p' ci-report/report.md | sed '$d' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: conformance-report
+          path: |
+            tsp-conformance/ci-report/
+            tsp-conformance/reports/logs/
+          if-no-files-found: ignore

.gitignore

@@ -2,3 +2,4 @@ target/
 reports/logs/
 drivers/*/node_modules/
 drivers.local.toml
+ci-report/

README.md

@@ -113,6 +113,25 @@ skipped, and the report says why.
 - **Warnings** (a correct rejection under an unexpected error code) and **skip
   reasons**.
 
+## CI and the gate
+
+`.github/workflows/conformance.yml` checks out all four implementation repos
+next to the suite, builds every driver from source and runs the full matrix on
+every push and pull request, weekly, and on demand (with a ref per repo). The
+seed is the run id, so each run uses fresh keys and the report says which.
+
+CI runs with `--gate known`. `make run` keeps the default `--gate all`, which is
+non-zero on any failure. `known` is non-zero only when:
+
+- a failure matches no finding in `findings.toml` — a new disagreement, which
+  needs investigating and either a fix or a new finding;
+- a failure matches a finding marked `fixed` — a regression;
+- a driver did not build or start.
+
+Findings that matched nothing are printed as a note, since the cause may have
+been fixed upstream; mark them `fixed = "<where>"` so that a recurrence fails.
+The report and driver logs are uploaded as the `conformance-report` artifact.
+
 ## Adding an implementation
 
 1. Create `drivers/<name>/` with a program that reads one JSON request per line

findings.toml

@@ -10,7 +10,10 @@
 #   kind: "spec-violation"   an implementation contradicts a MUST
 #         "disagreement"     implementations differ where the spec is silent or ambiguous
 #         "limit"            an implementation-chosen bound the spec does not set
-#         "api-gap"          the library cannot do what the protocol needs; integrators must
+#         "api-gap"          the library cannot do what the protocol needs; integrators must add it themselves
+#
+#   fixed: where the cause was fixed. A failure matching a fixed finding is a
+#          regression, and `--gate known` (the CI gate) fails on it.
 
 [[finding]]
 id = "sdk-non-canonical-lead-bytes"
@@ -62,6 +65,7 @@ variants, so this is a gap to close in the spec as much as a behaviour to align.
 
 [[finding]]
 id = "xscs-body-not-single-bytes"
+fixed = "affinidi-tdk-rs#803, vta-browser-plugin#249, affinidi-tsp-go and affinidi-tsp-dart main"
 kind = "disagreement"
 title = "An XSCS body other than exactly one Bytes primitive is accepted (tswg-tsp-specification#77)"
 match = ["xscs-body-", "data-after-payload-fields"]
@@ -71,6 +75,7 @@ Three bodies are derived from the `direct-signed-only` vector and re-signed with
 
 [[finding]]
 id = "affinidi-accepter-forgets-reply-digest"
+fixed = "affinidi-tdk-rs#804"
 kind = "spec-violation"
 title = "affinidi-tsp, as the accepting endpoint, discards a cancellation that names its own accept's digest"
 implementation = "affinidi-rust"
@@ -105,6 +110,7 @@ could state a minimum every receiver must accept."""
 
 [[finding]]
 id = "tsp-js-accept-correlation"
+fixed = "vta-browser-plugin#250 (resolveAccept)"
 kind = "api-gap"
 title = "tsp-js's relationship rules accept an RFA that names an invite never sent"
 implementation = "tsp-js"

runner/src/main.rs

@@ -69,6 +69,18 @@ struct Args {
     /// Known root causes used to group failures in the report.
     #[arg(long, default_value = "findings.toml")]
     findings: PathBuf,
+    /// What makes the exit status non-zero. `all`: any failing case. `known`:
+    /// only a failure no finding in `findings.toml` explains, or a driver that
+    /// did not run — the CI gate, which catches regressions and new
+    /// disagreements without going red on already-investigated ones.
+    #[arg(long, value_enum, default_value_t = Gate::All)]
+    gate: Gate,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)]
+enum Gate {
+    All,
+    Known,
 }
 
 fn now_rfc3339() -> String {
@@ -301,7 +313,45 @@ fn main() {
         eprintln!("{s:13} pass {:4}  fail {:4}  error {:3}  skip {:4}", c.pass, c.fail, c.error, c.skip);
     }
     eprintln!("reports: {} {}", args.json.display(), args.markdown.display());
-    std::process::exit(if failed > 0 { 1 } else { 0 });
+    let red = match args.gate {
+        Gate::All => failed > 0,
+        Gate::Known => {
+            let fixed = |id: &str| rep.findings.iter().any(|f| f.id == id && f.fixed.is_some());
+            let unexplained: Vec<_> = rep
+                .results
+                .iter()
+                .filter(|r| matches!(r.status, result::Status::Fail | result::Status::Error))
+                .filter(|r| r.finding.as_deref().is_none_or(fixed))
+                .collect();
+            for r in &unexplained {
+                let why = match &r.finding {
+                    Some(id) => format!("REGRESSION ({id})"),
+                    None => "UNEXPLAINED".to_string(),
+                };
+                eprintln!("{why}  {} [{} -> {}] {}", r.case, r.sender, r.receiver, r.detail);
+            }
+            let not_run: Vec<_> = rep.drivers.iter().filter(|d| d.status != "ran").collect();
+            for d in &not_run {
+                eprintln!("DRIVER NOT RUN  {}: {}", d.name, d.reason);
+            }
+            let stale: Vec<_> = rep
+                .findings
+                .iter()
+                .filter(|f| f.fixed.is_none() && !f.patterns.is_empty())
+                .filter(|f| !rep.results.iter().any(|r| r.finding.as_deref() == Some(f.id.as_str())))
+                .collect();
+            for f in &stale {
+                eprintln!("note: finding {} matched no failure this run (fixed?)", f.id);
+            }
+            eprintln!(
+                "gate known: {} unexplained or regressed failure(s), {} driver(s) not run",
+                unexplained.len(),
+                not_run.len()
+            );
+            !unexplained.is_empty() || !not_run.is_empty()
+        }
+    };
+    std::process::exit(if red { 1 } else { 0 });
 }
 
 fn write_out(path: &Path, text: &str) {

runner/src/report.rs

@@ -55,6 +55,10 @@ pub struct Finding {
     pub spec: String,
     #[serde(default)]
     pub explanation: String,
+    /// Where the cause was fixed (e.g. a merged PR). A failure attributed to a
+    /// fixed finding is a regression: `--gate known` does not excuse it.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub fixed: Option<String>,
 }
 
 #[derive(Deserialize)]