From e889e7d46eb67690cc1528c2a7dfc2b9a472bc9a Mon Sep 17 00:00:00 2001 From: Glenn Gore Date: Wed, 16 Sep 2026 15:28:33 +0200 Subject: [PATCH] ci: build all five implementations and gate on unexplained failures and regressions Signed-off-by: Glenn Gore --- .github/workflows/conformance.yml | 130 ++++++++++++++++++++++++++++++ .gitignore | 1 + README.md | 19 +++++ findings.toml | 8 +- runner/src/main.rs | 52 +++++++++++- runner/src/report.rs | 4 + 6 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/conformance.yml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..2d98fa2 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,130 @@ +name: conformance + +# Builds all five implementations from source and runs the full matrix. +# Gate: `--gate known` fails on any failure no finding in findings.toml explains, +# on a failure matching a finding marked `fixed` (a regression), and on any +# driver that did not build or start. + +on: + push: + branches: [main] + pull_request: + schedule: + - cron: "17 3 * * 1" # weekly, to catch upstream moves + workflow_dispatch: + inputs: + affinidi_tdk_rs_ref: + description: affinidi/affinidi-tdk-rs ref + default: main + vta_browser_plugin_ref: + description: OpenVTC/vta-browser-plugin ref (tsp-js) + default: feat/tsp-rev2-rev3-dual-handler + affinidi_tsp_go_ref: + description: affinidi/affinidi-tsp-go ref + default: main + affinidi_tsp_dart_ref: + description: affinidi/affinidi-tsp-dart ref + default: main + +permissions: + contents: read + +concurrency: + group: conformance-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + +jobs: + matrix: + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + # The drivers build each implementation from a sibling checkout, so the + # workspace mirrors the layout in README.md. + - uses: actions/checkout@v5 + with: + path: tsp-conformance + - uses: actions/checkout@v5 + with: + repository: affinidi/affinidi-tdk-rs + ref: ${{ inputs.affinidi_tdk_rs_ref || 'main' }} + path: affinidi-tdk-rs + - uses: actions/checkout@v5 + with: + repository: OpenVTC/vta-browser-plugin + ref: ${{ inputs.vta_browser_plugin_ref || 'feat/tsp-rev2-rev3-dual-handler' }} + path: pnm-browser-plugin + - uses: actions/checkout@v5 + with: + repository: affinidi/affinidi-tsp-go + ref: ${{ inputs.affinidi_tsp_go_ref || 'main' }} + path: affinidi-tsp-go + - uses: actions/checkout@v5 + with: + repository: affinidi/affinidi-tsp-dart + ref: ${{ inputs.affinidi_tsp_dart_ref || 'main' }} + path: affinidi-tsp-dart + + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: | + tsp-conformance/runner -> target + tsp-conformance/drivers/affinidi-rust -> target + tsp-conformance/drivers/reference-rust -> target + - uses: actions/setup-node@v5 + with: + node-version: 24 + cache: npm + cache-dependency-path: pnm-browser-plugin/package-lock.json + - uses: actions/setup-go@v6 + with: + go-version: "1.27" + cache-dependency-path: tsp-conformance/drivers/go/go.sum + - uses: dart-lang/setup-dart@v1 + with: + sdk: stable + + - name: Install tsp-js dependencies + working-directory: pnm-browser-plugin + run: npm ci --ignore-scripts --no-audit --no-fund + + - name: Runner unit tests + working-directory: tsp-conformance + run: cargo +stable test --manifest-path runner/Cargo.toml + + - name: Build runner + working-directory: tsp-conformance + run: cargo +stable build --release --manifest-path runner/Cargo.toml + + - name: Build drivers and run the matrix + working-directory: tsp-conformance + run: | + sha() { git -C "../$1" rev-parse --short HEAD; } + note="CI run ${GITHUB_RUN_ID}: affinidi-tdk-rs $(sha affinidi-tdk-rs), vta-browser-plugin $(sha pnm-browser-plugin), affinidi-tsp-go $(sha affinidi-tsp-go), affinidi-tsp-dart $(sha affinidi-tsp-dart), tsp_sdk from crates.io" + runner/target/release/tsp-conformance \ + --build \ + --seed "${GITHUB_RUN_ID}" \ + --gate known \ + --note "$note" \ + --json ci-report/report.json \ + --markdown ci-report/report.md + + - name: Summary + if: always() + working-directory: tsp-conformance + run: | + if [ -f ci-report/report.md ]; then + sed -n '1,/^## Capability matrix/p' ci-report/report.md | sed '$d' >> "$GITHUB_STEP_SUMMARY" + fi + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: conformance-report + path: | + tsp-conformance/ci-report/ + tsp-conformance/reports/logs/ + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 213c5b5..ca10c53 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ target/ reports/logs/ drivers/*/node_modules/ drivers.local.toml +ci-report/ diff --git a/README.md b/README.md index 0296579..58f4550 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,25 @@ skipped, and the report says why. - **Warnings** (a correct rejection under an unexpected error code) and **skip reasons**. +## CI and the gate + +`.github/workflows/conformance.yml` checks out all four implementation repos +next to the suite, builds every driver from source and runs the full matrix on +every push and pull request, weekly, and on demand (with a ref per repo). The +seed is the run id, so each run uses fresh keys and the report says which. + +CI runs with `--gate known`. `make run` keeps the default `--gate all`, which is +non-zero on any failure. `known` is non-zero only when: + +- a failure matches no finding in `findings.toml` — a new disagreement, which + needs investigating and either a fix or a new finding; +- a failure matches a finding marked `fixed` — a regression; +- a driver did not build or start. + +Findings that matched nothing are printed as a note, since the cause may have +been fixed upstream; mark them `fixed = ""` so that a recurrence fails. +The report and driver logs are uploaded as the `conformance-report` artifact. + ## Adding an implementation 1. Create `drivers//` with a program that reads one JSON request per line diff --git a/findings.toml b/findings.toml index d40d030..1ad7273 100644 --- a/findings.toml +++ b/findings.toml @@ -10,7 +10,10 @@ # kind: "spec-violation" an implementation contradicts a MUST # "disagreement" implementations differ where the spec is silent or ambiguous # "limit" an implementation-chosen bound the spec does not set -# "api-gap" the library cannot do what the protocol needs; integrators must +# "api-gap" the library cannot do what the protocol needs; integrators must add it themselves +# +# fixed: where the cause was fixed. A failure matching a fixed finding is a +# regression, and `--gate known` (the CI gate) fails on it. [[finding]] id = "sdk-non-canonical-lead-bytes" @@ -62,6 +65,7 @@ variants, so this is a gap to close in the spec as much as a behaviour to align. [[finding]] id = "xscs-body-not-single-bytes" +fixed = "affinidi-tdk-rs#803, vta-browser-plugin#249, affinidi-tsp-go and affinidi-tsp-dart main" kind = "disagreement" title = "An XSCS body other than exactly one Bytes primitive is accepted (tswg-tsp-specification#77)" match = ["xscs-body-", "data-after-payload-fields"] @@ -71,6 +75,7 @@ Three bodies are derived from the `direct-signed-only` vector and re-signed with [[finding]] id = "affinidi-accepter-forgets-reply-digest" +fixed = "affinidi-tdk-rs#804" kind = "spec-violation" title = "affinidi-tsp, as the accepting endpoint, discards a cancellation that names its own accept's digest" implementation = "affinidi-rust" @@ -105,6 +110,7 @@ could state a minimum every receiver must accept.""" [[finding]] id = "tsp-js-accept-correlation" +fixed = "vta-browser-plugin#250 (resolveAccept)" kind = "api-gap" title = "tsp-js's relationship rules accept an RFA that names an invite never sent" implementation = "tsp-js" diff --git a/runner/src/main.rs b/runner/src/main.rs index 300613b..f5068f3 100644 --- a/runner/src/main.rs +++ b/runner/src/main.rs @@ -69,6 +69,18 @@ struct Args { /// Known root causes used to group failures in the report. #[arg(long, default_value = "findings.toml")] findings: PathBuf, + /// What makes the exit status non-zero. `all`: any failing case. `known`: + /// only a failure no finding in `findings.toml` explains, or a driver that + /// did not run — the CI gate, which catches regressions and new + /// disagreements without going red on already-investigated ones. + #[arg(long, value_enum, default_value_t = Gate::All)] + gate: Gate, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, clap::ValueEnum)] +enum Gate { + All, + Known, } fn now_rfc3339() -> String { @@ -301,7 +313,45 @@ fn main() { eprintln!("{s:13} pass {:4} fail {:4} error {:3} skip {:4}", c.pass, c.fail, c.error, c.skip); } eprintln!("reports: {} {}", args.json.display(), args.markdown.display()); - std::process::exit(if failed > 0 { 1 } else { 0 }); + let red = match args.gate { + Gate::All => failed > 0, + Gate::Known => { + let fixed = |id: &str| rep.findings.iter().any(|f| f.id == id && f.fixed.is_some()); + let unexplained: Vec<_> = rep + .results + .iter() + .filter(|r| matches!(r.status, result::Status::Fail | result::Status::Error)) + .filter(|r| r.finding.as_deref().is_none_or(fixed)) + .collect(); + for r in &unexplained { + let why = match &r.finding { + Some(id) => format!("REGRESSION ({id})"), + None => "UNEXPLAINED".to_string(), + }; + eprintln!("{why} {} [{} -> {}] {}", r.case, r.sender, r.receiver, r.detail); + } + let not_run: Vec<_> = rep.drivers.iter().filter(|d| d.status != "ran").collect(); + for d in ¬_run { + eprintln!("DRIVER NOT RUN {}: {}", d.name, d.reason); + } + let stale: Vec<_> = rep + .findings + .iter() + .filter(|f| f.fixed.is_none() && !f.patterns.is_empty()) + .filter(|f| !rep.results.iter().any(|r| r.finding.as_deref() == Some(f.id.as_str()))) + .collect(); + for f in &stale { + eprintln!("note: finding {} matched no failure this run (fixed?)", f.id); + } + eprintln!( + "gate known: {} unexplained or regressed failure(s), {} driver(s) not run", + unexplained.len(), + not_run.len() + ); + !unexplained.is_empty() || !not_run.is_empty() + } + }; + std::process::exit(if red { 1 } else { 0 }); } fn write_out(path: &Path, text: &str) { diff --git a/runner/src/report.rs b/runner/src/report.rs index 9a6517b..f4d445e 100644 --- a/runner/src/report.rs +++ b/runner/src/report.rs @@ -55,6 +55,10 @@ pub struct Finding { pub spec: String, #[serde(default)] pub explanation: String, + /// Where the cause was fixed (e.g. a merged PR). A failure attributed to a + /// fixed finding is a regression: `--gate known` does not excuse it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fixed: Option, } #[derive(Deserialize)]