From 381f03315db2f7f8fb643d06d3e8da080c962cc1 Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 18 Aug 2026 13:29:21 +0000 Subject: [PATCH 1/2] Let an operator accept a live-benchmark miss with a recorded reason The live benchmark measures one model run per case against a provider whose output varies between identical runs, so its detection rate moves several points either side of the floor without any change to the reviewer. Six consecutive versions carry a tag and no published release as a result, which means a fix for a production fault cannot reach production because an unrelated metric moved. The release workflow can now be dispatched with a reason that accepts a miss and publishes anyway. The reason is required for the release to proceed, is echoed as a warning annotation, and is written to the run summary. The baseline is untouched, so the next release faces the same floor and an accepted miss stays legible as a miss rather than becoming the new normal. With no reason supplied the behaviour is unchanged and the benchmark still blocks. This is a release valve, not a fix for the measurement. A repeated-run acceptance rule remains the way to make the floor mean something. --- .github/workflows/release.yml | 47 +++++++++++++++++++++++++++++++---- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d85a85e..b61c945 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,15 @@ name: Release on: push: tags: ["v*"] + workflow_dispatch: + inputs: + bench_live_override_reason: + description: >- + Publish even when the live benchmark misses its floor, recording this + reason on the run. Leave empty to require the benchmark to pass. The + baseline is never rewritten; an accepted miss stays visible as a miss. + required: false + default: "" jobs: validate-tag: @@ -113,15 +122,43 @@ jobs: - name: Compare against the recorded baseline id: compare-baseline + # Failure is decided by the next step so an operator-supplied reason can + # accept a miss. Recording the miss and rewriting the baseline are + # different acts, and only the first one happens here. + continue-on-error: true working-directory: bench run: bun run bench:compare -- --result "${{ runner.temp }}/bench-live-report.json" - - name: Report benchmark regression - if: steps.compare-baseline.outcome == 'failure' + - name: Enforce the benchmark floor + env: + OVERRIDE_REASON: ${{ inputs.bench_live_override_reason }} + COMPARISON: ${{ steps.compare-baseline.outcome }} run: | - echo "::error::Release blocked by a benchmark regression. See the 'Compare against the recorded baseline' step above for the failing metric(s)." - echo "## Release blocked" >> "$GITHUB_STEP_SUMMARY" - echo "The live benchmark regressed past tolerance against \`bench/baseline.json\`. See the 'Compare against the recorded baseline' step log for the failing metric(s)." >> "$GITHUB_STEP_SUMMARY" + set -euo pipefail + if [[ "${COMPARISON}" == "success" ]]; then + echo "The live benchmark met its floor." + exit 0 + fi + if [[ -z "${OVERRIDE_REASON}" ]]; then + echo "::error::Release blocked by a benchmark regression. See the 'Compare against the recorded baseline' step above for the failing metric(s)." + { + echo "## Release blocked" + echo + echo "The live benchmark regressed past tolerance against \`bench/baseline.json\`." + echo "See the 'Compare against the recorded baseline' step log for the failing metric(s)." + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + echo "::warning title=Benchmark floor missed and accepted::${OVERRIDE_REASON}" + { + echo "## Benchmark floor missed and accepted" + echo + echo "The live benchmark missed its floor and this release was published anyway." + echo + echo "**Reason:** ${OVERRIDE_REASON}" + echo + echo "\`bench/baseline.json\` is unchanged, so the next release faces the same floor." + } >> "$GITHUB_STEP_SUMMARY" build: needs: [validate-tag, bench-live] diff --git a/Cargo.lock b/Cargo.lock index 6c72819..bde1bc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1104,7 +1104,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "postil-cli" -version = "0.8.22" +version = "0.8.23" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index bca50b2..a5b4324 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.8.22" +version = "0.8.23" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" From fac57f4ac3930908f08b0255901b5593316e1a6f Mon Sep 17 00:00:00 2001 From: Postil Maintainer Date: Tue, 18 Aug 2026 13:29:21 +0000 Subject: [PATCH 2/2] Set CLI package version to 0.8.23