From 7bfe24018c304819ad28e5916e5b8bd5983096eb Mon Sep 17 00:00:00 2001 From: Lalit Maganti Date: Wed, 16 Sep 2026 15:11:24 +0100 Subject: [PATCH] ci: test the action against the recorder being shipped The action installs a release by tag, so the smoke test could not name the release a commit prepares: it has no assets yet. It named a published tag instead, hardcoded at v0.2.5 when the action landed and never bumped, so preparing v0.2.7 tested a recorder two releases old. Give the action a `recorder` input that takes a binary instead of a release, build the recorder from the checkout, and record with that. The smoke test now exercises the code being shipped rather than a published one, and there is no version in it to go stale. Installing a release is still worth covering, so one further recording does that with the release before this one, and `just release-prepare` keeps that version current the way it already keeps the action's default and the docs example. --- .github/actions/record-build/install.sh | 16 ++++++++++++- .github/workflows/ci.yml | 30 +++++++++++++++++++++--- RELEASING.md | 28 +++++++++++++--------- action.yml | 7 ++++++ infra/prepare-action-release | 21 ++++++++++++++++- tests/conformance/test_action_release.py | 22 ++++++++++++++++- 6 files changed, 107 insertions(+), 17 deletions(-) diff --git a/.github/actions/record-build/install.sh b/.github/actions/record-build/install.sh index c1d2988..d3969f3 100644 --- a/.github/actions/record-build/install.sh +++ b/.github/actions/record-build/install.sh @@ -16,12 +16,26 @@ case "$(uname -m)" in exit 1 ;; esac +directory=$(mktemp -d "$RUNNER_TEMP/buildprof.XXXXXXXX") +# Testing this action against a recorder that has no release yet: the caller +# built it, so there is nothing to download or verify. +if [[ -n ${BUILDPROF_RECORDER:-} ]]; then + if [[ ! -x $BUILDPROF_RECORDER ]]; then + echo "recorder is not an executable file: $BUILDPROF_RECORDER" >&2 + exit 1 + fi + { + echo "binary=$BUILDPROF_RECORDER" + echo "trace=$directory/output.buildprof" + } >> "$GITHUB_OUTPUT" + exit 0 +fi + if [[ ! $BUILDPROF_VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then echo 'version must be a release tag such as v0.2.5.' >&2 exit 1 fi -directory=$(mktemp -d "$RUNNER_TEMP/buildprof.XXXXXXXX") archive="buildprof-$target.tar.xz" url="https://github.com/LalitMaganti/buildprof/releases/download/$BUILDPROF_VERSION" curl --fail --silent --show-error --location "$url/$archive" -o "$directory/$archive" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d2fedc..8141476 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,12 +88,22 @@ jobs: path: zlib persist-credentials: false + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ env.RUST_VERSION }} + + # The release this commit prepares has no assets yet, so recording a + # release tag would test an older recorder than the one being shipped. + # Build the recorder from this checkout and hand it to the action. + - name: Build the recorder under test + run: cargo build --release --locked + - name: Record a zlib build and test run id: success uses: ./ with: - # Before publishing, the upcoming recorder assets are unavailable. - version: v0.2.5 + recorder: ${{ github.workspace }}/target/release/buildprof artifact-name: action-success-${{ matrix.runner }} command: | cmake -S zlib -B zlib-build -DCMAKE_BUILD_TYPE=Release @@ -105,18 +115,32 @@ jobs: continue-on-error: true uses: ./ with: - version: v0.2.5 + recorder: ${{ github.workspace }}/target/release/buildprof artifact-name: action-failure-${{ matrix.runner }} command: exit 7 + # Nothing above downloads anything, so this covers what installing a + # release does: fetch the asset for this architecture and check it + # against its checksum. The newest published release, kept current by + # infra/prepare-action-release. + - name: Record with an installed release + id: installed + uses: ./ + with: + version: v0.2.6 + artifact-name: action-installed-${{ matrix.runner }} + command: cmake --version + - name: Verify outcomes and uploads env: SUCCESS_URL: ${{ steps.success.outputs.artifact-url }} FAILURE_URL: ${{ steps.failure.outputs.artifact-url }} + INSTALLED_URL: ${{ steps.installed.outputs.artifact-url }} FAILURE_OUTCOME: ${{ steps.failure.outcome }} run: | test -n "$SUCCESS_URL" test -n "$FAILURE_URL" + test -n "$INSTALLED_URL" test "$FAILURE_OUTCOME" = failure ui: diff --git a/RELEASING.md b/RELEASING.md index eeaa69b..65ee516 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -22,10 +22,10 @@ Also create the empty `homebrew-tap` repository. `/v/` directory and is what the CLI opens, so they must match. Re-record `examples/*.buildprof` with this version if the trace format changed (see `examples/README.md`). -2. Run `just release-prepare` to update the action's default recorder version - and the docs' release-tag example from `Cargo.toml`. Run `cargo check` to - refresh `Cargo.lock`. Then run `just release-check`, commit, push, - and wait for CI. +2. Run `just release-prepare`, which reads `Cargo.toml` and updates the + action's default recorder version, the docs' release-tag example, and the + earlier release CI installs. Run `cargo check` to refresh `Cargo.lock`. + Then run `just release-check`, commit, push, and wait for CI. 3. Tag and push: `git tag v0.2.2 && git push origin v0.2.2`. 4. Review the draft release `Buildprof ` that appears on GitHub, edit the notes as needed, and publish it. @@ -60,17 +60,22 @@ Manual follow-ups: ## GitHub Action releases The action ships under the crate's `vX.Y.Z` tags. `just release-prepare` -updates its recorder default and the docs example from `Cargo.toml`; CI and -release checks catch drift. Keep release tags fixed. Consumers can use tags -or release SHAs with Dependabot for upgrade PRs. +updates its recorder default, the docs example, and the earlier release CI +installs, all from `Cargo.toml`; CI and release checks catch drift. Keep +release tags fixed. Consumers can use tags or release SHAs with Dependabot for +upgrade PRs. Before the first action release, CI permits the labeled v0.2.5 preview; release checks reject it. Remove the preview exception in `infra/prepare-action-release` after that release. -PR smoke tests select a published recorder explicitly, since upcoming assets -are unavailable. Conformance tests use the source build; `action-release.yml` -tests the released action's default recorder on x86_64 and ARM64 after publication. +The release being prepared has no assets, so the action's own default cannot +be tested before it is published. PR smoke tests record with a recorder built +from the checkout instead, through the action's `recorder` input, so they +exercise the code being shipped; one further recording installs the previous +release so that path stays covered, and `just release-prepare` keeps that +version current. After publication, `action-release.yml` tests the released +action's default recorder on x86_64 and ARM64. ## The release workflow is hand-edited @@ -78,7 +83,8 @@ tests the released action's default recorder on x86_64 and ARM64 after publicati cannot express are edited in by hand, and `allow-dirty = ["ci"]` in `dist-workspace.toml` stops dist from overwriting them: -- the release metadata check validates the action default and the docs example; +- the release metadata check validates the action default, the docs example, + and the release CI installs; - the GitHub release is created as a draft titled `Buildprof `; - `infra/release-notes` replaces the install section of dist's notes with the documented commands; the changelog and the download table stay as dist diff --git a/action.yml b/action.yml index c92eb0a..2df3a53 100644 --- a/action.yml +++ b/action.yml @@ -10,6 +10,12 @@ inputs: version: description: Buildprof release tag to install. default: v0.2.7 + recorder: + description: >- + Path to a recorder binary to use instead of installing a release. This + is how the action is tested against a build that has not been released + yet; builds that want a release should name one with `version`. + default: '' artifact-name: description: Unique name for this recording within the workflow run. default: buildprof-trace @@ -29,6 +35,7 @@ runs: env: ACTION_PATH: ${{ github.action_path }} BUILDPROF_VERSION: ${{ inputs.version }} + BUILDPROF_RECORDER: ${{ inputs.recorder }} run: bash "$ACTION_PATH/.github/actions/record-build/install.sh" - name: Record build diff --git a/infra/prepare-action-release b/infra/prepare-action-release index 776c851..9215b24 100644 --- a/infra/prepare-action-release +++ b/infra/prepare-action-release @@ -2,7 +2,12 @@ # Copyright 2026 The Buildprof Authors. # SPDX-License-Identifier: Apache-2.0 -"""Synchronize the action and its documentation with the crate release version.""" +"""Synchronize the action, its documentation, and its tests with the crate version. + +The action installs a release by tag, so the test that covers installing +one cannot name the release being prepared: it has no assets yet. It names +the release before it instead, which this keeps current. +""" from __future__ import annotations @@ -20,6 +25,16 @@ PREVIEW_REF = "c1c29c7359a3ccd8b9c9205fa01cc46a1de650f9" PREVIEW_NOTE = "# Unreleased preview" +def previous_release(root: Path, version: str) -> str: + """The newest release before `version`, which is the newest with assets.""" + changelog = (root / "CHANGELOG.md").read_text() + headings = re.findall(r"^## \[(\d+\.\d+\.\d+[^\]]*)\]", changelog, re.MULTILINE) + for entry in headings: + if f"v{entry}" != version: + return f"v{entry}" + raise ValueError(f"CHANGELOG.md names no release before {version}") + + def replace_once(text: str, pattern: str, replacement: str) -> str: updated, count = re.subn(pattern, lambda _: replacement, text, flags=re.MULTILINE) if count != 1: @@ -44,6 +59,10 @@ def prepare(root: Path, *, check: bool, allow_preview: bool) -> list[str]: r"(?<= - uses: LalitMaganti/buildprof@)[^\n]+", reference, ), + root / ".github/workflows/ci.yml": ( + r"(?<=^ version: )v[^\n]+", + previous_release(root, version), + ), } originals = {path: path.read_text() for path in replacements} updated = { diff --git a/tests/conformance/test_action_release.py b/tests/conformance/test_action_release.py index 9ec3078..b04e549 100644 --- a/tests/conformance/test_action_release.py +++ b/tests/conformance/test_action_release.py @@ -33,6 +33,22 @@ def release_tree(tmp_path: Path) -> Path: steps: - uses: LalitMaganti/buildprof@{release["PREVIEW_REF"]} {release["PREVIEW_NOTE"]} ''')) + (tmp_path / "CHANGELOG.md").write_text(dedent('''\ + ## [Unreleased] + + ## [0.2.5] - 2026-09-11 + + ## [0.2.4] - 2026-09-10 + ''')) + # The test that installs a release names the one before this version. + (tmp_path / ".github/workflows").mkdir(parents=True) + # Indented as the workflow is, since that is what the pattern anchors on. + (tmp_path / ".github/workflows/ci.yml").write_text( + " - name: Record with an installed release\n" + " uses: ./\n" + " with:\n" + " version: v0.2.4\n" + ) return tmp_path @@ -45,7 +61,7 @@ def test_prepare_next_release(release_tree: Path, version: str) -> None: } assert set(prepare(release_tree, check=True, allow_preview=True)) == { - "action.yml", "docs/usage.md" + "action.yml", "docs/usage.md", ".github/workflows/ci.yml" } assert all(path.read_text() == text for path, text in before.items()) prepare(release_tree, check=False, allow_preview=False) @@ -54,6 +70,9 @@ def test_prepare_next_release(release_tree: Path, version: str) -> None: usage = (release_tree / "docs/usage.md").read_text() assert f"uses: LalitMaganti/buildprof@v{version}\n" in usage assert release["PREVIEW_NOTE"] not in usage + workflow = (release_tree / ".github/workflows/ci.yml").read_text() + # The release under test has no assets, so CI installs the one before it. + assert " version: v0.2.5\n" in workflow assert prepare(release_tree, check=True, allow_preview=False) == [] assert prepare(release_tree, check=False, allow_preview=False) == [] @@ -68,6 +87,7 @@ def test_preview_is_not_releasable(release_tree: Path) -> None: [ ("action.yml", "default: v0.2.5", "default: v0.1.0"), ("docs/usage.md", "@v0.2.5", "@v0.1.0"), + (".github/workflows/ci.yml", "version: v0.2.4", "version: v0.1.0"), ], ) def test_check_detects_drift(