diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml index 8bac031295..922dbca999 100644 --- a/.github/workflows/build-release.yaml +++ b/.github/workflows/build-release.yaml @@ -1,11 +1,15 @@ # (C) 2022 GoodData Corporation -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. +# The single workflow that publishes to PyPI -- both stable and dev releases. +# +# * Stable release: triggered by pushing a vX.Y.Z tag (see bump-version.yaml). Builds every +# component, creates the GitHub release, publishes to PyPI and notifies #releases. +# * Dev release: triggered manually (workflow_dispatch). Bumps the checked-out branch to the +# next N.N.N.devN version, builds and publishes to PyPI. No GitHub release, no notification. +# +# Both paths must stay in this one file. PyPI attestations record the workflow file that +# published each release, and PyPI flags a "Change of provenance" whenever it differs from +# the previous release's. With dev and stable releases in separate files, every stable +# release that followed a dev release was flagged. name: Build Python Package and Create Release @@ -13,13 +17,24 @@ on: push: tags: - v*.*.* + workflow_dispatch: + inputs: + BRANCH_NAME: + type: string + description: 'Branch name to dev-release from' + default: "master" -# One release at a time. Each tag is unique, so runs never collide on the tag itself, but -# is_latest is computed once per run: with two releases in flight, the earlier tag can -# finish last and take the "Latest" badge back from the newer one. Serializing keeps the -# badge in release order. Never cancels -- a cancelled run leaves components half-published. +# One release of each kind at a time. Each tag is unique, so runs never collide on the tag +# itself, but is_latest is computed once per run: with two releases in flight, the earlier +# tag can finish last and take the "Latest" badge back from the newer one. Serializing keeps +# the badge in release order. For dev releases it keeps two runs from computing the same +# devN version from PyPI. Never cancels -- a cancelled run leaves components half-published. +# +# Stable and dev runs use separate groups: a group holds only one pending run, and a newer +# pending run cancels it. In a shared group, a dev release dispatched while a tag release +# waits would cancel the tag release. concurrency: - group: build-release + group: build-release-${{ github.event_name }} cancel-in-progress: false env: @@ -27,14 +42,44 @@ env: jobs: matrix-components: - name: Prepare matrix components + name: Prepare components and version runs-on: ubuntu-latest + permissions: + contents: read outputs: components: ${{ steps.export.outputs.components }} + # Tag push: the tag. Dev release: the branch head, resolved once so every build job + # builds the same commit even if the branch moves mid-run. + ref: ${{ steps.dev.outputs.sha || github.ref }} + # Dev release only. Computed once and shared, so all components get the same version + # and re-running a failed build job reuses it instead of picking the next devN. + dev_version: ${{ steps.dev.outputs.version }} steps: - name: Export components JSON id: export run: echo "components=${COMPONENTS}" >> "$GITHUB_OUTPUT" + # The script runs from this workflow's revision, not from BRANCH_NAME, so dev releases + # also work from branches that predate it. The branch only supplies the commit to build + # and the version to start from. + - name: Checkout the workflow revision + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v5 + with: + persist-credentials: false + - name: Compute dev version + id: dev + if: github.event_name == 'workflow_dispatch' + env: + BRANCH_NAME: ${{ inputs.BRANCH_NAME }} + run: | + git fetch --no-tags --depth=1 origin "$BRANCH_NAME" + sha=$(git rev-parse FETCH_HEAD) + git show "$sha:pyproject.toml" > "$RUNNER_TEMP/branch-pyproject.toml" + # shellcheck disable=SC2046 # one argument per component + version=$(python3 scripts/next_dev_version.py --pyproject "$RUNNER_TEMP/branch-pyproject.toml" $(jq -r '.[]' <<< "$COMPONENTS")) + echo "Dev release $version from $sha" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" build: name: Build all components @@ -47,10 +92,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 + with: + ref: ${{ needs.matrix-components.outputs.ref }} + persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@v10.0.1 with: prune-cache: true + - name: Bump to dev version + if: github.event_name == 'workflow_dispatch' + env: + DEV_VERSION: ${{ needs.matrix-components.outputs.dev_version }} + run: | + uv sync --only-group release --locked + uv run tbump --only-patch --non-interactive "$DEV_VERSION" - name: Build ${{ matrix.component }} run: | if [ "${{ matrix.component }}" = "gooddata-api-client" ]; then @@ -69,6 +124,7 @@ jobs: tag-checks: name: Check the triggering tag + if: github.event_name == 'push' runs-on: ubuntu-latest permissions: contents: read @@ -84,6 +140,7 @@ jobs: github_release: name: Create GitHub release + if: github.event_name == 'push' runs-on: ubuntu-latest needs: - build @@ -140,6 +197,7 @@ jobs: packages-dir: dist/${{ matrix.component }} verbose: true slack-notification: + if: github.event_name == 'push' runs-on: ubuntu-latest needs: publish steps: diff --git a/.github/workflows/dev-release.yaml b/.github/workflows/dev-release.yaml deleted file mode 100644 index 9b3b6a3c03..0000000000 --- a/.github/workflows/dev-release.yaml +++ /dev/null @@ -1,84 +0,0 @@ -name: Dev release from master - -on: - workflow_dispatch: - inputs: - BRANCH_NAME: - type: string - description: 'Branch name to release from' - default: "master" - -env: - COMPONENTS: '["gooddata-api-client","gooddata-pandas","gooddata-fdw","gooddata-sdk","gooddata-dbt","gooddata-flight-server","gooddata-flexconnect","gooddata-pipelines","gooddata-eval"]' - -jobs: - matrix-components: - name: Prepare matrix components - runs-on: ubuntu-latest - outputs: - components: ${{ steps.export.outputs.components }} - steps: - - name: Export components JSON - id: export - run: echo "components=${COMPONENTS}" >> "$GITHUB_OUTPUT" - - build: - name: Build ${{ matrix.component }} - runs-on: ubuntu-latest - needs: matrix-components - permissions: - contents: read - strategy: - matrix: - component: ${{ fromJSON(needs.matrix-components.outputs.components) }} - steps: - - name: Checkout Repository - uses: actions/checkout@v5 - with: - ref: ${{ inputs.BRANCH_NAME }} - - name: Install uv - uses: astral-sh/setup-uv@v10.0.1 - with: - prune-cache: true - - name: Install dependencies - run: | - uv sync --only-group release --locked - - name: Bump to dev - run: uv run tbump --only-patch --non-interactive $(./scripts/next_dev.sh) - - name: Build ${{ matrix.component }} - run: | - if [ "${{ matrix.component }}" = "gooddata-api-client" ]; then - cd ${{ matrix.component }} - else - cd packages/${{ matrix.component }} - fi - uv build --out-dir dist - - name: Upload ${{ matrix.component }} artifacts - uses: actions/upload-artifact@v6 - with: - name: dist-${{ matrix.component }} - path: ${{ matrix.component == 'gooddata-api-client' && format('{0}/dist', matrix.component) || format('packages/{0}/dist', matrix.component) }} - if-no-files-found: error - - publish: - name: Publish ${{ matrix.component }} - runs-on: ubuntu-latest - needs: - - matrix-components - - build - permissions: - id-token: write - strategy: - matrix: - component: ${{ fromJSON(needs.matrix-components.outputs.components) }} - steps: - - name: Download ${{ matrix.component }} artifacts - uses: actions/download-artifact@v6 - with: - name: dist-${{ matrix.component }} - path: dist - - name: Push ${{ matrix.component}} to pypi - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: dist - verbose: true diff --git a/.github/workflows/rw-collect-changes.yaml b/.github/workflows/rw-collect-changes.yaml index 9954e15436..4815017216 100644 --- a/.github/workflows/rw-collect-changes.yaml +++ b/.github/workflows/rw-collect-changes.yaml @@ -41,6 +41,8 @@ jobs: - '.docker/**' - 'packages/**' - 'scripts/docs/**' + - 'scripts/next_dev_version.py' + - 'scripts/tests/**' - '*.mk' - 'Makefile' - Dockerfile diff --git a/AGENTS.md b/AGENTS.md index a5e08c9426..c9f872e431 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -287,8 +287,8 @@ release. Add a dependency to the package's `pyproject.toml`, then run `uv lock`. Three entries in the root `pyproject.toml` — `[project].dependencies`, `[tool.uv.sources]`, and `[tool.uv.workspace].members` (listed explicitly, not globbed, so -a stray directory cannot break every `uv` command). Then add it to `COMPONENTS` in both -`.github/workflows/dev-release.yaml` and `build-release.yaml`, to the Codecov file list in +a stray directory cannot break every `uv` command). Then add it to `COMPONENTS` in +`.github/workflows/build-release.yaml`, to the Codecov file list in `rw-python-tests.yaml`, and to `scopes` in `.gitlint`. ## Documentation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72c3daec8a..9ebef86626 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,11 +44,9 @@ When adding a new distributable package to this monorepo, update release automation and PyPI configuration together: -1. Add the package name to `COMPONENTS` in: - - `.github/workflows/dev-release.yaml` - - `.github/workflows/build-release.yaml` +1. Add the package name to `COMPONENTS` in `.github/workflows/build-release.yaml` (it publishes both stable and dev releases). 2. Verify the package is built by release workflows and artifacts are uploaded from its `dist/` directory. -3. Configure the package on PyPI to use Trusted Publisher for this repository/workflow combination. +3. Configure the package on PyPI to use Trusted Publisher for this repository and the `build-release.yaml` workflow. 4. Run/observe a release workflow and confirm publishing succeeds via OIDC (no `PYPI_API_TOKEN` required). ### Adding Support for a New Python Version diff --git a/MAINTENANCE.md b/MAINTENANCE.md index 8e25abf506..7b32da7e50 100644 --- a/MAINTENANCE.md +++ b/MAINTENANCE.md @@ -109,7 +109,12 @@ The docs site keeps the four newest release branches, sorted by `major.minor`, a appears in the docs. ### How-to dev release -To publish current master as a dev release version, use [Dev release from master](.github/workflows/dev-release.yaml) GitHub workflow. +To publish current master as a dev release version, manually run the +[Build Python Package and Create Release](.github/workflows/build-release.yaml) workflow (optionally with a +different `BRANCH_NAME`). A manual run bumps to the next `N.N.N.devN` version and publishes to PyPI only, with +no GitHub release and no Slack notification. Dev and stable releases share one workflow on purpose: PyPI +shows a "Change of provenance" warning when a release is published from a different workflow file than the +previous one. ### Errors that may appear diff --git a/Makefile b/Makefile index 1da4a4240c..b9c747d1c8 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,7 @@ remove-cassettes: .PHONY: test-docs-scripts test-docs-scripts: - uv run pytest scripts/docs/tests/ -v + uv run pytest scripts/docs/tests/ scripts/tests/ -v .PHONY: new-docs new-docs: diff --git a/scripts/next_dev.sh b/scripts/next_dev.sh deleted file mode 100755 index 3b2fd10477..0000000000 --- a/scripts/next_dev.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# (C) 2023 GoodData Corporation - -set -e - -# TODO: once alpha releases are removed, remove the grep -vE 'a[0-9]+$' part -latest_version=$( curl -L -s https://pypi.org/pypi/gooddata-sdk/json | jq -r '.releases | keys | .[]' | grep -vE 'a[0-9]+$' | sort -Vr | awk 'NR==1 {print}') - -if [[ $latest_version =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then - # split the version number string into its components - IFS='.' read -r x y z dev_part <<< "$latest_version" - - # extract the numeric portion of the dev component - dev_num="${dev_part#dev}" - - # increment the dev component and convert it back to a string - ((dev_num=dev_num+1)) - - # combine the components back into a single version number string - echo "$x.$y.$z.dev$dev_num" -elif [[ $latest_version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - # split the version number string into its components - IFS='.' read -r x y z <<< "$latest_version" - # increment the last version - ((z=z+1)) - echo "$x.$y.$z.dev1" -else - echo "Error occurred $latest_version does not match N.N.N nor N.N.N.devN pattern." - exit 1 -fi diff --git a/scripts/next_dev_version.py b/scripts/next_dev_version.py new file mode 100644 index 0000000000..9ea7f3fb3c --- /dev/null +++ b/scripts/next_dev_version.py @@ -0,0 +1,94 @@ +# (C) 2026 GoodData Corporation +"""Print the next dev release version for a branch. + +Usage: next_dev_version.py [--pyproject PATH] COMPONENT [COMPONENT ...] + +PATH is the branch's pyproject.toml (default: the one next to this script), so the script +can run from a newer revision than the branch being released. + +The base is the branch's own version with the patch bumped: 1.79.0 in pyproject.toml gives +1.79.1.devN. N is one past the highest 1.79.1.devN already on PyPI for any of the given +components, so every component of one dev release gets the same version even if an earlier +run published only some of them. + +Only versions with exactly that base are compared, so there is no general version ordering +to get wrong (`sort -V` ranks 1.79.1.dev10 above 1.79.1, PEP 440 does the opposite). +""" + +from __future__ import annotations + +import argparse +import json +import re +import urllib.error +import urllib.request +from collections.abc import Iterable +from pathlib import Path + +import tomllib + +_ROOT_DIR = Path(__file__).resolve().parent.parent +_STABLE = re.compile(r"^(?P\d+)\.(?P\d+)\.(?P\d+)$") +_VERSION = re.compile(r"^(?P\d+\.\d+\.\d+)(?:\.dev(?P\d+))?$") + + +def dev_base(current: str) -> str: + """Return the version the dev releases of a branch at ``current`` lead up to.""" + match = _STABLE.match(current) + if match is None: + raise ValueError(f"Branch version {current!r} is not N.N.N") + return f"{match['major']}.{match['minor']}.{int(match['patch']) + 1}" + + +def next_dev_version(base: str, released: Iterable[str]) -> str: + """Return ``base.devN`` with N one past the highest dev number already released for ``base``.""" + dev_numbers = [] + for version in released: + match = _VERSION.match(version) + if match is None or match["base"] != base: + continue + if match["dev"] is None: + # A dev release of an already released version would sort below it. + raise ValueError(f"{base} is already released; the branch is behind, merge the release into it first") + dev_numbers.append(int(match["dev"])) + return f"{base}.dev{max(dev_numbers, default=0) + 1}" + + +def released_versions(component: str) -> list[str]: + """Return every version of ``component`` on PyPI, yanked ones included (their numbers stay taken).""" + request = urllib.request.Request( + f"https://pypi.org/pypi/{component}/json", + headers={"Accept": "application/json", "User-Agent": "gooddata-python-sdk-release"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as e: + if e.code == 404: # a component that has never been published + return [] + raise + # Fail rather than read an unexpected shape as "nothing released" and reuse a devN. + releases = data.get("releases") if isinstance(data, dict) else None + if not isinstance(releases, dict): + raise ValueError(f"Unexpected PyPI response for {component}: no 'releases' mapping") + return list(releases) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Print the next dev release version for a branch.") + parser.add_argument( + "--pyproject", + type=Path, + default=_ROOT_DIR / "pyproject.toml", + help="pyproject.toml of the branch being released", + ) + parser.add_argument("components", nargs="+", metavar="COMPONENT", help="PyPI project to check") + args = parser.parse_args(argv) + with open(args.pyproject, "rb") as f: + current = tomllib.load(f)["tool"]["tbump"]["version"]["current"] + released = [version for component in args.components for version in released_versions(component)] + print(next_dev_version(dev_base(current), released)) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/conftest.py b/scripts/tests/conftest.py new file mode 100644 index 0000000000..7226b7f040 --- /dev/null +++ b/scripts/tests/conftest.py @@ -0,0 +1,6 @@ +# (C) 2026 GoodData Corporation +import sys +from pathlib import Path + +# The scripts are standalone files, not a package: make them importable by module name. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/scripts/tests/test_next_dev_version.py b/scripts/tests/test_next_dev_version.py new file mode 100644 index 0000000000..7b1c6516c2 --- /dev/null +++ b/scripts/tests/test_next_dev_version.py @@ -0,0 +1,76 @@ +# (C) 2026 GoodData Corporation +from __future__ import annotations + +import io +import json +import urllib.request +from pathlib import Path + +import pytest +from next_dev_version import dev_base, main, next_dev_version, released_versions + + +def test_dev_base_bumps_patch() -> None: + assert dev_base("1.79.0") == "1.79.1" + assert dev_base("1.60.9") == "1.60.10" + + +@pytest.mark.parametrize("current", ["1.79", "1.79.1.dev2", "1.79.0a1", ""]) +def test_dev_base_rejects_non_stable(current: str) -> None: + with pytest.raises(ValueError): + dev_base(current) + + +def test_first_dev_release() -> None: + assert next_dev_version("1.79.1", ["1.78.0", "1.78.1.dev4", "1.79.0"]) == "1.79.1.dev1" + + +def test_compares_dev_numbers_numerically() -> None: + assert next_dev_version("1.79.1", ["1.79.0", "1.79.1.dev9", "1.79.1.dev10"]) == "1.79.1.dev11" + + +def test_ignores_other_lines() -> None: + # A newer minor or an older line's dev releases do not affect the base. + released = ["1.60.1", "1.60.2.dev3", "1.79.0", "1.79.1.dev5", "1.80.0a1"] + assert next_dev_version("1.60.2", released) == "1.60.2.dev4" + + +def test_union_of_components_after_partial_publish() -> None: + # gooddata-sdk failed to publish dev3, gooddata-pandas got it: dev3 is taken. + sdk = ["1.79.0", "1.79.1.dev1", "1.79.1.dev2"] + pandas = ["1.79.0", "1.79.1.dev1", "1.79.1.dev2", "1.79.1.dev3"] + assert next_dev_version("1.79.1", sdk + pandas) == "1.79.1.dev4" + + +def test_refuses_dev_of_released_version() -> None: + # The branch is still at 1.79.0 while 1.79.1 is out: 1.79.1.devN would sort below 1.79.1. + with pytest.raises(ValueError, match="already released"): + next_dev_version("1.79.1", ["1.79.0", "1.79.1.dev4", "1.79.1"]) + + +def _serve(monkeypatch: pytest.MonkeyPatch, payload: object) -> None: + body = json.dumps(payload).encode() + monkeypatch.setattr(urllib.request, "urlopen", lambda *_, **__: io.BytesIO(body)) + + +def test_released_versions_reads_release_keys(monkeypatch: pytest.MonkeyPatch) -> None: + _serve(monkeypatch, {"releases": {"1.79.0": [], "1.79.1.dev1": []}}) + assert released_versions("gooddata-sdk") == ["1.79.0", "1.79.1.dev1"] + + +@pytest.mark.parametrize("payload", [{"releases": []}, {"info": {}}, []]) +def test_released_versions_rejects_unexpected_shape(monkeypatch: pytest.MonkeyPatch, payload: object) -> None: + _serve(monkeypatch, payload) + with pytest.raises(ValueError, match="Unexpected PyPI response"): + released_versions("gooddata-sdk") + + +def test_main_uses_given_pyproject( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + # An old release line: its own version decides the base, not the script's checkout. + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.tbump.version]\ncurrent = "1.60.1"\n') + _serve(monkeypatch, {"releases": {"1.60.1": [], "1.60.2.dev1": [], "1.79.0": []}}) + main(["--pyproject", str(pyproject), "gooddata-sdk"]) + assert capsys.readouterr().out == "1.60.2.dev2\n"