diff --git a/.github/scripts/check_version_sync.py b/.github/scripts/check_version_sync.py new file mode 100644 index 0000000..7a57151 --- /dev/null +++ b/.github/scripts/check_version_sync.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +PACKAGE_JSON = ROOT / "package.json" +PY_VERSION = ROOT / "py" / "autoevals" / "version.py" + +package_version = json.loads(PACKAGE_JSON.read_text(encoding="utf-8"))["version"] +match = re.search( + r'^VERSION\s*=\s*["\']([^"\']+)["\']\s*$', + PY_VERSION.read_text(encoding="utf-8"), + re.MULTILINE, +) + +if not match: + print(f"Could not parse VERSION from {PY_VERSION}", file=sys.stderr) + sys.exit(1) + +python_version = match.group(1) + +if package_version != python_version: + print( + "Version mismatch detected:\n" + f"- package.json: {package_version}\n" + f"- py/autoevals/version.py: {python_version}", + file=sys.stderr, + ) + sys.exit(1) + +print(f"Versions are in sync: {package_version}") diff --git a/.github/workflows/publish-js.yaml b/.github/workflows/publish-js.yaml index e7a2884..4edfc78 100644 --- a/.github/workflows/publish-js.yaml +++ b/.github/workflows/publish-js.yaml @@ -20,6 +20,11 @@ on: required: true default: main type: string + prerelease_suffix: + description: Optional shared prerelease suffix + required: false + default: "" + type: string jobs: prepare-release: @@ -36,6 +41,8 @@ jobs: with: fetch-depth: 1 ref: ${{ inputs.branch }} + - name: Check version sync + run: python3 .github/scripts/check_version_sync.py - name: Set up Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: @@ -45,12 +52,17 @@ jobs: env: RELEASE_TYPE: ${{ inputs.release_type }} TARGET_BRANCH: ${{ inputs.branch }} + PRERELEASE_SUFFIX: ${{ inputs.prerelease_suffix }} run: | set -euo pipefail CURRENT_VERSION=$(node -p "require('./package.json').version") RELEASE_COMMIT=$(git rev-parse HEAD) + if [[ -z "${PRERELEASE_SUFFIX}" ]]; then + PRERELEASE_SUFFIX="${GITHUB_RUN_NUMBER}" + fi + echo "release_type=${RELEASE_TYPE}" >> "$GITHUB_OUTPUT" echo "branch=${TARGET_BRANCH}" >> "$GITHUB_OUTPUT" echo "commit=${RELEASE_COMMIT}" >> "$GITHUB_OUTPUT" @@ -66,7 +78,7 @@ jobs: echo "version=${CURRENT_VERSION}" >> "$GITHUB_OUTPUT" echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT" else - VERSION="${CURRENT_VERSION}-rc.${GITHUB_RUN_NUMBER}" + VERSION="${CURRENT_VERSION}-rc.${PRERELEASE_SUFFIX}" echo "version=${VERSION}" >> "$GITHUB_OUTPUT" echo "release_tag=" >> "$GITHUB_OUTPUT" @@ -79,7 +91,6 @@ jobs: permissions: contents: write id-token: write - environment: npm-publish env: PACKAGE_NAME: autoevals VERSION: ${{ needs.prepare-release.outputs.version }} @@ -93,6 +104,9 @@ jobs: fetch-depth: 0 ref: ${{ needs.prepare-release.outputs.branch }} + - name: Check version sync + run: python3 .github/scripts/check_version_sync.py + - name: Set up Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: @@ -160,7 +174,7 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, tag_name: process.env.RELEASE_TAG, - name: `autoevals v${process.env.VERSION}`, + name: `autoevals JavaScript v${process.env.VERSION}`, draft: false, prerelease: false, generate_release_notes: true, diff --git a/.github/workflows/publish-py.yaml b/.github/workflows/publish-py.yaml new file mode 100644 index 0000000..ce1f956 --- /dev/null +++ b/.github/workflows/publish-py.yaml @@ -0,0 +1,228 @@ +name: publish-py + +concurrency: + group: publish-py-${{ inputs.release_type }}-${{ inputs.branch }} + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + release_type: + description: Release type + required: true + default: stable + type: choice + options: + - stable + - prerelease + branch: + description: Branch to release from + required: true + default: main + type: string + prerelease_suffix: + description: Optional shared prerelease suffix + required: false + default: "" + type: string + +jobs: + prepare-release: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + version: ${{ steps.release_metadata.outputs.version }} + release_tag: ${{ steps.release_metadata.outputs.release_tag }} + branch: ${{ steps.release_metadata.outputs.branch }} + commit: ${{ steps.release_metadata.outputs.commit }} + release_type: ${{ steps.release_metadata.outputs.release_type }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 1 + ref: ${{ inputs.branch }} + + - name: Check version sync + run: python3 .github/scripts/check_version_sync.py + + - name: Set up Python + uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 + with: + python-version: "3.12" + + - name: Determine release metadata + id: release_metadata + env: + RELEASE_TYPE: ${{ inputs.release_type }} + TARGET_BRANCH: ${{ inputs.branch }} + PRERELEASE_SUFFIX: ${{ inputs.prerelease_suffix }} + run: | + set -euo pipefail + + CURRENT_VERSION=$(python -c 'from pathlib import Path; ns = {}; exec(Path("py/autoevals/version.py").read_text(encoding="utf-8"), ns); print(ns["VERSION"])') + RELEASE_COMMIT=$(git rev-parse HEAD) + + if [[ -z "${PRERELEASE_SUFFIX}" ]]; then + PRERELEASE_SUFFIX="${GITHUB_RUN_NUMBER}" + fi + + echo "release_type=${RELEASE_TYPE}" >> "$GITHUB_OUTPUT" + echo "branch=${TARGET_BRANCH}" >> "$GITHUB_OUTPUT" + echo "commit=${RELEASE_COMMIT}" >> "$GITHUB_OUTPUT" + + if [[ "$RELEASE_TYPE" == "stable" ]]; then + RELEASE_TAG="py-${CURRENT_VERSION}" + + if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then + echo "Tag ${RELEASE_TAG} already exists on origin" >&2 + exit 1 + fi + + echo "version=${CURRENT_VERSION}" >> "$GITHUB_OUTPUT" + echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT" + else + VERSION="${CURRENT_VERSION}rc${PRERELEASE_SUFFIX}" + + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "release_tag=" >> "$GITHUB_OUTPUT" + fi + + publish: + needs: prepare-release + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: write + id-token: write # Required for PyPI trusted publishing + env: + PACKAGE_NAME: autoevals + VERSION: ${{ needs.prepare-release.outputs.version }} + RELEASE_TAG: ${{ needs.prepare-release.outputs.release_tag }} + RELEASE_TYPE: ${{ needs.prepare-release.outputs.release_type }} + TARGET_BRANCH: ${{ needs.prepare-release.outputs.branch }} + RELEASE_COMMIT: ${{ needs.prepare-release.outputs.commit }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + ref: ${{ needs.prepare-release.outputs.branch }} + + - name: Check version sync + run: python3 .github/scripts/check_version_sync.py + + - name: Set up Python + uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4 + with: + python-version: "3.12" + + - name: Check PyPI version availability + run: | + set -euo pipefail + + python - <<'PY' + import os + import sys + import urllib.error + import urllib.request + + package = os.environ["PACKAGE_NAME"] + version = os.environ["VERSION"] + url = f"https://pypi.org/pypi/{package}/{version}/json" + + try: + urllib.request.urlopen(url) + except urllib.error.HTTPError as exc: + if exc.code == 404: + raise SystemExit(0) + raise + except urllib.error.URLError as exc: + print(f"Failed to query PyPI: {exc}", file=sys.stderr) + raise + else: + print(f"{package}=={version} already exists on PyPI", file=sys.stderr) + raise SystemExit(1) + PY + + - name: Install build dependencies + run: python -m pip install --upgrade pip build twine + + - name: Prepare prerelease package metadata + if: ${{ env.RELEASE_TYPE == 'prerelease' }} + run: | + set -euo pipefail + + python - <<'PY' + import os + import re + from pathlib import Path + + path = Path("py/autoevals/version.py") + text = path.read_text(encoding="utf-8") + new_text, count = re.subn( + r'^VERSION\s*=\s*["\'][^"\']+["\']\s*$', + f'VERSION = "{os.environ["VERSION"]}"', + text, + count=1, + flags=re.MULTILINE, + ) + if count != 1: + raise SystemExit("Could not update py/autoevals/version.py for prerelease publish") + path.write_text(new_text + ("" if new_text.endswith("\n") else "\n"), encoding="utf-8") + PY + + - name: Build package + run: python -m build + + - name: Verify package metadata + run: python -m twine check dist/* + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # release/v1 + with: + packages-dir: dist/ + + - name: Create and push stable release tag + if: ${{ env.RELEASE_TYPE == 'stable' }} + run: | + set -euo pipefail + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag "${RELEASE_TAG}" "${RELEASE_COMMIT}" + git push origin "${RELEASE_TAG}" + + - name: Create GitHub release + if: ${{ env.RELEASE_TYPE == 'stable' }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + RELEASE_TAG: ${{ env.RELEASE_TAG }} + VERSION: ${{ env.VERSION }} + with: + script: | + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: process.env.RELEASE_TAG, + name: `autoevals Python v${process.env.VERSION}`, + draft: false, + prerelease: false, + generate_release_notes: true, + }); + + - name: Summarize release + run: | + set -euo pipefail + + { + echo "## PyPI publish complete" + echo + echo "- Package: \`${PACKAGE_NAME}\`" + echo "- Version: \`${VERSION}\`" + echo "- Release type: \`${RELEASE_TYPE}\`" + if [ "${RELEASE_TYPE}" = "prerelease" ]; then + echo "- Install: \`pip install --pre ${PACKAGE_NAME}\`" + else + echo "- Git tag: \`${RELEASE_TAG}\`" + echo "- Install: \`pip install ${PACKAGE_NAME}\`" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..55393bd --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,66 @@ +name: publish + +concurrency: + group: publish-${{ inputs.release_type }}-${{ inputs.branch }} + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + release_type: + description: Release type + required: true + default: stable + type: choice + options: + - stable + - prerelease + branch: + description: Branch to publish from + required: true + default: main + type: string + +jobs: + dispatch-package-publishes: + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Dispatch JS and Python publish workflows + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + RELEASE_TYPE: ${{ inputs.release_type }} + TARGET_BRANCH: ${{ inputs.branch }} + PRERELEASE_SUFFIX: ${{ github.run_number }} + WORKFLOW_REF: main + with: + script: | + const workflows = ["publish-js.yaml", "publish-py.yaml"]; + for (const workflow_id of workflows) { + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id, + ref: process.env.WORKFLOW_REF, + inputs: { + release_type: process.env.RELEASE_TYPE, + branch: process.env.TARGET_BRANCH, + prerelease_suffix: process.env.PRERELEASE_SUFFIX, + }, + }); + } + + - name: Summarize dispatch + run: | + { + echo "## Package publishes queued" + echo + echo "- Workflows: \`publish-js.yaml\`, \`publish-py.yaml\`" + echo "- Release type: \`${{ inputs.release_type }}\`" + echo "- Branch: \`${{ inputs.branch }}\`" + if [ "${{ inputs.release_type }}" = "prerelease" ]; then + echo "- Shared prerelease suffix: \`${{ github.run_number }}\`" + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/version-sync.yaml b/.github/workflows/version-sync.yaml new file mode 100644 index 0000000..21fbf9b --- /dev/null +++ b/.github/workflows/version-sync.yaml @@ -0,0 +1,15 @@ +name: version-sync + +on: + pull_request: + push: + branches: [main] + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Check JS and Python package versions match + run: python3 .github/scripts/check_version_sync.py diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 15d4868..50dbc88 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -1,6 +1,64 @@ # Publishing -This repository contains both JavaScript and Python packages. The JavaScript package (`autoevals`) is published to npm via GitHub Actions trusted publishing with provenance attestations. +This repository contains both JavaScript and Python packages, both published as `autoevals`: + +- npm package: `autoevals` +- PyPI package: `autoevals` + +Publishing is handled via GitHub Actions trusted publishing. + +## Workflows + +Publishing workflows: + +- `.github/workflows/publish.yaml` — manual dispatcher that triggers both package publish workflows +- `.github/workflows/publish-js.yaml` — npm publish workflow +- `.github/workflows/publish-py.yaml` — PyPI publish workflow +- `.github/workflows/version-sync.yaml` — CI check that JS/Python versions stay in sync + +## Versioning policy + +JavaScript and Python package versions must always match. + +The canonical version files are: + +- `package.json` +- `py/autoevals/version.py` + +CI enforces this with: + +- `.github/workflows/version-sync.yaml` +- `.github/scripts/check_version_sync.py` + +If these versions do not match, CI fails and publish workflows fail early. + +## Recommended publish flow + +Use the top-level `publish` workflow for normal releases. + +In GitHub Actions, manually run: + +- `publish` + +Inputs: + +- `release_type=stable` or `prerelease` +- `branch=main` (or another branch to publish from) + +This workflow dispatches both: + +- `publish-js.yaml` +- `publish-py.yaml` + +For prereleases, the dispatcher passes a shared prerelease suffix to both workflows so the releases stay aligned: + +- npm: `-rc.` +- PyPI: `rc` + +Example for base version `0.2.0` and suffix `123`: + +- npm: `0.2.0-rc.123` +- PyPI: `0.2.0rc123` ## JavaScript npm publishing @@ -11,14 +69,14 @@ The JavaScript publish workflow lives at: It supports two release types: - `stable`: publishes the exact version in `package.json` -- `prerelease`: publishes `-rc.` with the `rc` dist-tag +- `prerelease`: publishes `-rc.` with the `rc` dist-tag For stable releases, the workflow also: - creates and pushes a git tag named `js-` -- creates a GitHub Release named `autoevals v` +- creates a GitHub Release named `autoevals JavaScript v` -## npm trusted publishing setup +### npm trusted publishing setup Configure trusted publishing for the `autoevals` package in npm with these values: @@ -27,35 +85,50 @@ Configure trusted publishing for the `autoevals` package in npm with these value - Repository owner: `braintrustdata` - Repository name: `autoevals` - Workflow file: `.github/workflows/publish-js.yaml` -- Environment: `npm-publish` Notes: - The workflow uses GitHub OIDC, so no `NPM_TOKEN` is required. - The workflow publishes with provenance enabled via `npm publish --provenance`. -## GitHub environment setup +## Python PyPI publishing + +The Python publish workflow lives at: + +- `.github/workflows/publish-py.yaml` -Create a GitHub Actions environment named: +It supports two release types: -- `npm-publish` +- `stable`: publishes the exact version in `py/autoevals/version.py` +- `prerelease`: publishes a PEP 440 prerelease version `rc` -Recommended configuration: +For stable releases, the workflow also: -- restrict deployments to `main` -- add required reviewers if you want manual approval before publish +- creates and pushes a git tag named `py-` +- creates a GitHub Release named `autoevals Python v` -The workflow already references this environment: +### PyPI trusted publishing setup -```yaml -environment: npm-publish -``` +Configure trusted publishing for the `autoevals` project in PyPI with these values: + +- Project name: `autoevals` +- Owner: `braintrustdata` +- Repository name: `autoevals` +- Workflow file: `.github/workflows/publish-py.yaml` + +Notes: + +- The workflow uses GitHub OIDC, so no PyPI API token is required. +- The workflow publishes via `pypa/gh-action-pypi-publish`. +- The workflow must have `id-token: write` permission for trusted publishing. ## How to publish a stable release -1. Bump the JavaScript package version in `package.json`. +1. Bump both versions together: + - `package.json` + - `py/autoevals/version.py` 2. Merge the change to `main`. -3. In GitHub Actions, run the `publish-js` workflow. +3. In GitHub Actions, run the `publish` workflow. 4. Choose: - `release_type=stable` - `branch=main` @@ -63,39 +136,67 @@ environment: npm-publish Expected outcome: - npm package `autoevals@` is published +- PyPI package `autoevals==` is published - git tag `js-` is created and pushed -- GitHub Release `autoevals v` is created +- git tag `py-` is created and pushed +- GitHub Release `autoevals JavaScript v` is created +- GitHub Release `autoevals Python v` is created ## How to publish a prerelease -1. Make sure `package.json` contains the base version you want to prerelease from. -2. In GitHub Actions, run the `publish-js` workflow. +1. Make sure both version files contain the same base version: + - `package.json` + - `py/autoevals/version.py` +2. In GitHub Actions, run the `publish` workflow. 3. Choose: - `release_type=prerelease` - `branch=main` Expected outcome: -- npm package `autoevals@-rc.` is published +- npm package `autoevals@-rc.` is published - npm dist-tag `rc` is updated -- no git tag is created -- no GitHub Release is created +- PyPI package `autoevals==rc` is published +- no stable git tags are created +- no GitHub Releases are created + +## Publishing package-specific workflows directly + +If needed, you can manually trigger either workflow directly: -## Safeguards in the workflow +- `publish-js` +- `publish-py` -The workflow will fail early if: +Both accept: -- the stable tag `js-` already exists on `origin` +- `release_type` +- `branch` +- `prerelease_suffix` (optional) + +Normally you should prefer the top-level `publish` workflow so JS and Python prereleases use the same suffix. + +## Safeguards in the workflows + +The workflows fail early if: + +- `package.json` and `py/autoevals/version.py` do not match +- the stable JS tag `js-` already exists on `origin` +- the stable Python tag `py-` already exists on `origin` - the npm version being published already exists +- the PyPI version being published already exists ## Local validation Useful commands before triggering a release: ```bash +python3 .github/scripts/check_version_sync.py pnpm install --frozen-lockfile pnpm run build npm publish --dry-run --access public +python3 -m pip install --upgrade build twine +python3 -m build +python3 -m twine check dist/* ``` ## Historical releases and source mapping @@ -105,8 +206,6 @@ Older npm releases may not be traceable back to an exact git commit from npm alo - npm metadata for older releases may not include `gitHead` - those releases do not have OIDC/provenance attestations tying the package to a workflow run and commit -For those historical versions, the best commit mapping may need to be inferred from repository history, publish timestamps, and version bumps. New releases published through `.github/workflows/publish-js.yaml` are expected to be easier to trace because they use trusted publishing with provenance. - -## Future publishing work +For those historical versions, the best commit mapping may need to be inferred from repository history, publish timestamps, and version bumps. New npm releases published through `.github/workflows/publish-js.yaml` are easier to trace because they use trusted publishing with provenance. -Python publishing is not yet covered by this document. When a Python release workflow is added, keep Python tags and release process separate from the JavaScript `js-` tag namespace. +Python releases published through `.github/workflows/publish-py.yaml` are similarly expected to be easier to trace because they use PyPI trusted publishing via GitHub Actions OIDC. diff --git a/package.json b/package.json index 7f82577..631f780 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "autoevals", - "version": "0.0.132", + "version": "0.2.0", "description": "Universal library for evaluating AI models", "repository": { "type": "git", diff --git a/py/autoevals/version.py b/py/autoevals/version.py index 1cf6267..6c5007c 100644 --- a/py/autoevals/version.py +++ b/py/autoevals/version.py @@ -1 +1 @@ -VERSION = "0.1.0" +VERSION = "0.2.0"