chore: Publish JS package via gha trusted publishing

.github/workflows/lint.yaml

@@ -12,4 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
       - uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566 # v3.1.4
-      - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0
+      - name: Install pre-commit
+        run: python -m pip install pre-commit
+      - name: Run pre-commit
+        run: pre-commit run --all-files

.github/workflows/publish-js.yaml

@@ -0,0 +1,186 @@
+name: publish-js
+
+concurrency:
+  group: publish-js-${{ inputs.release_type }}-${{ inputs.branch }}
+  cancel-in-progress: false
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_type:
+        description: Release type
+        required: true
+        default: stable
+        type: choice
+        options:
+          - stable
+          - prerelease
+      branch:
+        description: Branch to release from
+        required: true
+        default: main
+        type: string
+
+jobs:
+  prepare-release:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      version: ${{ steps.release_metadata.outputs.version }}
+      release_tag: ${{ steps.release_metadata.outputs.release_tag }}
+      branch: ${{ steps.release_metadata.outputs.branch }}
+      commit: ${{ steps.release_metadata.outputs.commit }}
+      release_type: ${{ steps.release_metadata.outputs.release_type }}
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 1
+          ref: ${{ inputs.branch }}
+      - name: Set up Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: 22
+      - name: Determine release metadata
+        id: release_metadata
+        env:
+          RELEASE_TYPE: ${{ inputs.release_type }}
+          TARGET_BRANCH: ${{ inputs.branch }}
+        run: |
+          set -euo pipefail
+
+          CURRENT_VERSION=$(node -p "require('./package.json').version")
+          RELEASE_COMMIT=$(git rev-parse HEAD)
+
+          echo "release_type=${RELEASE_TYPE}" >> "$GITHUB_OUTPUT"
+          echo "branch=${TARGET_BRANCH}" >> "$GITHUB_OUTPUT"
+          echo "commit=${RELEASE_COMMIT}" >> "$GITHUB_OUTPUT"
+
+          if [[ "$RELEASE_TYPE" == "stable" ]]; then
+            RELEASE_TAG="js-${CURRENT_VERSION}"
+
+            if git ls-remote --exit-code --tags origin "refs/tags/${RELEASE_TAG}" >/dev/null 2>&1; then
+              echo "Tag ${RELEASE_TAG} already exists on origin" >&2
+              exit 1
+            fi
+
+            echo "version=${CURRENT_VERSION}" >> "$GITHUB_OUTPUT"
+            echo "release_tag=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"
+          else
+            VERSION="${CURRENT_VERSION}-rc.${GITHUB_RUN_NUMBER}"
+
+            echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+            echo "release_tag=" >> "$GITHUB_OUTPUT"
+          fi
+
+  publish:
+    needs: prepare-release
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: write
+      id-token: write
+    environment: npm-publish
+    env:
+      PACKAGE_NAME: autoevals
+      VERSION: ${{ needs.prepare-release.outputs.version }}
+      RELEASE_TAG: ${{ needs.prepare-release.outputs.release_tag }}
+      RELEASE_TYPE: ${{ needs.prepare-release.outputs.release_type }}
+      TARGET_BRANCH: ${{ needs.prepare-release.outputs.branch }}
+      RELEASE_COMMIT: ${{ needs.prepare-release.outputs.commit }}
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0
+          ref: ${{ needs.prepare-release.outputs.branch }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: 22
+          registry-url: https://registry.npmjs.org
+
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
+        with:
+          version: 10.26.2
+
+      - name: Check npm version availability
+        run: |
+          set -euo pipefail
+
+          if npm view "${PACKAGE_NAME}@${VERSION}" version --registry=https://registry.npmjs.org >/dev/null 2>&1; then
+            echo "${PACKAGE_NAME}@${VERSION} already exists on npm" >&2
+            exit 1
+          fi
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Prepare prerelease package metadata
+        if: ${{ env.RELEASE_TYPE == 'prerelease' }}
+        run: |
+          set -euo pipefail
+
+          node -e '
+            const fs = require("fs");
+            const pkg = JSON.parse(fs.readFileSync("package.json", "utf8"));
+            pkg.version = process.env.VERSION;
+            fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n");
+          '
+
+      - name: Build package
+        run: pnpm run build
+
+      - name: Publish stable release to npm
+        if: ${{ env.RELEASE_TYPE == 'stable' }}
+        run: npm publish --provenance --access public
+
+      - name: Publish prerelease to npm
+        if: ${{ env.RELEASE_TYPE == 'prerelease' }}
+        run: npm publish --tag rc --provenance --access public
+
+      - name: Create and push stable release tag
+        if: ${{ env.RELEASE_TYPE == 'stable' }}
+        run: |
+          set -euo pipefail
+
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git tag "${RELEASE_TAG}" "${RELEASE_COMMIT}"
+          git push origin "${RELEASE_TAG}"
+
+      - name: Create GitHub release
+        if: ${{ env.RELEASE_TYPE == 'stable' }}
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        env:
+          RELEASE_TAG: ${{ env.RELEASE_TAG }}
+          VERSION: ${{ env.VERSION }}
+        with:
+          script: |
+            await github.rest.repos.createRelease({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              tag_name: process.env.RELEASE_TAG,
+              name: `autoevals v${process.env.VERSION}`,
+              draft: false,
+              prerelease: false,
+              generate_release_notes: true,
+            });
+
+      - name: Summarize release
+        run: |
+          set -euo pipefail
+
+          {
+            echo "## npm publish complete"
+            echo
+            echo "- Package: \`${PACKAGE_NAME}\`"
+            echo "- Version: \`${VERSION}\`"
+            echo "- Release type: \`${RELEASE_TYPE}\`"
+            if [ "${RELEASE_TYPE}" = "prerelease" ]; then
+              echo "- npm tag: \`rc\`"
+              echo "- Install: \`npm install ${PACKAGE_NAME}@rc\`"
+            else
+              echo "- Git tag: \`${RELEASE_TAG}\`"
+              echo "- Install: \`npm install ${PACKAGE_NAME}\`"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"

docs/PUBLISHING.md

@@ -0,0 +1,112 @@
+# Publishing
+
+This repository contains both JavaScript and Python packages. The JavaScript package (`autoevals`) is published to npm via GitHub Actions trusted publishing with provenance attestations.
+
+## JavaScript npm publishing
+
+The JavaScript publish workflow lives at:
+
+- `.github/workflows/publish-js.yaml`
+
+It supports two release types:
+
+- `stable`: publishes the exact version in `package.json`
+- `prerelease`: publishes `<package.json version>-rc.<github run number>` with the `rc` dist-tag
+
+For stable releases, the workflow also:
+
+- creates and pushes a git tag named `js-<version>`
+- creates a GitHub Release named `autoevals v<version>`
+
+## npm trusted publishing setup
+
+Configure trusted publishing for the `autoevals` package in npm with these values:
+
+- Package: `autoevals`
+- Provider: `GitHub Actions`
+- Repository owner: `braintrustdata`
+- Repository name: `autoevals`
+- Workflow file: `.github/workflows/publish-js.yaml`
+- Environment: `npm-publish`
+
+Notes:
+
+- The workflow uses GitHub OIDC, so no `NPM_TOKEN` is required.
+- The workflow publishes with provenance enabled via `npm publish --provenance`.
+
+## GitHub environment setup
+
+Create a GitHub Actions environment named:
+
+- `npm-publish`
+
+Recommended configuration:
+
+- restrict deployments to `main`
+- add required reviewers if you want manual approval before publish
+
+The workflow already references this environment:
+
+```yaml
+environment: npm-publish
+```
+
+## How to publish a stable release
+
+1. Bump the JavaScript package version in `package.json`.
+2. Merge the change to `main`.
+3. In GitHub Actions, run the `publish-js` workflow.
+4. Choose:
+   - `release_type=stable`
+   - `branch=main`
+
+Expected outcome:
+
+- npm package `autoevals@<version>` is published
+- git tag `js-<version>` is created and pushed
+- GitHub Release `autoevals v<version>` is created
+
+## How to publish a prerelease
+
+1. Make sure `package.json` contains the base version you want to prerelease from.
+2. In GitHub Actions, run the `publish-js` workflow.
+3. Choose:
+   - `release_type=prerelease`
+   - `branch=main`
+
+Expected outcome:
+
+- npm package `autoevals@<version>-rc.<run_number>` is published
+- npm dist-tag `rc` is updated
+- no git tag is created
+- no GitHub Release is created
+
+## Safeguards in the workflow
+
+The workflow will fail early if:
+
+- the stable tag `js-<version>` already exists on `origin`
+- the npm version being published already exists
+
+## Local validation
+
+Useful commands before triggering a release:
+
+```bash
+pnpm install --frozen-lockfile
+pnpm run build
+npm publish --dry-run --access public
+```
+
+## Historical releases and source mapping
+
+Older npm releases may not be traceable back to an exact git commit from npm alone because they were published before trusted publishing and provenance attestations were enabled. In particular:
+
+- npm metadata for older releases may not include `gitHead`
+- those releases do not have OIDC/provenance attestations tying the package to a workflow run and commit
+
+For those historical versions, the best commit mapping may need to be inferred from repository history, publish timestamps, and version bumps. New releases published through `.github/workflows/publish-js.yaml` are expected to be easier to trace because they use trusted publishing with provenance.
+
+## Future publishing work
+
+Python publishing is not yet covered by this document. When a Python release workflow is added, keep Python tags and release process separate from the JavaScript `js-<version>` tag namespace.

package.json

@@ -26,9 +26,7 @@
     "build": "tsup",
     "watch": "tsup --watch",
     "docs": "npx typedoc --options typedoc.json js/index.ts",
-    "test": "vitest",
-    "prepublishOnly": "../scripts/node_prepublish_autoevals.py",
-    "postpublish": "../scripts/node_postpublish_autoevals.py"
+    "test": "vitest"
   },
   "author": "",
   "license": "MIT",