From 5acfdcd0109880b68f152731223c408ea1f415ff Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:48:41 +0200 Subject: [PATCH 01/10] feat: add sign-oci action + image smoke test Adds a keyless (cosign/Fulcio/Rekor) sign-oci composite action that signs an OCI image by digest and attests build provenance + SPDX SBOM, plus a sign-smoke workflow that builds a throwaway ghcr image, signs/attests it, and verifies the signature, provenance attestation, and SBOM package count end-to-end. --- .github/actions/sign-oci/action.yml | 53 +++++++++++++++ .github/workflows/lint-test.yaml | 9 +-- .github/workflows/sign-smoke.yaml | 84 ++++++++++++++++++++++++ .github/workflows/smoke-image/Dockerfile | 2 + 4 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 .github/actions/sign-oci/action.yml create mode 100644 .github/workflows/sign-smoke.yaml create mode 100644 .github/workflows/smoke-image/Dockerfile diff --git a/.github/actions/sign-oci/action.yml b/.github/actions/sign-oci/action.yml new file mode 100644 index 0000000..e56435b --- /dev/null +++ b/.github/actions/sign-oci/action.yml @@ -0,0 +1,53 @@ +name: sign-oci +description: Keyless-sign and attest (build provenance + SBOM) an OCI artifact by digest. + +inputs: + image: + description: "Registry reference without digest, e.g. ghcr.io/owner/repo/name." + required: true + digest: + description: "The sha256:... digest of the pushed artifact." + required: true + sbom: + description: "Whether to generate and attest an SPDX SBOM." + required: false + default: "true" + +runs: + using: composite + steps: + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Install syft + if: ${{ inputs.sbom == 'true' }} + uses: anchore/sbom-action/download-syft@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + + - name: Sign image (keyless) + shell: bash + env: + REF: ${{ inputs.image }}@${{ inputs.digest }} + run: cosign sign --yes "$REF" + + - name: Attest build provenance + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-name: ${{ inputs.image }} + subject-digest: ${{ inputs.digest }} + push-to-registry: true + + - name: Generate SBOM + if: ${{ inputs.sbom == 'true' }} + shell: bash + env: + REF: ${{ inputs.image }}@${{ inputs.digest }} + run: syft "$REF" -o spdx-json=sbom.spdx.json + + - name: Attest SBOM + if: ${{ inputs.sbom == 'true' }} + uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + with: + subject-name: ${{ inputs.image }} + subject-digest: ${{ inputs.digest }} + sbom-path: sbom.spdx.json + push-to-registry: true diff --git a/.github/workflows/lint-test.yaml b/.github/workflows/lint-test.yaml index 1ca4658..7040ff4 100644 --- a/.github/workflows/lint-test.yaml +++ b/.github/workflows/lint-test.yaml @@ -3,9 +3,9 @@ name: lint-test # CI for this repo's own reusable workflows and helper scripts. # # Scoped to the workflows introduced/maintained under this effort -# (pack-build-image.yaml, pack-release.yaml, and this file) rather than -# the whole .github/workflows/ directory: the other pre-existing -# workflows here (sync-issue-templates.yaml, sync-project-priority.yaml) +# (pack-build-image.yaml, pack-release.yaml, sign-smoke.yaml, and this +# file) rather than the whole .github/workflows/ directory: the other +# pre-existing workflows here (sync-issue-templates.yaml, sync-project-priority.yaml) # already have unrelated actionlint findings (a floating, EOL # actions/checkout@v3 pin and a few shellcheck info-level notes) that # predate this change and are out of scope for it. Widen the file list @@ -33,7 +33,8 @@ jobs: -color \ .github/workflows/pack-build-image.yaml \ .github/workflows/pack-release.yaml \ - .github/workflows/lint-test.yaml + .github/workflows/lint-test.yaml \ + .github/workflows/sign-smoke.yaml pytest: name: pytest diff --git a/.github/workflows/sign-smoke.yaml b/.github/workflows/sign-smoke.yaml new file mode 100644 index 0000000..8c467fa --- /dev/null +++ b/.github/workflows/sign-smoke.yaml @@ -0,0 +1,84 @@ +name: sign-smoke + +# Exercises the sign-oci / sign-blob actions end-to-end against throwaway +# artifacts on every push to a signing-work branch, and on demand. Proves +# journeys 1-5. Signs against ghcr only (this repo has no quay creds). + +on: + push: + branches: + - "feat/reusable-signing-actions" + workflow_dispatch: + +env: + # github.repository for this repo is "nebari-dev/.github". A path segment + # cannot start with a dot per the OCI/Docker reference-name grammar + # (path-component must start with [a-z0-9]), so ghcr.io//sign-smoke + # would be an invalid reference. Use repository_owner + a sanitized name. + SMOKE_IMAGE: ghcr.io/${{ github.repository_owner }}/dot-github-sign-smoke + +jobs: + image: + name: sign-oci smoke (image) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + attestations: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Log in to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push throwaway image + id: build + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: .github/workflows/smoke-image + push: true + tags: ${{ env.SMOKE_IMAGE }}:${{ github.sha }} + + - name: Sign + attest + uses: ./.github/actions/sign-oci + with: + image: ${{ env.SMOKE_IMAGE }} + digest: ${{ steps.build.outputs.digest }} + + - name: Verify signature + env: + REF: ${{ env.SMOKE_IMAGE }}@${{ steps.build.outputs.digest }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + cosign verify "$REF" \ + --certificate-identity-regexp "https://github.com/$REPO/.github/workflows/sign-smoke.yaml@.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" + + - name: Verify provenance attestation + env: + REF: ${{ env.SMOKE_IMAGE }}@${{ steps.build.outputs.digest }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh attestation verify "oci://$REF" --repo "$REPO" + + - name: Verify SBOM attestation has packages + env: + REF: ${{ env.SMOKE_IMAGE }}@${{ steps.build.outputs.digest }} + REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + gh attestation verify "oci://$REF" --repo "$REPO" \ + --predicate-type https://spdx.dev/Document/v2.3 \ + --format json \ + | jq -e '.[0].verificationResult.statement.predicate.packages | length > 0' diff --git a/.github/workflows/smoke-image/Dockerfile b/.github/workflows/smoke-image/Dockerfile new file mode 100644 index 0000000..ec856b3 --- /dev/null +++ b/.github/workflows/smoke-image/Dockerfile @@ -0,0 +1,2 @@ +FROM alpine:3.20 +RUN echo "nebari sign-smoke" > /smoke.txt From 95a086807561dccd14c4832d7b3e26e5e48ae828 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:03:53 +0200 Subject: [PATCH 02/10] feat: add sign-blob action + chart smoke test --- .github/actions/sign-blob/action.yml | 56 ++++++++++++++++++++++++++++ .github/workflows/sign-smoke.yaml | 43 +++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 .github/actions/sign-blob/action.yml diff --git a/.github/actions/sign-blob/action.yml b/.github/actions/sign-blob/action.yml new file mode 100644 index 0000000..ee02eba --- /dev/null +++ b/.github/actions/sign-blob/action.yml @@ -0,0 +1,56 @@ +name: sign-blob +description: Keyless-sign and attest (build provenance + SBOM) a local file. + +inputs: + path: + description: "Path to the file to sign." + required: true + sbom: + description: "Whether to generate and attest an SPDX SBOM." + required: false + default: "true" + +outputs: + bundle: + description: "Path to the .sigstore.json bundle produced by cosign sign-blob." + value: ${{ steps.sign.outputs.bundle }} + +runs: + using: composite + steps: + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Install syft + if: ${{ inputs.sbom == 'true' }} + uses: anchore/sbom-action/download-syft@e22c389904149dbc22b58101806040fa8d37a610 # v0.24.0 + + - name: Sign blob (keyless) + id: sign + shell: bash + env: + FILE: ${{ inputs.path }} + run: | + set -euo pipefail + bundle="${FILE}.sigstore.json" + cosign sign-blob --yes --bundle "$bundle" "$FILE" + echo "bundle=$bundle" >> "$GITHUB_OUTPUT" + + - name: Attest build provenance + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-path: ${{ inputs.path }} + + - name: Generate SBOM + if: ${{ inputs.sbom == 'true' }} + shell: bash + env: + FILE: ${{ inputs.path }} + run: syft "$FILE" -o spdx-json="${FILE}.spdx.json" + + - name: Attest SBOM + if: ${{ inputs.sbom == 'true' }} + uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0 + with: + subject-path: ${{ inputs.path }} + sbom-path: ${{ inputs.path }}.spdx.json diff --git a/.github/workflows/sign-smoke.yaml b/.github/workflows/sign-smoke.yaml index 8c467fa..e8dbf9e 100644 --- a/.github/workflows/sign-smoke.yaml +++ b/.github/workflows/sign-smoke.yaml @@ -82,3 +82,46 @@ jobs: --predicate-type https://spdx.dev/Document/v2.3 \ --format json \ | jq -e '.[0].verificationResult.statement.predicate.packages | length > 0' + + chart: + name: sign-blob smoke (chart) + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + attestations: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Set up Helm + uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + + - name: Package a dummy chart + id: pkg + run: | + set -euo pipefail + helm create smokechart + helm package smokechart --destination . + echo "tgz=$(ls smokechart-*.tgz)" >> "$GITHUB_OUTPUT" + + - name: Sign + attest the .tgz + id: sign + uses: ./.github/actions/sign-blob + with: + path: ${{ steps.pkg.outputs.tgz }} + + - name: Verify blob signature + env: + TGZ: ${{ steps.pkg.outputs.tgz }} + BUNDLE: ${{ steps.sign.outputs.bundle }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + cosign verify-blob --bundle "$BUNDLE" \ + --certificate-identity-regexp "https://github.com/$REPO/.github/workflows/sign-smoke.yaml@.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + "$TGZ" From 7f45aa73898e4d77b876c4178ac03b0d020ec984 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:13:35 +0200 Subject: [PATCH 03/10] test: add tamper + wrong-identity negative smoke cases --- .github/workflows/sign-smoke.yaml | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/.github/workflows/sign-smoke.yaml b/.github/workflows/sign-smoke.yaml index e8dbf9e..0b5c00e 100644 --- a/.github/workflows/sign-smoke.yaml +++ b/.github/workflows/sign-smoke.yaml @@ -125,3 +125,54 @@ jobs: --certificate-identity-regexp "https://github.com/$REPO/.github/workflows/sign-smoke.yaml@.*" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ "$TGZ" + + negative: + name: negative cases (must fail closed) + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + attestations: write + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + + - name: Prepare payload + run: echo "original" > payload.txt + + - name: Sign a blob + id: sign + uses: ./.github/actions/sign-blob + with: + path: payload.txt + + - name: Tampered artifact must fail + env: + BUNDLE: ${{ steps.sign.outputs.bundle }} + run: | + set -euo pipefail + echo "tampered" > payload.txt + if cosign verify-blob --bundle "$BUNDLE" \ + --certificate-identity-regexp ".*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + payload.txt; then + echo "::error::verify-blob accepted a tampered file"; exit 1 + fi + echo "OK: tamper rejected" + + - name: Wrong identity must fail + env: + BUNDLE: ${{ steps.sign.outputs.bundle }} + run: | + set -euo pipefail + echo "original" > payload.txt + if cosign verify-blob --bundle "$BUNDLE" \ + --certificate-identity-regexp "https://github.com/some-other-org/.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + payload.txt; then + echo "::error::verify-blob accepted a wrong identity"; exit 1 + fi + echo "OK: wrong identity rejected" From d76852fbf6ec1d4f81f1425ee9c12f81d276ea8c Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:20:04 +0200 Subject: [PATCH 04/10] feat: sign + attest pack images in pack-build-image --- .github/workflows/pack-build-image.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/pack-build-image.yaml b/.github/workflows/pack-build-image.yaml index 5784d01..6d31a05 100644 --- a/.github/workflows/pack-build-image.yaml +++ b/.github/workflows/pack-build-image.yaml @@ -48,6 +48,8 @@ jobs: permissions: contents: read packages: write + id-token: write + attestations: write steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -87,6 +89,7 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} - name: Build and push + id: build uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 with: context: ${{ inputs.context }} @@ -98,3 +101,17 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=${{ env.GHCR_IMAGE }}:cache cache-to: ${{ inputs.push && format('type=registry,ref={0}:cache,mode=max', env.GHCR_IMAGE) || '' }} + + - name: Sign + attest (GHCR) + if: ${{ inputs.push }} + uses: ./.github/actions/sign-oci + with: + image: ${{ env.GHCR_IMAGE }} + digest: ${{ steps.build.outputs.digest }} + + - name: Sign + attest (Quay) + if: ${{ inputs.push }} + uses: ./.github/actions/sign-oci + with: + image: ${{ env.QUAY_IMAGE }} + digest: ${{ steps.build.outputs.digest }} From f4382d1a68744947671101c806f1878e8d7a122f Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:29:50 +0200 Subject: [PATCH 05/10] feat: sign + attest packaged chart in pack-release Sign the packaged Helm chart .tgz with keyless cosign and attach the bundle + SBOM to the GitHub Release alongside the chart. --- .github/workflows/pack-release.yaml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pack-release.yaml b/.github/workflows/pack-release.yaml index 7f90195..93588a5 100644 --- a/.github/workflows/pack-release.yaml +++ b/.github/workflows/pack-release.yaml @@ -80,6 +80,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + id-token: write + attestations: write steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -195,6 +197,25 @@ jobs: *) echo "flag=" >> "$GITHUB_OUTPUT" ;; esac + - name: Locate packaged chart + id: tgz + if: steps.exists.outputs.exists == 'false' + run: | + set -euo pipefail + shopt -s nullglob + files=(*.tgz) + if [ "${#files[@]}" -ne 1 ]; then + echo "::error::expected exactly one .tgz, found ${#files[@]}"; exit 1 + fi + echo "path=${files[0]}" >> "$GITHUB_OUTPUT" + + - name: Sign + attest chart + id: sign + if: steps.exists.outputs.exists == 'false' + uses: ./.github/actions/sign-blob + with: + path: ${{ steps.tgz.outputs.path }} + - name: Create GitHub Release if: steps.exists.outputs.exists == 'false' env: @@ -202,6 +223,7 @@ jobs: GH_REPO: ${{ github.repository }} TAG: ${{ steps.chart.outputs.tag }} PRERELEASE_FLAG: ${{ steps.pre.outputs.flag }} + BUNDLE: ${{ steps.sign.outputs.bundle }} run: | set -euo pipefail shopt -s nullglob @@ -216,7 +238,10 @@ jobs: release_args+=("$PRERELEASE_FLAG") fi - gh release create "$TAG" "${release_args[@]}" "${tgz_files[0]}" + gh release create "$TAG" "${release_args[@]}" \ + "${tgz_files[0]}" \ + "$BUNDLE" \ + "${tgz_files[0]}.spdx.json" - name: Sync chart to nebari-dev/helm-repository if: steps.exists.outputs.exists == 'false' From bc00e6ac9662610ec4aaf85832d81d5a6cb58353 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:40:43 +0200 Subject: [PATCH 06/10] docs: add verifying-nebari-artifacts guide --- verifying-nebari-artifacts.md | 187 ++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 verifying-nebari-artifacts.md diff --git a/verifying-nebari-artifacts.md b/verifying-nebari-artifacts.md new file mode 100644 index 0000000..68395d3 --- /dev/null +++ b/verifying-nebari-artifacts.md @@ -0,0 +1,187 @@ +# Verifying Nebari pack artifacts + +Nebari software packs (for example `llm-serving-pack`) build and release their +artifacts through this repository's reusable workflows: + +- [`.github/workflows/pack-build-image.yaml`](.github/workflows/pack-build-image.yaml) + builds and pushes container images to both GHCR and Quay, then signs and + attests each one with the + [`sign-oci`](.github/actions/sign-oci/action.yml) action. +- [`.github/workflows/pack-release.yaml`](.github/workflows/pack-release.yaml) + packages the pack's Helm chart into a `.tgz`, signs it with the + [`sign-blob`](.github/actions/sign-blob/action.yml) action, and attaches the + `.tgz`, its `.sigstore.json` bundle, and its `.spdx.json` SBOM to the + GitHub Release. + +All signing is keyless (Sigstore/Fulcio, backed by the GitHub Actions OIDC +token), so there is no public key to distribute. Verification instead checks +that the Fulcio certificate was issued to the expected workflow identity by +the expected OIDC issuer. This doc gives copy-pasteable commands for that. + +> This covers artifacts produced by a pack's own repo: its container images +> and its release `.tgz`. It does **not** cover the canonical OCI Helm chart +> published to `quay.io/nebari/charts` - see +> [Phase 2: the canonical OCI Helm chart](#phase-2-the-canonical-oci-helm-chart) +> below. + +## Prerequisites + +- [`cosign`](https://docs.sigstore.dev/system_config/installation/) v2 or later. +- [`gh`](https://cli.github.com/) (GitHub CLI) with the `attestation` command, + authenticated via `gh auth login`. +- If the registry requires auth to pull, log in first (`docker login ghcr.io` + / `docker login quay.io`). + +## Why the identity points at this repo, not the pack's + +Both reusable workflows run their signing steps directly (via the `sign-oci` +and `sign-blob` composite actions), so the OIDC token minted for that job is +the one GitHub issues for **the reusable workflow that owns the job's +steps** - not the pack repo that called it. GitHub's OIDC token includes a +`job_workflow_ref` claim specifically for this case: + +> "For jobs using a reusable workflow, the ref path to the reusable workflow." +> - [GitHub Actions OIDC reference](https://docs.github.com/en/actions/reference/security/oidc) + +> "If a job is part of a reusable workflow, the token will include the +> standard claims that contain information about the calling workflow, and +> will also include a custom claim called `job_workflow_ref` that contains +> information about the called workflow." +> - [Using OpenID Connect with reusable workflows](https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-with-reusable-workflows) + +Fulcio, in turn, maps that claim onto the certificate's Subject Alternative +Name via the "Build Signer URI" extension: + +> "Reference to specific build instructions that are responsible for +> signing. SHOULD be fully qualified. MAY be the same as Build Config URI." +> Example: `https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.4.0` +> - [Fulcio OID information](https://github.com/sigstore/fulcio/blob/main/docs/oid-info.md) + +and a maintainer discussion on `sigstore/cosign` confirms this in practice for +reusable workflows: the certificate's SAN reflects the reusable workflow's own +ref (its "Build Signer URI"), while the caller's workflow ref is carried +separately (its "Build Config URI") - +[sigstore/cosign discussion #2936](https://github.com/sigstore/cosign/discussions/2936). + +In practice this means: no matter which pack repo produced an image or chart, +the certificate identity you verify against is always a path inside +`nebari-dev/.github` - `pack-build-image.yaml` for images, +`pack-release.yaml` for the chart `.tgz` - at whatever ref that pack pinned in +its `uses:` line (for example `@v1`, which resolves to `refs/tags/v1`). + +> **Confirm before relying on this in production.** No pack has published a +> real signed artifact yet (that's journey 8, exercised post-merge by the +> first real `llm-serving-pack` release). The claim/OID mechanics above are +> documented behavior, but the exact literal identity string - in particular +> whether the ref resolves to `refs/tags/v1` versus a commit SHA - should be +> confirmed against that first real release before being treated as gospel. +> See [Confirming the exact identity](#confirming-the-exact-identity) for how +> to check. + +## 1. Container images (GHCR or Quay) + +Both registries are signed the same way, by the same workflow +(`pack-build-image.yaml`), so the identity regexp is identical regardless of +which registry you pulled from. Always verify by digest, not by tag. + +```bash +IMAGE="ghcr.io/nebari-dev/llm-serving-pack/operator" # or: quay.io/nebari/llm-serving-pack-operator +DIGEST="sha256:" + +cosign verify "${IMAGE}@${DIGEST}" \ + --certificate-identity-regexp "https://github\.com/nebari-dev/\.github/\.github/workflows/pack-build-image\.yaml@.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" +``` + +## 2. Build provenance and SBOM attestations + +In addition to the `cosign` signature above, both `sign-oci` and `sign-blob` +generate GitHub artifact attestations (SLSA build provenance and an SPDX +SBOM), verifiable with `gh attestation verify` against the **pack repo** +(that's where the workflow run - and therefore the attestation - is +recorded), not `nebari-dev/.github`. + +Build provenance: + +```bash +gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: \ + --repo nebari-dev/llm-serving-pack +``` + +SBOM (pass the SPDX predicate type explicitly; `gh attestation verify` +otherwise defaults to the SLSA provenance predicate): + +```bash +gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: \ + --repo nebari-dev/llm-serving-pack \ + --predicate-type https://spdx.dev/Document/v2.3 +``` + +## 3. Helm chart `.tgz` from a GitHub Release + +Every pack release publishes three assets together: the packaged chart, its +`cosign` bundle, and its SBOM. Download all three, then verify the bundle +against the chart before doing anything else with it. + +```bash +gh release download llm-serving-pack-1.2.3 \ + --repo nebari-dev/llm-serving-pack \ + --pattern 'llm-serving-pack-1.2.3.tgz*' +# downloads: +# llm-serving-pack-1.2.3.tgz +# llm-serving-pack-1.2.3.tgz.sigstore.json +# llm-serving-pack-1.2.3.tgz.spdx.json + +cosign verify-blob \ + --bundle llm-serving-pack-1.2.3.tgz.sigstore.json \ + --certificate-identity-regexp "https://github\.com/nebari-dev/\.github/\.github/workflows/pack-release\.yaml@.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + llm-serving-pack-1.2.3.tgz +``` + +The chart also carries the same kind of GitHub attestations as the images +(see [section 2](#2-build-provenance-and-sbom-attestations)), verifiable +against the local file instead of an `oci://` reference: + +```bash +gh attestation verify llm-serving-pack-1.2.3.tgz --repo nebari-dev/llm-serving-pack +gh attestation verify llm-serving-pack-1.2.3.tgz --repo nebari-dev/llm-serving-pack \ + --predicate-type https://spdx.dev/Document/v2.3 +``` + +## Confirming the exact identity + +Until the first real pack release, treat the regexp above as expected-but-unconfirmed +for the exact ref suffix. Once a real image or chart exists, inspect what was +actually issued rather than trusting this doc blindly: + +```bash +# Full certificate details behind a cosign verification, as JSON: +cosign verify "${IMAGE}@${DIGEST}" \ + --certificate-identity-regexp "https://github\.com/nebari-dev/\.github/\.github/workflows/pack-build-image\.yaml@.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ + --output json | jq '.[0].optional.Issuer, .[0].optional.Subject' + +# Raw attestation claims, as JSON: +gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: \ + --repo nebari-dev/llm-serving-pack --format json | jq . +``` + +If the observed identity ever differs from what's documented here, this file +is out of date - update it rather than working around it. + +## Phase 2: the canonical OCI Helm chart + +`pack-release.yaml` only packages the chart as a `.tgz` GitHub Release asset +and opens a PR to sync the chart source into +[`nebari-dev/helm-repository`](https://github.com/nebari-dev/helm-repository). +The canonical OCI chart pushed to `quay.io/nebari/charts` is produced later, +by `helm-repository`'s own `release-helm-charts.yml` workflow, once that PR +merges. + +That means the OCI chart has a **different** signer identity from the one +documented above: it will be a path inside `nebari-dev/helm-repository`, not +`nebari-dev/.github`. Wiring `sign-oci` into that workflow is tracked as a +follow-up (phase 2) and is not yet done as of this writing - there is nothing +to verify at `quay.io/nebari/charts` yet. This doc will be updated with that +identity and command once phase 2 ships. From ba4f44d734981bd1f879b1fa447430ac18429389 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:49:42 +0200 Subject: [PATCH 07/10] style: prettier-format verifying-nebari-artifacts.md --- verifying-nebari-artifacts.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/verifying-nebari-artifacts.md b/verifying-nebari-artifacts.md index 68395d3..c486bd4 100644 --- a/verifying-nebari-artifacts.md +++ b/verifying-nebari-artifacts.md @@ -41,12 +41,14 @@ steps** - not the pack repo that called it. GitHub's OIDC token includes a `job_workflow_ref` claim specifically for this case: > "For jobs using a reusable workflow, the ref path to the reusable workflow." +> > - [GitHub Actions OIDC reference](https://docs.github.com/en/actions/reference/security/oidc) > "If a job is part of a reusable workflow, the token will include the > standard claims that contain information about the calling workflow, and > will also include a custom claim called `job_workflow_ref` that contains > information about the called workflow." +> > - [Using OpenID Connect with reusable workflows](https://docs.github.com/en/actions/how-tos/secure-your-work/security-harden-deployments/oidc-with-reusable-workflows) Fulcio, in turn, maps that claim onto the certificate's Subject Alternative @@ -55,6 +57,7 @@ Name via the "Build Signer URI" extension: > "Reference to specific build instructions that are responsible for > signing. SHOULD be fully qualified. MAY be the same as Build Config URI." > Example: `https://github.com/slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v1.4.0` +> > - [Fulcio OID information](https://github.com/sigstore/fulcio/blob/main/docs/oid-info.md) and a maintainer discussion on `sigstore/cosign` confirms this in practice for From f116e6f4a10c4858b11adbdc2b679a374725b042 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:10:18 +0200 Subject: [PATCH 08/10] fix: load composite sign actions from _dot-github in reusable workflows; document reusable-workflow signer verification pack-build-image.yaml's sign-oci steps used `uses: ./.github/actions/sign-oci`, which resolves against the CALLING pack repo's checkout when invoked as a reusable workflow, not this repo -- so it would fail at a real pack release. Add the same job.workflow_repository@job.workflow_sha checkout that pack-release.yaml already uses for its Python script, and point both sign-oci steps and pack-release.yaml's sign-blob step at that checkout instead. Extend .github/actionlint.yaml's existing ignore-list workaround (actionlint doesn't yet know about job.workflow_repository/job.workflow_sha) to cover pack-build-image.yaml too. verifying-nebari-artifacts.md's gh attestation verify commands only passed --repo, which is not sufficient to pin the signer when the attestation was generated by a reusable workflow (per the gh CLI manual) -- add --signer-workflow pointing at pack-build-image.yaml or pack-release.yaml as appropriate, for every affected command. Repoint sign-smoke.yaml's trigger from the throwaway feature branch to push on main + pull_request (path-filtered to the sign actions/workflow), so it gets real post-merge regression coverage instead of only running on this branch. --- .github/actionlint.yaml | 8 +++++++ .github/workflows/pack-build-image.yaml | 18 +++++++++++++-- .github/workflows/pack-release.yaml | 2 +- .github/workflows/sign-smoke.yaml | 17 ++++++++++---- verifying-nebari-artifacts.md | 30 +++++++++++++++++++++---- 5 files changed, 64 insertions(+), 11 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index af7696c..faf9f5f 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -15,3 +15,11 @@ paths: # ships support for the job workflow-identity properties. - 'property "workflow_repository" is not defined in object type' - 'property "workflow_sha" is not defined in object type' + .github/workflows/pack-build-image.yaml: + ignore: + # Same job.workflow_repository/job.workflow_sha checkout pattern as + # pack-release.yaml above (this workflow's composite sign actions must + # also be loaded from a checkout of this repo, not the caller's) -- + # same actionlint gap, see the comment on the pack-release.yaml entry. + - 'property "workflow_repository" is not defined in object type' + - 'property "workflow_sha" is not defined in object type' diff --git a/.github/workflows/pack-build-image.yaml b/.github/workflows/pack-build-image.yaml index 6d31a05..b317bd3 100644 --- a/.github/workflows/pack-build-image.yaml +++ b/.github/workflows/pack-build-image.yaml @@ -102,16 +102,30 @@ jobs: cache-from: type=registry,ref=${{ env.GHCR_IMAGE }}:cache cache-to: ${{ inputs.push && format('type=registry,ref={0}:cache,mode=max', env.GHCR_IMAGE) || '' }} + # Reusable workflows resolve `uses: ./` against the CALLER's workspace, + # not this repo, so the composite sign actions must be loaded from a + # checkout of THIS repo. job.workflow_repository@job.workflow_sha pins + # exactly the nebari-dev/.github commit the caller invoked, keeping the + # action version coherent with the workflow version. (Same pattern as + # pack-release.yaml's _dot-github checkout.) + - name: Check out the reusable workflow's repo (for the sign action) + if: ${{ inputs.push }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: _dot-github + - name: Sign + attest (GHCR) if: ${{ inputs.push }} - uses: ./.github/actions/sign-oci + uses: ./_dot-github/.github/actions/sign-oci with: image: ${{ env.GHCR_IMAGE }} digest: ${{ steps.build.outputs.digest }} - name: Sign + attest (Quay) if: ${{ inputs.push }} - uses: ./.github/actions/sign-oci + uses: ./_dot-github/.github/actions/sign-oci with: image: ${{ env.QUAY_IMAGE }} digest: ${{ steps.build.outputs.digest }} diff --git a/.github/workflows/pack-release.yaml b/.github/workflows/pack-release.yaml index 93588a5..b91d504 100644 --- a/.github/workflows/pack-release.yaml +++ b/.github/workflows/pack-release.yaml @@ -212,7 +212,7 @@ jobs: - name: Sign + attest chart id: sign if: steps.exists.outputs.exists == 'false' - uses: ./.github/actions/sign-blob + uses: ./_dot-github/.github/actions/sign-blob with: path: ${{ steps.tgz.outputs.path }} diff --git a/.github/workflows/sign-smoke.yaml b/.github/workflows/sign-smoke.yaml index 0b5c00e..7380d1d 100644 --- a/.github/workflows/sign-smoke.yaml +++ b/.github/workflows/sign-smoke.yaml @@ -1,13 +1,22 @@ name: sign-smoke # Exercises the sign-oci / sign-blob actions end-to-end against throwaway -# artifacts on every push to a signing-work branch, and on demand. Proves -# journeys 1-5. Signs against ghcr only (this repo has no quay creds). +# artifacts whenever the sign actions/workflow change (push to main, or PR), +# and on demand. Proves journeys 1-5. Signs against ghcr only (this repo has +# no quay creds). on: push: - branches: - - "feat/reusable-signing-actions" + branches: [main] + paths: + - ".github/actions/**" + - ".github/workflows/sign-smoke.yaml" + - ".github/workflows/smoke-image/**" + pull_request: + paths: + - ".github/actions/**" + - ".github/workflows/sign-smoke.yaml" + - ".github/workflows/smoke-image/**" workflow_dispatch: env: diff --git a/verifying-nebari-artifacts.md b/verifying-nebari-artifacts.md index c486bd4..63a4574 100644 --- a/verifying-nebari-artifacts.md +++ b/verifying-nebari-artifacts.md @@ -104,11 +104,26 @@ SBOM), verifiable with `gh attestation verify` against the **pack repo** (that's where the workflow run - and therefore the attestation - is recorded), not `nebari-dev/.github`. +Because the attestation was generated by this repo's reusable workflow +rather than a workflow defined in the pack repo itself, `--repo` alone is +not enough to pin the signer - you must also pass `--signer-workflow`. Per +the [`gh attestation verify` +manual](https://cli.github.com/manual/gh_attestation_verify): + +> Please note: if your attestation was generated via a reusable workflow +> then that reusable workflow is the signer whose identity needs to be +> validated. In this situation, you must use either the +> `--signer-workflow` or the `--signer-repo` flag. + +`--signer-workflow` takes `[host/]////` - +no `@ref` and no `https://` prefix. + Build provenance: ```bash gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: \ - --repo nebari-dev/llm-serving-pack + --repo nebari-dev/llm-serving-pack \ + --signer-workflow nebari-dev/.github/.github/workflows/pack-build-image.yaml ``` SBOM (pass the SPDX predicate type explicitly; `gh attestation verify` @@ -117,6 +132,7 @@ otherwise defaults to the SLSA provenance predicate): ```bash gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: \ --repo nebari-dev/llm-serving-pack \ + --signer-workflow nebari-dev/.github/.github/workflows/pack-build-image.yaml \ --predicate-type https://spdx.dev/Document/v2.3 ``` @@ -144,11 +160,15 @@ cosign verify-blob \ The chart also carries the same kind of GitHub attestations as the images (see [section 2](#2-build-provenance-and-sbom-attestations)), verifiable -against the local file instead of an `oci://` reference: +against the local file instead of an `oci://` reference. As with the image +attestations, `--signer-workflow` is required - here it points at +`pack-release.yaml`, the reusable workflow that actually signed the chart: ```bash -gh attestation verify llm-serving-pack-1.2.3.tgz --repo nebari-dev/llm-serving-pack gh attestation verify llm-serving-pack-1.2.3.tgz --repo nebari-dev/llm-serving-pack \ + --signer-workflow nebari-dev/.github/.github/workflows/pack-release.yaml +gh attestation verify llm-serving-pack-1.2.3.tgz --repo nebari-dev/llm-serving-pack \ + --signer-workflow nebari-dev/.github/.github/workflows/pack-release.yaml \ --predicate-type https://spdx.dev/Document/v2.3 ``` @@ -167,7 +187,9 @@ cosign verify "${IMAGE}@${DIGEST}" \ # Raw attestation claims, as JSON: gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: \ - --repo nebari-dev/llm-serving-pack --format json | jq . + --repo nebari-dev/llm-serving-pack \ + --signer-workflow nebari-dev/.github/.github/workflows/pack-build-image.yaml \ + --format json | jq . ``` If the observed identity ever differs from what's documented here, this file From 97b6a4897ed7767816101e8f07476a70cd90fd19 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:46:45 +0200 Subject: [PATCH 09/10] feat: add sync opt-out to pack-release for test/non-product callers Adds a boolean `sync` input (default true) gating the sync-chart step, so callers can run the full release path (read version, pin tags, package, sign/attest, GitHub Release) without opening a PR in nebari-dev/helm-repository. Makes NEBARI_HELM_REPO_TOKEN required: false since it is only used by the sync step (needed only when sync is true). Default preserves existing behavior. Closes #47 --- .github/workflows/pack-release.yaml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pack-release.yaml b/.github/workflows/pack-release.yaml index b91d504..31b33d6 100644 --- a/.github/workflows/pack-release.yaml +++ b/.github/workflows/pack-release.yaml @@ -42,6 +42,13 @@ name: pack-release # workflow deliberately does not add its own ensure-quay-repos step: it # has no access to that token, and calling it here would be redundant # with (and could race) the downstream automation that already runs it. +# +# Sync opt-out: callers can set `sync: false` to run the full release path +# (read version, pin tags, package, sign/attest, GitHub Release) WITHOUT +# syncing the chart into nebari-dev/helm-repository -- useful for +# test/non-product callers (e.g. an integration test bed) that must not +# publish to the shared catalog. Defaults to true (unchanged behavior). +# NEBARI_HELM_REPO_TOKEN is only needed when sync is true. on: workflow_call: @@ -67,12 +74,22 @@ on: worker.image.tag required: true type: string + sync: + description: >- + Sync the packaged chart to nebari-dev/helm-repository. Set false to + skip publishing to the shared catalog (e.g. test/non-product callers) + while still running packaging, signing, attestation, and the GitHub + Release. + required: false + type: boolean + default: true secrets: NEBARI_HELM_REPO_TOKEN: description: >- Fine-grained PAT with contents + pull-request write on - nebari-dev/helm-repository. Used only by the sync-chart step. - required: true + nebari-dev/helm-repository. Used only by the sync-chart step, so it + is only required when `sync` is true. + required: false jobs: release: @@ -244,7 +261,7 @@ jobs: "${tgz_files[0]}.spdx.json" - name: Sync chart to nebari-dev/helm-repository - if: steps.exists.outputs.exists == 'false' + if: steps.exists.outputs.exists == 'false' && inputs.sync uses: nebari-dev/helm-repository/.github/actions/sync-chart@5cbd23a45c014bf2fa34b4683d4e5ac70ad34fa4 # main 2026-07-03 with: token: ${{ secrets.NEBARI_HELM_REPO_TOKEN }} From c73e72215ce2d3fec660652bb8fdfad19bacace8 Mon Sep 17 00:00:00 2001 From: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:29:04 +0200 Subject: [PATCH 10/10] docs: document canonical OCI chart signing + verification (phase 2 shipped) --- verifying-nebari-artifacts.md | 56 ++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/verifying-nebari-artifacts.md b/verifying-nebari-artifacts.md index 63a4574..4c56e42 100644 --- a/verifying-nebari-artifacts.md +++ b/verifying-nebari-artifacts.md @@ -18,11 +18,10 @@ token), so there is no public key to distribute. Verification instead checks that the Fulcio certificate was issued to the expected workflow identity by the expected OIDC issuer. This doc gives copy-pasteable commands for that. -> This covers artifacts produced by a pack's own repo: its container images -> and its release `.tgz`. It does **not** cover the canonical OCI Helm chart -> published to `quay.io/nebari/charts` - see -> [Phase 2: the canonical OCI Helm chart](#phase-2-the-canonical-oci-helm-chart) -> below. +> This covers three artifact types: a pack's container images, its release +> `.tgz`, and the canonical OCI Helm chart published to `quay.io/nebari/charts` +> (which has a different signer identity - see +> [The canonical OCI Helm chart](#the-canonical-oci-helm-chart) below). ## Prerequisites @@ -195,18 +194,35 @@ gh attestation verify oci://ghcr.io/nebari-dev/llm-serving-pack/operator@sha256: If the observed identity ever differs from what's documented here, this file is out of date - update it rather than working around it. -## Phase 2: the canonical OCI Helm chart - -`pack-release.yaml` only packages the chart as a `.tgz` GitHub Release asset -and opens a PR to sync the chart source into -[`nebari-dev/helm-repository`](https://github.com/nebari-dev/helm-repository). -The canonical OCI chart pushed to `quay.io/nebari/charts` is produced later, -by `helm-repository`'s own `release-helm-charts.yml` workflow, once that PR -merges. - -That means the OCI chart has a **different** signer identity from the one -documented above: it will be a path inside `nebari-dev/helm-repository`, not -`nebari-dev/.github`. Wiring `sign-oci` into that workflow is tracked as a -follow-up (phase 2) and is not yet done as of this writing - there is nothing -to verify at `quay.io/nebari/charts` yet. This doc will be updated with that -identity and command once phase 2 ships. +## The canonical OCI Helm chart + +`pack-release.yaml` signs the chart `.tgz` attached to the pack's GitHub Release +(above). The chart consumers actually `helm install` is a **separate** artifact: +the OCI chart pushed to `quay.io/nebari/charts` by +[`nebari-dev/helm-repository`](https://github.com/nebari-dev/helm-repository)'s +own `release-helm-charts.yml` workflow. That workflow signs and +provenance-attests each OCI chart with the same `sign-oci` action (signature + +SLSA provenance, but **no SBOM** - an SBOM of a Helm chart artifact is not +meaningful). + +Because the signing happens inside `helm-repository`'s workflow, the OCI chart's +signer identity is `release-helm-charts.yml` in **that** repo - distinct from the +`pack-build-image.yaml` / `pack-release.yaml` identities above. Verify the +signature: + +```bash +cosign verify quay.io/nebari/charts/: \ + --certificate-identity-regexp "https://github.com/nebari-dev/helm-repository/\.github/workflows/release-helm-charts\.yml@.*" \ + --certificate-oidc-issuer "https://token.actions.githubusercontent.com" +``` + +And the build provenance: + +```bash +gh attestation verify oci://quay.io/nebari/charts/@sha256: \ + --repo nebari-dev/helm-repository \ + --signer-workflow nebari-dev/helm-repository/.github/workflows/release-helm-charts.yml +``` + +The identity uses `@.*` because the exact ref depends on how the release ran +(e.g. `@refs/heads/main`); match on the workflow path, not a fixed ref.