From 36e95e59050f55bbfa3240aa060bd201e3d8b6e9 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 23 Sep 2026 17:09:58 +0530 Subject: [PATCH 1/7] Split package publish from from-source image and Helm release. Remove the npm-install Dockerfile so Docker and Helm no longer wait for npm, and publish the chart in the same image workflow instead of a bot PR. Signed-off-by: Raman Tehlan --- .../build-and-prepare-chart-release.yml | 197 ----------------- .github/workflows/build-dev-image.yml | 4 +- .github/workflows/ci.yml | 1 - .github/workflows/release-chart.yml | 62 +----- .github/workflows/release-image-chart.yml | 203 ++++++++++++++++++ .github/workflows/release.yml | 71 +----- .railway/README.md | 2 +- .railway/railway.ts | 12 +- Dockerfile | 118 ++++++++-- Dockerfile.dev | 116 ---------- RELEASING.md | 158 +++++++------- charts/trueforge/Chart.yaml | 5 +- charts/trueforge/README.md | 12 +- charts/trueforge/values.yaml | 10 +- docker-compose.yml | 3 - docs/quickstart.mdx | 4 +- 16 files changed, 416 insertions(+), 562 deletions(-) delete mode 100644 .github/workflows/build-and-prepare-chart-release.yml create mode 100644 .github/workflows/release-image-chart.yml delete mode 100644 Dockerfile.dev diff --git a/.github/workflows/build-and-prepare-chart-release.yml b/.github/workflows/build-and-prepare-chart-release.yml deleted file mode 100644 index 654c172b2..000000000 --- a/.github/workflows/build-and-prepare-chart-release.yml +++ /dev/null @@ -1,197 +0,0 @@ -name: Build and prepare chart release - -# Prod lane: build the npm-install Dockerfile, push {appVersion}-{shortSha}, -# then open/update the chart-release bot PR (merge of that PR tags + publishes). -# -# Called from release.yml after npm publish (same commit as the publish job). -# workflow_dispatch is for manual rebuilds. Do not use `gh workflow run --ref -# $GITHUB_SHA`: the dispatch API only accepts a branch or tag name (HTTP 422). -on: - workflow_call: - inputs: - app_version: - description: npm version to install (empty = Chart.yaml appVersion) - required: false - type: string - update_app_version: - description: Also set Chart.yaml appVersion to app_version on the bot PR - required: false - type: boolean - default: false - workflow_dispatch: - inputs: - app_version: - description: npm version to install (empty = Chart.yaml appVersion) - required: false - type: string - update_app_version: - description: Also set Chart.yaml appVersion to app_version on the bot PR - required: false - type: boolean - default: false - -concurrency: - group: release-image-and-chart - cancel-in-progress: false - -permissions: {} - -jobs: - resolve: - name: Resolve versions - runs-on: ubuntu-latest - outputs: - app_version: ${{ steps.resolve.outputs.app_version }} - image_tag: ${{ steps.resolve.outputs.image_tag }} - update_app_version: ${{ steps.resolve.outputs.update_app_version }} - short_sha: ${{ steps.resolve.outputs.short_sha }} - permissions: - contents: read - steps: - - name: Check out - uses: actions/checkout@v7 - - - name: Install yq - run: | - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.53.3/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - - name: Resolve app version and image tag - id: resolve - env: - INPUT_APP_VERSION: ${{ inputs.app_version }} - INPUT_UPDATE_APP_VERSION: ${{ inputs.update_app_version }} - run: | - set -euo pipefail - CHART_APP=$(yq -r '.appVersion' charts/trueforge/Chart.yaml | tr -d '"') - if [[ -n "${INPUT_APP_VERSION}" ]]; then - APP_VERSION="$INPUT_APP_VERSION" - else - APP_VERSION="$CHART_APP" - fi - if [[ -z "$APP_VERSION" || "$APP_VERSION" == "null" ]]; then - echo "Could not resolve APP_VERSION" >&2 - exit 1 - fi - if ! printf '%s' "$APP_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$'; then - echo "APP_VERSION '$APP_VERSION' is not a semver-like version" >&2 - exit 1 - fi - SHORT_SHA="${GITHUB_SHA:0:7}" - IMAGE_TAG="${APP_VERSION}-${SHORT_SHA}" - UPDATE_APP="${INPUT_UPDATE_APP_VERSION:-false}" - # workflow_dispatch boolean can be the string "true"/"false" - if [[ "$UPDATE_APP" == "true" ]]; then - UPDATE_OUT=true - else - UPDATE_OUT=false - fi - echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" - echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" - echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" - echo "update_app_version=$UPDATE_OUT" >> "$GITHUB_OUTPUT" - echo "Resolved APP_VERSION=$APP_VERSION IMAGE_TAG=$IMAGE_TAG update_app_version=$UPDATE_OUT" - - build: - name: Build and push server image - needs: [resolve] - uses: truefoundry/github-workflows-public/.github/workflows/build.yml@main - permissions: - id-token: write - contents: read - with: - artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} - artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }} - image_artifact_name: trueforge - image_tag: ${{ needs.resolve.outputs.image_tag }} - image_context: . - dockerfile_path: Dockerfile - image_build_args: | - APP_VERSION=${{ needs.resolve.outputs.app_version }} - platforms: linux/amd64 - enable_jfrog: true - enable_public_ecr: false - secrets: - artifactory_username: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} - artifactory_password: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} - - open-chart-pr: - name: Open chart release PR - needs: [resolve, build] - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - env: - PR_BRANCH: release-chart/trueforge - steps: - - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_ID }} - private-key: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_PRIVATE_KEY }} - - - name: Check out - uses: actions/checkout@v7 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Install yq - run: | - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.53.3/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - - name: Update chart metadata - id: chart - env: - IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} - APP_VERSION: ${{ needs.resolve.outputs.app_version }} - UPDATE_APP_VERSION: ${{ needs.resolve.outputs.update_app_version }} - run: | - set -euo pipefail - CURRENT=$(yq -r '.version' charts/trueforge/Chart.yaml) - PR_VERSION="" - - # create-pull-request rebuilds the PR branch from base plus this - # workspace, so a reviewer's major/minor bump on the open PR is dropped - # unless it is re-applied here. - if git ls-remote --exit-code --heads origin "$PR_BRANCH" >/dev/null 2>&1; then - git fetch --depth=1 origin "$PR_BRANCH" - PR_VERSION=$(git show FETCH_HEAD:charts/trueforge/Chart.yaml | yq -r '.version') - fi - VERSION=$(bash scripts/resolve-chart-version.sh "$CURRENT" "$APP_VERSION" "$PR_VERSION") - if [[ -n "$PR_VERSION" && "$VERSION" == "$PR_VERSION" ]]; then - echo "Keeping chart version $PR_VERSION from $PR_BRANCH" - fi - export VERSION IMAGE_TAG APP_VERSION - yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml - yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml - if [[ "$UPDATE_APP_VERSION" == "true" ]]; then - yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml - fi - APP_VERSION=$(yq -r '.appVersion' charts/trueforge/Chart.yaml) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" - - - name: Open or update chart-release PR - uses: peter-evans/create-pull-request@v8 - with: - token: ${{ steps.app-token.outputs.token }} - branch: ${{ env.PR_BRANCH }} - base: main - commit-message: 'release(chart): ${{ steps.chart.outputs.version }}' - title: 'release(chart): ${{ steps.chart.outputs.version }} (image ${{ needs.resolve.outputs.image_tag }})' - body: | - ## Chart release - - | Field | Value | - | --- | --- | - | Chart version | `${{ steps.chart.outputs.version }}` | - | appVersion | `${{ steps.chart.outputs.app_version }}` | - | image.tag | `${{ needs.resolve.outputs.image_tag }}` | - - Merging this PR creates `charts/trueforge@${{ steps.chart.outputs.version }}`, - creates a GitHub Release, and publishes the OCI Helm chart. - add-paths: | - charts/trueforge/Chart.yaml - charts/trueforge/values.yaml diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-dev-image.yml index 096727891..fdbf97863 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-dev-image.yml @@ -1,7 +1,7 @@ name: Build dev image # From-source image for floating-main / external deploy repos. -# Builds Dockerfile.dev and pushes .../trueforge:. No chart publish. +# Builds the root Dockerfile and pushes .../trueforge:. No chart publish. on: workflow_dispatch: @@ -25,7 +25,7 @@ jobs: image_artifact_name: trueforge image_tag: ${{ github.sha }} image_context: . - dockerfile_path: Dockerfile.dev + dockerfile_path: Dockerfile platforms: linux/amd64 enable_jfrog: true enable_public_ecr: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f89a36777..1a3471b47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,6 @@ jobs: chart: - 'charts/trueforge/**' - 'Dockerfile' - - 'Dockerfile.dev' - 'package.json' - '.github/workflows/ci.yml' diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index 3a51464e7..298fad421 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -1,11 +1,8 @@ name: Publish Helm chart -# A chart-release PR merge creates the tag/GitHub Release and publishes the -# chart. Human charts/trueforge@* tags use the same publish job. +# Retry / human-tag path. The prod lane publishes from release-image-chart.yml. +# Human charts/trueforge@* tags and workflow_dispatch reuse this job. on: - pull_request: - types: [closed] - branches: [main] push: tags: - 'charts/trueforge@*' @@ -24,14 +21,7 @@ permissions: {} jobs: release: - name: Create release and publish chart - if: >- - github.event_name != 'pull_request' || - ( - github.event.pull_request.merged == true && - github.event.pull_request.head.ref == 'release-chart/trueforge' && - github.event.pull_request.head.repo.full_name == github.repository - ) + name: Publish chart runs-on: ubuntu-latest permissions: contents: write @@ -43,7 +33,7 @@ jobs: - name: Check out chart commit uses: actions/checkout@v7 with: - ref: ${{ github.event.pull_request.merge_commit_sha || inputs.tag || github.ref }} + ref: ${{ inputs.tag || github.ref }} fetch-depth: 0 - name: Install yq @@ -56,18 +46,12 @@ jobs: with: version: v3.16.4 - - name: Resolve or create chart release + - name: Resolve chart tag env: - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} INPUT_TAG: ${{ inputs.tag }} - MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} - PR_BODY: ${{ github.event.pull_request.body }} - PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail - # Read the chart identity from the checked-out commit. VERSION=$(yq -r '.version' charts/trueforge/Chart.yaml) APP_VERSION=$(yq -r '.appVersion' charts/trueforge/Chart.yaml) IMAGE_TAG=$(yq -r '.image.tag' charts/trueforge/values.yaml) @@ -77,44 +61,12 @@ jobs: exit 1 fi - # Path A: bot chart-release PR was merged. - # Create the charts/trueforge@ tag + GitHub Release here, - # because GITHUB_TOKEN tag pushes do not start a separate push workflow. - if [[ "$EVENT_NAME" == "pull_request" ]]; then - TAG="charts/trueforge@${VERSION}" - - if git rev-parse "$TAG" >/dev/null 2>&1; then - echo "Tag $TAG already exists; refusing to move it" >&2 - exit 1 - fi - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag -a "$TAG" -m "Helm chart ${VERSION} (appVersion ${APP_VERSION}, image ${IMAGE_TAG})" - git push origin "$TAG" - - # Write notes via a file so PR body cannot inject into the shell. - NOTES_FILE="$(mktemp)" - { - echo "Source PR: #${PR_NUMBER}" - echo - printf '%s\n' "$PR_BODY" - } >"$NOTES_FILE" - gh release create "$TAG" \ - --target "$MERGE_SHA" \ - --title "$TAG" \ - --notes-file "$NOTES_FILE" - rm -f "$NOTES_FILE" - # Path B: manual retry via workflow_dispatch with an explicit tag. - elif [[ -n "${INPUT_TAG:-}" ]]; then + if [[ -n "${INPUT_TAG:-}" ]]; then TAG="$INPUT_TAG" - - # Path C: human pushed charts/trueforge@* themselves. else TAG="${GITHUB_REF#refs/tags/}" fi - # Every path must publish the same chart version the tag names. case "$TAG" in charts/trueforge@*) EXPECTED="${TAG#charts/trueforge@}" ;; *) echo "Invalid chart tag: $TAG" >&2; exit 1 ;; @@ -126,11 +78,9 @@ jobs: exit 1 fi - # VERSION is consumed by the package/push step below. echo "VERSION=$VERSION" >> "$GITHUB_ENV" echo "Publishing chart version=$ACTUAL appVersion=$APP_VERSION image.tag=$IMAGE_TAG" - # Pull deps before registry login. - name: Build chart dependencies run: helm dependency build "$CHART_DIR" diff --git a/.github/workflows/release-image-chart.yml b/.github/workflows/release-image-chart.yml new file mode 100644 index 000000000..6f6b4fcf0 --- /dev/null +++ b/.github/workflows/release-image-chart.yml @@ -0,0 +1,203 @@ +name: Release image and Helm chart + +# Prod lane: build the from-source Dockerfile, push {appVersion}-{shortSha}, +# then package and publish the Helm chart (tag + GitHub Release + OCI). +# +# Called from release.yml after pack/smoke (same commit; does not wait for npm). +# workflow_dispatch is for manual rebuilds. Do not use `gh workflow run --ref +# $GITHUB_SHA`: the dispatch API only accepts a branch or tag name (HTTP 422). +on: + workflow_call: + inputs: + app_version: + description: App version for the image tag (empty = packages/trueforge/package.json) + required: false + type: string + workflow_dispatch: + inputs: + app_version: + description: App version for the image tag (empty = packages/trueforge/package.json) + required: false + type: string + +concurrency: + group: release-image-and-chart + cancel-in-progress: false + +permissions: {} + +jobs: + resolve: + name: Resolve versions + runs-on: ubuntu-latest + outputs: + app_version: ${{ steps.resolve.outputs.app_version }} + image_tag: ${{ steps.resolve.outputs.image_tag }} + short_sha: ${{ steps.resolve.outputs.short_sha }} + permissions: + contents: read + steps: + - name: Check out + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Resolve app version and image tag + id: resolve + env: + INPUT_APP_VERSION: ${{ inputs.app_version }} + run: | + set -euo pipefail + PKG_VERSION=$(node -p "require('./packages/trueforge/package.json').version") + if [[ -n "${INPUT_APP_VERSION}" ]]; then + APP_VERSION="$INPUT_APP_VERSION" + else + APP_VERSION="$PKG_VERSION" + fi + if [[ -z "$APP_VERSION" || "$APP_VERSION" == "undefined" ]]; then + echo "Could not resolve APP_VERSION" >&2 + exit 1 + fi + if ! printf '%s' "$APP_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$'; then + echo "APP_VERSION '$APP_VERSION' is not a semver-like version" >&2 + exit 1 + fi + SHORT_SHA="${GITHUB_SHA:0:7}" + IMAGE_TAG="${APP_VERSION}-${SHORT_SHA}" + echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "Resolved APP_VERSION=$APP_VERSION IMAGE_TAG=$IMAGE_TAG (package.json=$PKG_VERSION)" + + build: + name: Build and push server image + needs: [resolve] + uses: truefoundry/github-workflows-public/.github/workflows/build.yml@main + permissions: + id-token: write + contents: read + with: + artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} + artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }} + image_artifact_name: trueforge + image_tag: ${{ needs.resolve.outputs.image_tag }} + image_context: . + dockerfile_path: Dockerfile + platforms: linux/amd64 + enable_jfrog: true + enable_public_ecr: false + secrets: + artifactory_username: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} + artifactory_password: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} + + publish-chart: + name: Publish Helm chart + needs: [resolve, build] + runs-on: ubuntu-latest + permissions: + contents: write + env: + CHART_DIR: charts/trueforge + HELM_REGISTRY_URL: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} + HELM_CHART_REPOSITORY: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_HELM_REPOSITORY }} + steps: + - name: Check out + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Install yq + run: | + sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.53.3/yq_linux_amd64 -O /usr/bin/yq + sudo chmod +x /usr/bin/yq + + - name: Install Helm + uses: azure/setup-helm@v5 + with: + version: v3.16.4 + + - name: Update chart metadata + id: chart + env: + IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + APP_VERSION: ${{ needs.resolve.outputs.app_version }} + run: | + set -euo pipefail + CURRENT=$(yq -r '.version' charts/trueforge/Chart.yaml) + VERSION=$(bash scripts/resolve-chart-version.sh "$CURRENT" "$APP_VERSION") + export VERSION IMAGE_TAG APP_VERSION + yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml + yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml + yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" + echo "Chart version=$VERSION appVersion=$APP_VERSION image.tag=$IMAGE_TAG" + + - name: Build chart dependencies + run: helm dependency build "$CHART_DIR" + + - name: Lint chart + run: helm lint "$CHART_DIR" --values charts/trueforge/ci/lint-values.yaml + + - name: Commit chart metadata, tag, and GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.chart.outputs.version }} + APP_VERSION: ${{ steps.chart.outputs.app_version }} + IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + run: | + set -euo pipefail + TAG="charts/trueforge@${VERSION}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "Tag $TAG already exists; skipping commit and GitHub Release" + else + git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml + if git diff --staged --quiet; then + echo "Chart metadata already matches $VERSION; skipping commit" >&2 + else + git commit -m "release(chart): ${VERSION}" + git push origin HEAD:main + fi + + git tag -a "$TAG" -m "Helm chart ${VERSION} (appVersion ${APP_VERSION}, image ${IMAGE_TAG})" + git push origin "$TAG" + + NOTES_FILE="$(mktemp)" + { + echo "Chart version: ${VERSION}" + echo "appVersion: ${APP_VERSION}" + echo "image.tag: ${IMAGE_TAG}" + } >"$NOTES_FILE" + gh release create "$TAG" \ + --title "$TAG" \ + --notes-file "$NOTES_FILE" + rm -f "$NOTES_FILE" + fi + + - name: Helm registry login + env: + HELM_REGISTRY_USERNAME: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} + HELM_REGISTRY_PASSWORD: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} + run: | + printf '%s' "$HELM_REGISTRY_PASSWORD" \ + | helm registry login -u "$HELM_REGISTRY_USERNAME" --password-stdin "$HELM_REGISTRY_URL" + + - name: Package and push chart + env: + VERSION: ${{ steps.chart.outputs.version }} + run: | + set -euo pipefail + CHART_NAME=$(yq -r '.name' "$CHART_DIR/Chart.yaml") + mkdir -p dist + helm package "$CHART_DIR" --destination dist + PACKAGE="dist/${CHART_NAME}-${VERSION}.tgz" + echo "Pushing $PACKAGE to oci://$HELM_CHART_REPOSITORY" + helm push "$PACKAGE" "oci://$HELM_CHART_REPOSITORY" + { + echo "### Helm chart" + echo "Pushed \`oci://${HELM_CHART_REPOSITORY}/${CHART_NAME}:${VERSION}\`" + echo "Image tag \`${{ needs.resolve.outputs.image_tag }}\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 560552b86..a2e332911 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,13 +8,14 @@ # (install only; that PR is gated by CI.yml). When @truefoundry/trueforge-sdk # moves, scripts/version.mjs mirrors that version into python/trueforge_sdk and rebakes Fern. # - none pending + unpublished versions → pack (build/test) + Windows npx smoke, -# then npm publish and PyPI publish in parallel +# then npm publish, PyPI publish, and image+Helm release in parallel # Merge the Version Packages PR to publish. There is no separate publish Action # and no `v*` tag trigger. # # Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc` # disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN / -# NODE_AUTH_TOKEN. id-token is granted only on the publish jobs. +# NODE_AUTH_TOKEN. id-token is granted only on the npm/PyPI publish jobs (the +# image workflow requests its own). name: Release on: @@ -172,9 +173,6 @@ jobs: needs: [pack, windows-npx-smoke] runs-on: ubuntu-latest timeout-minutes: 20 - outputs: - trueforge_published: ${{ steps.trueforge.outputs.published }} - trueforge_version: ${{ steps.trueforge.outputs.version }} permissions: contents: write id-token: write @@ -209,51 +207,6 @@ jobs: # publishConfig.provenance is true on every public package. NPM_CONFIG_PROVENANCE: 'true' - - name: Detect @truefoundry/trueforge publish - id: trueforge - env: - PUBLISHED: ${{ steps.publish.outputs.published }} - PACKAGES_KEBAB: ${{ steps.publish.outputs['published-packages'] }} - PACKAGES_CAMEL: ${{ steps.publish.outputs.publishedPackages }} - run: | - set -euo pipefail - if [[ "${PUBLISHED:-}" != "true" ]]; then - echo "published=false" >> "$GITHUB_OUTPUT" - echo "version=" >> "$GITHUB_OUTPUT" - exit 0 - fi - PACKAGES="${PACKAGES_KEBAB:-${PACKAGES_CAMEL:-[]}}" - VERSION=$(printf '%s' "$PACKAGES" | python3 -c ' - import json,sys - pkgs=json.load(sys.stdin) - for p in pkgs: - if p.get("name")=="@truefoundry/trueforge": - print(p["version"]) - break - ') - if [[ -n "$VERSION" ]]; then - echo "published=true" >> "$GITHUB_OUTPUT" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "Published @truefoundry/trueforge@$VERSION" - else - echo "published=false" >> "$GITHUB_OUTPUT" - echo "version=" >> "$GITHUB_OUTPUT" - echo "@truefoundry/trueforge was not in this publish set" - fi - - - name: Wait for npm to advertise package - if: steps.trueforge.outputs.published == 'true' - env: - VERSION: ${{ steps.trueforge.outputs.version }} - run: | - echo "Newly published npm packages are typically available in about five minutes, and may take up to 15 minutes or more at peak times." - echo "https://github.blog/changelog/2026-07-28-npm-publish-time-malware-scanning-and-dual-use-metadata/" - for _ in $(seq 30); do - npm view "@truefoundry/trueforge@$VERSION" version && exit 0 - sleep 30 - done - exit 1 - # Sibling of npm publish (same gates). Skips when pyproject version is already on PyPI. publish-python: name: Publish Python SDK @@ -314,18 +267,14 @@ jobs: with: packages-dir: python/trueforge_sdk/dist - # Same commit as this publish run (GITHUB_SHA). A workflow_dispatch --ref SHA - # is rejected: the dispatch API only accepts a branch or tag name (HTTP 422). - build-and-prepare-chart-release: - name: Build and prepare chart release - needs: publish - if: needs.publish.outputs.trueforge_published == 'true' - uses: ./.github/workflows/build-and-prepare-chart-release.yml + # Same commit as this publish run (GITHUB_SHA). Does not wait for npm. + # A workflow_dispatch --ref SHA is rejected: the dispatch API only accepts + # a branch or tag name (HTTP 422). + release-image-and-chart: + name: Release image and Helm chart + needs: [pack, windows-npx-smoke] + uses: ./.github/workflows/release-image-chart.yml permissions: contents: write id-token: write - pull-requests: write secrets: inherit - with: - app_version: ${{ needs.publish.outputs.trueforge_version }} - update_app_version: true diff --git a/.railway/README.md b/.railway/README.md index 5375b3a89..9eef0348b 100644 --- a/.railway/README.md +++ b/.railway/README.md @@ -15,7 +15,7 @@ railway config apply railway domain # public URL for the trueforge service ``` -- **Image:** `RAILWAY_DOCKERFILE_PATH=Dockerfile.dev` selects the from-source build. +- **Image:** Railway builds the root from-source [`Dockerfile`](../Dockerfile). - **API key:** Set `TRUEFORGE_API_KEY` yourself as a Railway shared variable. IaC only wires `${{shared.TRUEFORGE_API_KEY}}` onto `trueforge` and `trueforge-controller`. - **Auth:** Off by default — anyone who can reach the URL is admin. Before sharing a deployment, enable [OIDC login](https://trueforge.dev/authentication/overview) (optional shared-variable block is commented in `railway.ts`). diff --git a/.railway/railway.ts b/.railway/railway.ts index d94a378db..288be706b 100644 --- a/.railway/railway.ts +++ b/.railway/railway.ts @@ -15,7 +15,7 @@ * Docs: https://docs.railway.com/infrastructure-as-code * * Auth is off by default (anyone who can reach the URL is admin). Before - * sharing a deployment, enable OIDC — see the commented block below and + * sharing a deployment, enable OIDC - see the commented block below and * https://trueforge.dev/authentication/overview */ import { defineRailway, github, group, postgres, project, redis, service } from 'railway/iac'; @@ -28,10 +28,7 @@ export default defineRailway(_ctx => { // Deploys from this repository's default branch. Forks: change owner/repo // (and optionally branch) to build your own copy. source: github('truefoundry/trueforge'), - // IaC `build` is a build *command* string (not CaC's builder/dockerfilePath object). - // Non-root Dockerfile is selected via RAILWAY_DOCKERFILE_PATH below — without it, - // Railway picks the root Dockerfile (npm install of a published version) and fails - // without APP_VERSION. + // Railway uses the root from-source Dockerfile by default. healthcheck: '/healthz', healthcheckTimeout: 300, deploy: { @@ -41,15 +38,13 @@ export default defineRailway(_ctx => { drainingSeconds: 35, }, env: { - // From-source image; root Dockerfile is the published npm recipe and needs APP_VERSION. - RAILWAY_DOCKERFILE_PATH: 'Dockerfile.dev', // Postgres + Redis (hosted topology) STANDALONE: 'false', DATABASE_URL: db.env.DATABASE_URL, REDIS_URL: cache.env.REDIS_URL, // Expanded by Railway at runtime once a public domain exists on this service. PUBLIC_BASE_URL: 'https://${{RAILWAY_PUBLIC_DOMAIN}}', - // Set once in Railway (shared); IaC only references it — see header. + // Set once in Railway (shared); IaC only references it - see header. TRUEFORGE_API_KEY: _ctx.shared.TRUEFORGE_API_KEY, // Optional OIDC (login off until these are set). Create matching *shared* @@ -82,7 +77,6 @@ export default defineRailway(_ctx => { drainingSeconds: 35, }, env: { - RAILWAY_DOCKERFILE_PATH: 'Dockerfile.dev', STANDALONE: 'false', DATABASE_URL: db.env.DATABASE_URL, // Same shared key as the app (Bearer auth for schedule dispatch). diff --git a/Dockerfile b/Dockerfile index afb5dc4b2..8f353fcb5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,115 @@ # syntax=docker/dockerfile:1 # -# Production image: installs published @truefoundry/trueforge from npm. -# The app bits match the npm package exactly (not floating monorepo source). +# From-source multi-stage image for prod Helm, local smoke, Railway, and +# SHA-tagged floating-main builds. Lives at the repository root because the +# build needs the whole pnpm workspace as its context: the server depends on +# the workspace package @truefoundry/trueforge-core. # -# Required build-arg: -# APP_VERSION — npm version to install, e.g. 0.1.0 -# -# Example: -# docker build --build-arg APP_VERSION=0.1.0 -t trueforge:0.1.0 . +# Dependency install uses pnpm fetch (lockfile-only) then install --offline so +# the download layer stays cached when only package.json / scripts change. +# See https://pnpm.io/cli/fetch. BuildKit cache mounts are avoided so the same +# file builds on Railway Metal (which requires a hardcoded service id in mount ids). -FROM node:24-slim AS runner +FROM node:24-slim AS base +ENV PNPM_HOME=/pnpm +ENV PATH="$PNPM_HOME:$PATH" +RUN corepack enable && pnpm config set store-dir /pnpm/store WORKDIR /app -# HOST=0.0.0.0 so Kubernetes Service/probe traffic reaches the process. + +# --------------------------------------------------------------------------- +# store: the pnpm store, from the lockfile only (stable when manifests churn). +# BuildKit cache mounts are omitted: Railway's Metal builder requires +# `id=s/-…` (hardcoded per deploy), which cannot live in a shared +# OSS Dockerfile. Layer cache on this stage still hits when the lockfile is +# unchanged. +# --------------------------------------------------------------------------- +FROM base AS store +COPY pnpm-lock.yaml pnpm-workspace.yaml ./ +RUN pnpm fetch + +# --------------------------------------------------------------------------- +# workspace: install inputs shared by every stage below - the manifests plus the +# sources the root postinstall hook (build:gen) inlines. +# --------------------------------------------------------------------------- +FROM store AS workspace +COPY package.json .npmrc tsconfig.base.json ./ +# Root scripts used by package build steps (rm-path.mjs, chmod-path.mjs). +COPY scripts scripts +COPY packages/trueforge-core/package.json packages/trueforge-core/package.json +COPY packages/trueforge/package.json packages/trueforge/package.json +COPY packages/trueforge-sdk/package.json packages/trueforge-sdk/package.json +COPY packages/frontend/package.json packages/frontend/package.json +COPY packages/trueforge-ui/package.json packages/trueforge-ui/package.json +COPY packages/assistant-ui-runtime/package.json packages/assistant-ui-runtime/package.json +COPY packages/trueforge-core/scripts packages/trueforge-core/scripts +COPY packages/trueforge-core/src/core/sandbox/scripts packages/trueforge-core/src/core/sandbox/scripts + +# --------------------------------------------------------------------------- +# builder: install all deps (incl. dev) and build trueforge-core + server. +# SDK dist is copied into the production runner; host-dev/typecheck resolve SDK +# from src/ via trueforge-dev. Build it here once for the image. +# --------------------------------------------------------------------------- +FROM workspace AS builder +RUN pnpm install --frozen-lockfile --offline --filter @truefoundry/trueforge... +COPY packages/trueforge-core packages/trueforge-core +COPY packages/trueforge-sdk packages/trueforge-sdk +RUN pnpm --filter @truefoundry/trueforge-sdk build +COPY packages/trueforge packages/trueforge +RUN pnpm --filter @truefoundry/trueforge-core build && pnpm --filter @truefoundry/trueforge build + +# --------------------------------------------------------------------------- +# frontend-builder: build the UI the server serves (parallel to builder above). +# Bundlers read SDK src/ (trueforge-dev); the UI prebuild emits the declarations +# tsc needs, so this stage never compiles SDK JavaScript. +# --------------------------------------------------------------------------- +FROM workspace AS frontend-builder +RUN pnpm install --frozen-lockfile --offline --filter frontend... +COPY packages/trueforge-sdk packages/trueforge-sdk +COPY packages/assistant-ui-runtime packages/assistant-ui-runtime +COPY packages/trueforge-ui packages/trueforge-ui +RUN pnpm --filter @truefoundry/trueforge-ui build +COPY packages/frontend packages/frontend +RUN pnpm --filter frontend build + +# --------------------------------------------------------------------------- +# prod-deps: production dependency tree (no dev tooling), resolved offline. +# --------------------------------------------------------------------------- +FROM workspace AS prod-deps +RUN pnpm install --frozen-lockfile --offline --prod --filter @truefoundry/trueforge... + +# --------------------------------------------------------------------------- +# runner: minimal image with prod node_modules + built artifacts. +# --------------------------------------------------------------------------- +FROM base AS runner + ENV NODE_ENV=production \ - STANDALONE=false \ HOST=0.0.0.0 -ARG APP_VERSION -RUN test -n "$APP_VERSION" || (echo "APP_VERSION build-arg is required" >&2 && exit 1) +# Production dependency tree (pnpm workspace symlinks preserved). +COPY --from=prod-deps /app/node_modules ./node_modules +COPY --from=prod-deps /app/packages/trueforge-core/node_modules ./packages/trueforge-core/node_modules +COPY --from=prod-deps /app/packages/trueforge/node_modules ./packages/trueforge/node_modules + +# Built workspace dependencies (@truefoundry/trueforge-core + SDK). +COPY --from=builder /app/packages/trueforge-core/package.json ./packages/trueforge-core/package.json +COPY --from=builder /app/packages/trueforge-core/dist ./packages/trueforge-core/dist +COPY --from=builder /app/packages/trueforge-sdk/package.json ./packages/trueforge-sdk/package.json +COPY --from=builder /app/packages/trueforge-sdk/dist ./packages/trueforge-sdk/dist + +# Built server (JS). UI is copied below from the parallel frontend stage into +# dist/_frontend - same path as the npm tarball / `pnpm build` copy step. +COPY --from=builder /app/packages/trueforge/package.json ./packages/trueforge/package.json +COPY --from=builder /app/packages/trueforge/dist ./packages/trueforge/dist +# Frontend builds in a parallel stage; place it at the same path as the npm tarball. +COPY --from=frontend-builder /app/packages/frontend/dist ./packages/trueforge/dist/_frontend -# Fail closed if the version is not on the registry (no workspace fallback). -RUN npm install --omit=dev "@truefoundry/trueforge@${APP_VERSION}" \ - && npm cache clean --force +WORKDIR /app/packages/trueforge RUN groupadd --gid 10001 trueforge \ - && useradd --uid 10001 --gid trueforge --shell /usr/sbin/nologin trueforge + && useradd --uid 10001 --gid trueforge trueforge EXPOSE 8790 -# Same entry as the from-source image / `pnpm start` (launch-only; dist is in the package). +# Launch-only (matches root `pnpm start` / `standalone:start`). Image already contains dist. USER 10001:10001 -CMD ["node", "node_modules/@truefoundry/trueforge/dist/main.js"] +CMD ["node", "dist/main.js"] diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 3e44d270b..000000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,116 +0,0 @@ -# syntax=docker/dockerfile:1 -# -# From-source multi-stage build for local smoke / floating-main / SHA-tagged -# dev images. Production/OSS Helm images use the root Dockerfile (npm install). -# -# Lives at the repository root because the build needs the whole pnpm workspace -# as its context: the server depends on the workspace package @truefoundry/trueforge-core. -# -# Dependency install uses pnpm fetch (lockfile-only) then install --offline so -# the download layer stays cached when only package.json / scripts change. -# See https://pnpm.io/cli/fetch. BuildKit cache mounts are avoided so the same -# file builds on Railway Metal (which requires a hardcoded service id in mount ids). - -FROM node:24-slim AS base -ENV PNPM_HOME=/pnpm -ENV PATH="$PNPM_HOME:$PATH" -RUN corepack enable && pnpm config set store-dir /pnpm/store -WORKDIR /app - -# --------------------------------------------------------------------------- -# store: the pnpm store, from the lockfile only (stable when manifests churn). -# BuildKit cache mounts are omitted: Railway's Metal builder requires -# `id=s/-…` (hardcoded per deploy), which cannot live in a shared -# OSS Dockerfile. Layer cache on this stage still hits when the lockfile is -# unchanged. -# --------------------------------------------------------------------------- -FROM base AS store -COPY pnpm-lock.yaml pnpm-workspace.yaml ./ -RUN pnpm fetch - -# --------------------------------------------------------------------------- -# workspace: install inputs shared by every stage below — the manifests plus the -# sources the root postinstall hook (build:gen) inlines. -# --------------------------------------------------------------------------- -FROM store AS workspace -COPY package.json .npmrc tsconfig.base.json ./ -# Root scripts used by package build steps (rm-path.mjs, chmod-path.mjs). -COPY scripts scripts -COPY packages/trueforge-core/package.json packages/trueforge-core/package.json -COPY packages/trueforge/package.json packages/trueforge/package.json -COPY packages/trueforge-sdk/package.json packages/trueforge-sdk/package.json -COPY packages/frontend/package.json packages/frontend/package.json -COPY packages/trueforge-ui/package.json packages/trueforge-ui/package.json -COPY packages/assistant-ui-runtime/package.json packages/assistant-ui-runtime/package.json -COPY packages/trueforge-core/scripts packages/trueforge-core/scripts -COPY packages/trueforge-core/src/core/sandbox/scripts packages/trueforge-core/src/core/sandbox/scripts - -# --------------------------------------------------------------------------- -# builder: install all deps (incl. dev) and build trueforge-core + server. -# SDK dist is copied into the production runner; host-dev/typecheck resolve SDK -# from src/ via trueforge-dev. Build it here once for the image. -# --------------------------------------------------------------------------- -FROM workspace AS builder -RUN pnpm install --frozen-lockfile --offline --filter @truefoundry/trueforge... -COPY packages/trueforge-core packages/trueforge-core -COPY packages/trueforge-sdk packages/trueforge-sdk -RUN pnpm --filter @truefoundry/trueforge-sdk build -COPY packages/trueforge packages/trueforge -RUN pnpm --filter @truefoundry/trueforge-core build && pnpm --filter @truefoundry/trueforge build - -# --------------------------------------------------------------------------- -# frontend-builder: build the UI the server serves (parallel to builder above). -# Bundlers read SDK src/ (trueforge-dev); the UI prebuild emits the declarations -# tsc needs, so this stage never compiles SDK JavaScript. -# --------------------------------------------------------------------------- -FROM workspace AS frontend-builder -RUN pnpm install --frozen-lockfile --offline --filter frontend... -COPY packages/trueforge-sdk packages/trueforge-sdk -COPY packages/assistant-ui-runtime packages/assistant-ui-runtime -COPY packages/trueforge-ui packages/trueforge-ui -RUN pnpm --filter @truefoundry/trueforge-ui build -COPY packages/frontend packages/frontend -RUN pnpm --filter frontend build - -# --------------------------------------------------------------------------- -# prod-deps: production dependency tree (no dev tooling), resolved offline. -# --------------------------------------------------------------------------- -FROM workspace AS prod-deps -RUN pnpm install --frozen-lockfile --offline --prod --filter @truefoundry/trueforge... - -# --------------------------------------------------------------------------- -# runner: minimal image with prod node_modules + built artifacts. -# --------------------------------------------------------------------------- -FROM base AS runner - -ENV NODE_ENV=production \ - HOST=0.0.0.0 - -# Production dependency tree (pnpm workspace symlinks preserved). -COPY --from=prod-deps /app/node_modules ./node_modules -COPY --from=prod-deps /app/packages/trueforge-core/node_modules ./packages/trueforge-core/node_modules -COPY --from=prod-deps /app/packages/trueforge/node_modules ./packages/trueforge/node_modules - -# Built workspace dependencies (@truefoundry/trueforge-core + SDK). -COPY --from=builder /app/packages/trueforge-core/package.json ./packages/trueforge-core/package.json -COPY --from=builder /app/packages/trueforge-core/dist ./packages/trueforge-core/dist -COPY --from=builder /app/packages/trueforge-sdk/package.json ./packages/trueforge-sdk/package.json -COPY --from=builder /app/packages/trueforge-sdk/dist ./packages/trueforge-sdk/dist - -# Built server (JS). UI is copied below from the parallel frontend stage into -# dist/_frontend — same path as the npm tarball / `pnpm build` copy step. -COPY --from=builder /app/packages/trueforge/package.json ./packages/trueforge/package.json -COPY --from=builder /app/packages/trueforge/dist ./packages/trueforge/dist -# Frontend builds in a parallel stage; place it at the same path as the npm tarball. -COPY --from=frontend-builder /app/packages/frontend/dist ./packages/trueforge/dist/_frontend - -WORKDIR /app/packages/trueforge - -RUN groupadd --gid 10001 trueforge \ - && useradd --uid 10001 --gid trueforge trueforge - -EXPOSE 8790 - -# Launch-only (matches root `pnpm start` / `standalone:start`). Image already contains dist. -USER 10001:10001 -CMD ["node", "dist/main.js"] diff --git a/RELEASING.md b/RELEASING.md index b52b2b6ae..cc58c1e2f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,27 +3,27 @@ This repo ships npm packages, a production container image, a Helm chart, a sandbox image, and optional from-source **dev** images. -| What | Trigger | Workflow | -| ----------------------------------- | ---------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| npm packages | Push to `main` (Changesets) | [`release.yml`](.github/workflows/release.yml) | -| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) | -| Prod image + chart-release PR | After `@truefoundry/trueforge` npm publish (reusable workflow), or manual dispatch | [`build-and-prepare-chart-release.yml`](.github/workflows/build-and-prepare-chart-release.yml) | -| Chart tag, GitHub Release, OCI push | Merge of `release-chart/trueforge`, or push/dispatch of `charts/trueforge@*` | [`release-chart.yml`](.github/workflows/release-chart.yml) | -| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | -| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | +| What | Trigger | Workflow | +| ----------------------- | ---------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| npm packages | Push to `main` (Changesets) | [`release.yml`](.github/workflows/release.yml) | +| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) | +| Prod image + Helm chart | Same `mode=publish` run as npm (parallel, after pack/smoke; does not wait for npm) | [`release-image-chart.yml`](.github/workflows/release-image-chart.yml) | +| Chart OCI retry | Push or dispatch of `charts/trueforge@*` | [`release-chart.yml`](.github/workflows/release-chart.yml) | +| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | +| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | ## Versioning | Artifact | Identity | | ---------------------------- | --------------------------------------------------------------------------------------------------------- | -| npm `@truefoundry/trueforge` | SemVer `X.Y.Z` — source of truth for app bits | +| npm `@truefoundry/trueforge` | SemVer `X.Y.Z` - source of truth for published packages | | PyPI `trueforge-sdk` | Same SemVer as `@truefoundry/trueforge-sdk` (mirrored into `pyproject.toml` on Version Packages) | -| Chart `appVersion` | A **published** npm version | -| Prod image | Root [`Dockerfile`](Dockerfile): `npm install @truefoundry/trueforge@$APP_VERSION` | +| Chart `appVersion` | `packages/trueforge/package.json` version at the build commit | +| Prod image | Root [`Dockerfile`](Dockerfile): from-source workspace build | | Prod image tag | `{appVersion}-{shortSha}` (shortSha of the build commit) | | Chart `version` | Independent SemVer; git tag `charts/trueforge@A.B.C` must match | | Sandbox image | [`sandbox.Dockerfile`](packages/trueforge-core/scripts/sandbox/sandbox.Dockerfile); tag = full commit SHA | -| Dev image | [`Dockerfile.dev`](Dockerfile.dev); tag = full commit SHA | +| Dev image | Same [`Dockerfile`](Dockerfile); tag = full commit SHA | Install a published chart: @@ -39,7 +39,7 @@ helm install trueforge oci://tfy.jfrog.io/tfy-helm/trueforge --version `). SDK regen already adds @@ -58,14 +58,10 @@ No `v*` tag publish. [`release.yml`](.github/workflows/release.yml) does both ve `scripts/version.mjs` mirrors that version into `python/trueforge_sdk` and regenerates both SDKs. Review and merge. 3. With no pending changesets, **pack** (build/test) and **Windows npx smoke** - run in parallel, then **npm publish** and **PyPI publish** run in parallel - via trusted publishing (OIDC; no `NPM_TOKEN` / `PYPI_TOKEN`). PyPI skips when - that `pyproject.toml` version is already published. -4. If `@truefoundry/trueforge` was published, **Release** calls **Build and - prepare chart release** as a reusable workflow on the same commit (so a - newer `main` push cannot change the Dockerfile / shortSha). GitHub's - `workflow_dispatch` API only accepts a branch or tag name, not a SHA. -5. Pin dependents to exact versions during early `0.x`. + run in parallel, then **npm publish**, **PyPI publish**, and **image+Helm + release** run in parallel. PyPI skips when that `pyproject.toml` version is + already published. The image lane does not wait for npm. +4. Pin dependents to exact versions during early `0.x`. `workflow_dispatch` on **Release** re-runs the same workflow. @@ -89,8 +85,9 @@ Each public **npm** package must list this repo + workflow as a trusted publishe - Workflow: `release.yml` (exact filename) - No GitHub Environment name -Do not set `NPM_TOKEN` / `_authToken` on the npm publish job — that disables OIDC. -Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`). +Do not set `NPM_TOKEN` / `_authToken` on the npm publish job - that disables OIDC. +Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`) for +registries. The image workflow uses Artifactory credentials. Publish attaches npm provenance (`NPM_CONFIG_PROVENANCE` on the publish job, and `publishConfig.provenance: true` on every public package). That publicly attests @@ -114,92 +111,87 @@ pnpm clean && pnpm build && pnpm standalone:start ## Troubleshooting -- **No Version Packages PR** — no `.changeset/*.md` on `main`. Add one, or re-run **Release**. -- **Publish wants a tag** — RCs need the `rc` dist-tag (set automatically while `pre.json` exists). -- **403** — version already on npm, or trusted-publisher config mismatch. -- **OIDC fail** — pnpm >= 11.0.7; remove registry `_authToken`. -- **Missing `dist/_frontend/index.html`** — root `pnpm build` must build `frontend` first. -- **SDK not regenerated on Version PR** — only when `@truefoundry/trueforge-sdk` version moved +- **No Version Packages PR** - no `.changeset/*.md` on `main`. Add one, or re-run **Release**. +- **Publish wants a tag** - RCs need the `rc` dist-tag (set automatically while `pre.json` exists). +- **403** - version already on npm, or trusted-publisher config mismatch. +- **OIDC fail** - pnpm >= 11.0.7; remove registry `_authToken`. +- **Missing `dist/_frontend/index.html`** - root `pnpm build` must build `frontend` first. +- **SDK not regenerated on Version PR** - only when `@truefoundry/trueforge-sdk` version moved (`scripts/version.mjs`; needs Docker). That path also mirrors the version into `python/trueforge_sdk`. -- **PyPI 403 / invalid-publisher** — register a trusted publisher for `trueforge-sdk` bound to +- **PyPI 403 / invalid-publisher** - register a trusted publisher for `trueforge-sdk` bound to `release.yml` (and create the project if it does not exist yet). -- **Prod image missing after npm publish** — dispatch the chart workflow on a - **branch or tag** (not a SHA): `gh workflow run build-and-prepare-chart-release.yml --ref main -f app_version=X.Y.Z -f update_app_version=true`. +- **Prod image / chart missing after package publish** - dispatch the image workflow on a + **branch or tag** (not a SHA): `gh workflow run release-image-chart.yml --ref main`. +- **Chart OCI retry** - `gh workflow run release-chart.yml -f tag=charts/trueforge@X.Y.Z`. --- # Image and Helm chart ```text -npm publish @truefoundry/trueforge@X.Y.Z - → call build-and-prepare-chart-release (same commit as publish) - → build Dockerfile (APP_VERSION=X.Y.Z) → push X.Y.Z- - → open/update PR on branch release-chart/trueforge - → merge PR → tag + GH Release + OCI push (release-chart.yml) - -manual rebuild (same or other app version) - → workflow_dispatch build-and-prepare-chart-release - → same PR path - -chart-only - → human PR bumps Chart.yaml version - → human tags charts/trueforge@A.B.C (or gh release create) +pack + Windows npx smoke + → in parallel: npm publish | PyPI publish | release-image-chart + → build from-source Dockerfile → push X.Y.Z- + → bump Chart.yaml / values.yaml, lint, package + → commit to main, tag charts/trueforge@A.B.C, GH Release, OCI push + +manual rebuild + → workflow_dispatch release-image-chart + → same publish path + +chart OCI retry + → human tags charts/trueforge@A.B.C (or gh workflow run release-chart.yml) → release-chart.yml publishes OCI (no image rebuild) ``` -## Dockerfiles +## Dockerfile -| File | Role | -| ---------------------------------- | --------------------------------------------------------------------------------------- | -| [`Dockerfile`](Dockerfile) | Prod/OSS. `ARG APP_VERSION` → `npm install @truefoundry/trueforge@$APP_VERSION` | -| [`Dockerfile.dev`](Dockerfile.dev) | From-source. Used by [`docker-compose.yml`](docker-compose.yml) and **Build dev image** | +| File | Role | +| -------------------------- | ------------------------------------------------------------------------------------------------ | +| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build dev image** | -Prod fails if that npm version is missing (no workspace fallback), so `appVersion` stays honest -even when `main` has moved on. +The image is the workspace at `GITHUB_SHA`. Chart `appVersion` is that commit's +`packages/trueforge/package.json` version (not a requirement that npm already +advertises the package). -## Build and prepare chart release +## Release image and Helm chart -[`build-and-prepare-chart-release.yml`](.github/workflows/build-and-prepare-chart-release.yml) +[`release-image-chart.yml`](.github/workflows/release-image-chart.yml) (`workflow_call` from **Release**, or manual `workflow_dispatch`): -| Input | Default | Meaning | -| -------------------- | ------------------------- | -------------------------------------------------------------------- | -| `app_version` | `Chart.yaml` `appVersion` | npm version to install into the image | -| `update_app_version` | `false` | Also write that version into `Chart.yaml` `appVersion` on the bot PR | +| Input | Default | Meaning | +| ------------- | --------------------------------- | -------------------------------------------- | +| `app_version` | `packages/trueforge/package.json` | Version used in the image tag and appVersion | -Always: build/push `{appVersion}-{shortSha}`, patch-bump chart `version`, set `image.tag`, -open/update one PR on `release-chart/trueforge`. +Always: build/push `{appVersion}-{shortSha}`, patch-bump chart `version`, set +`appVersion` and `image.tag`, lint, package, push OCI, tag, GitHub Release, and +commit the chart metadata to `main` with `GITHUB_TOKEN` (that push does not +start another workflow). ```bash -gh workflow run build-and-prepare-chart-release.yml -gh workflow run build-and-prepare-chart-release.yml -f app_version=0.1.0 -# after npm publish of a new app version: -gh workflow run build-and-prepare-chart-release.yml \ - -f app_version=0.1.0 -f update_app_version=true +gh workflow run release-image-chart.yml +gh workflow run release-image-chart.yml -f app_version=0.1.0 ``` -You may edit chart SemVer (minor/major) on the PR before merging; the tag follows -`Chart.yaml` `version` at merge time. Each run rebuilds the `release-chart/trueforge` branch -from `main`, but a chart `version` on the branch that outranks the patch bump is carried over, -so a manual bump survives later image rebuilds. Other manual edits on that branch do not — -commit them to `main` instead. +Chart SemVer is chosen by [`scripts/resolve-chart-version.sh`](scripts/resolve-chart-version.sh) +from the current `Chart.yaml` on the build commit. -## Publish Helm chart +## Publish Helm chart (retry) -[`release-chart.yml`](.github/workflows/release-chart.yml) is one job with three entry points: +[`release-chart.yml`](.github/workflows/release-chart.yml) publishes an existing +tagged chart. It does not build an image. -| Trigger | What it does | -| --------------------------------------------------------- | ------------------------------------------------------------------- | -| Merged PR from `release-chart/trueforge` (same repo only) | Create `charts/trueforge@` + GitHub Release, then OCI push | -| Push of tag `charts/trueforge@*` | OCI push only (tag already exists) | -| `workflow_dispatch` with `tag=` | OCI push for an existing tag (retry) | +| Trigger | What it does | +| -------------------------------- | ----------------------------- | +| Push of tag `charts/trueforge@*` | OCI push (tag already exists) | +| `workflow_dispatch` with `tag=` | OCI push for an existing tag | -Only the `release-chart/trueforge` branch auto-tags. Ordinary merges never create chart tags. +`GITHUB_TOKEN` tag pushes from **Release image and Helm chart** do not start this +workflow; that job publishes OCI itself. A human PAT tag push does. -Chart-only example: +Chart-only example (already-bumped `Chart.yaml` on a commit): ```bash -# after merging a PR that bumped Chart.yaml version: git tag charts/trueforge@0.1.0 git push origin charts/trueforge@0.1.0 ``` @@ -213,7 +205,7 @@ gh workflow run build-dev-image.yml --ref main # → tfy.jfrog.io/tfy-images/trueforge: ``` -Patch that SHA into `image.tag`. Secrets via `secretKeyRef` only — never plaintext in git. +Patch that SHA into `image.tag`. Secrets via `secretKeyRef` only - never plaintext in git. Do not use SHA-tagged images as production chart defaults. ## Bundled chart dependencies diff --git a/charts/trueforge/Chart.yaml b/charts/trueforge/Chart.yaml index 950d9cb36..8a2a4594e 100644 --- a/charts/trueforge/Chart.yaml +++ b/charts/trueforge/Chart.yaml @@ -2,8 +2,9 @@ apiVersion: v2 name: trueforge description: TrueForge server (API + UI) served from a single container image. type: application -# version / appVersion are maintained on main (bot chart-release PR or human). -# Publishing is gated by git tag charts/trueforge@ (must match version). +# version / appVersion / image.tag are written on main by the image+Helm release +# workflow (or by a human). Publishing is gated by git tag charts/trueforge@ +# (must match version). version: "0.2.3-rc.0" appVersion: "0.3.0-rc.0" kubeVersion: ">=1.25.0-0" diff --git a/charts/trueforge/README.md b/charts/trueforge/README.md index d80dc2068..238840f0f 100644 --- a/charts/trueforge/README.md +++ b/charts/trueforge/README.md @@ -1,16 +1,16 @@ # trueforge Helm chart Deploys the TrueForge server, a single container image that serves both the API -and the UI. **Production** images install `@truefoundry/trueforge` from npm -(repository-root [`Dockerfile`](../../Dockerfile) with `APP_VERSION`). -**Local smoke / from-source** builds use [`Dockerfile.dev`](../../Dockerfile.dev) -(see [`docker-compose.yml`](../../docker-compose.yml)). +and the UI. Production, Compose smoke, and Railway all build the repository-root +[`Dockerfile`](../../Dockerfile) from the workspace checkout (see +[`docker-compose.yml`](../../docker-compose.yml)). The chart always runs the server in **distributed** mode (`STANDALONE=false`) against Postgres and Redis. -Chart `version` / `appVersion` / `image.tag` are maintained on `main` (chart-release -bot PR or human). Publishing is gated by git tag `charts/trueforge@`. +Chart `version` / `appVersion` / `image.tag` are written on `main` by the +image+Helm release workflow (or a human). Publishing is gated by git tag +`charts/trueforge@`. See [`RELEASING.md`](../../RELEASING.md). ## Dev defaults (read before exposing) diff --git a/charts/trueforge/values.yaml b/charts/trueforge/values.yaml index 5bed10d0c..3eb81f538 100644 --- a/charts/trueforge/values.yaml +++ b/charts/trueforge/values.yaml @@ -38,8 +38,9 @@ global: sizeLimit: 10Mi image: repository: tfy.jfrog.io/tfy-images/trueforge - # Tag defaults to chart appVersion when empty. Prod chart-release PRs set this - # to {appVersion}-{shortSha} for the npm-install image already in the registry. + # Tag defaults to chart appVersion when empty. The image+Helm release workflow + # sets this to {appVersion}-{shortSha} for the from-source image already in + # the registry. tag: "0.3.0-rc.0-346221b" pullPolicy: IfNotPresent imagePullSecrets: [] @@ -116,11 +117,10 @@ controller: enabled: true # Annotations on the controller Deployment object (for example, Argo CD sync waves). deploymentAnnotations: {} - # Container command. Runs the controller entry from the published package inside - # the image (root Dockerfile layout: /app/node_modules/@truefoundry/trueforge). + # Container command. Image WORKDIR is /app/packages/trueforge (from-source Dockerfile). command: - node - - node_modules/@truefoundry/trueforge/dist/controller-main.js + - dist/controller-main.js # Base URL the controller uses to reach the server (SERVER_URL). # Empty → the in-cluster server Service (http(s)://:). serverUrl: "" diff --git a/docker-compose.yml b/docker-compose.yml index df226c641..135ba13ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,8 +33,6 @@ services: server: build: context: . - # From-source image; root Dockerfile is the npm-install prod recipe. - dockerfile: Dockerfile.dev image: truefoundry-server:latest ports: # Host 8791 avoids conflict with host `pnpm dev` on 8790. @@ -86,7 +84,6 @@ services: controller: build: context: . - dockerfile: Dockerfile.dev image: truefoundry-server:latest command: ['node', 'dist/controller-main.js'] environment: diff --git a/docs/quickstart.mdx b/docs/quickstart.mdx index 0a9a99e5b..9fbc2dd42 100644 --- a/docs/quickstart.mdx +++ b/docs/quickstart.mdx @@ -116,7 +116,7 @@ TrueForge runs in [two modes](/introduction#two-ways-to-run-it): **local mode** - Hosted topology on [Railway](https://railway.com) is defined with [Infrastructure as Code](https://docs.railway.com/infrastructure-as-code) in [`.railway/railway.ts`](https://github.com/truefoundry/trueforge/blob/main/.railway/railway.ts): one project with the server (UI + API), Postgres, and Redis. The app sets `RAILWAY_DOCKERFILE_PATH=Dockerfile.dev` so Railway builds the from-source image ([`Dockerfile.dev`](https://github.com/truefoundry/trueforge/blob/main/Dockerfile.dev)), and wires `DATABASE_URL`, `REDIS_URL`, and `PUBLIC_BASE_URL` in that file. + Hosted topology on [Railway](https://railway.com) is defined with [Infrastructure as Code](https://docs.railway.com/infrastructure-as-code) in [`.railway/railway.ts`](https://github.com/truefoundry/trueforge/blob/main/.railway/railway.ts): one project with the server (UI + API), Postgres, and Redis. Railway builds the from-source image ([`Dockerfile`](https://github.com/truefoundry/trueforge/blob/main/Dockerfile)), and wires `DATABASE_URL`, `REDIS_URL`, and `PUBLIC_BASE_URL` in that file. From a clone of this repo: @@ -136,7 +136,7 @@ TrueForge runs in [two modes](/introduction#two-ways-to-run-it): **local mode** a Railway deployment beyond personal use. - `STANDALONE=false` and `HOST=0.0.0.0` are baked into `Dockerfile.dev`. Railway injects `PORT` automatically. + `STANDALONE=false` and `HOST=0.0.0.0` are baked into `Dockerfile`. Railway injects `PORT` automatically. From 0e29845af060a447fd6db36926271fc6d9e9b1c8 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 23 Sep 2026 17:54:43 +0530 Subject: [PATCH 2/7] Keep package publish and Helm release as separate workflows. Rename the package workflow to release-packages.yml, drop the Helm tag-retry job, and start the chart workflow on main with its own Changesets select-mode. Signed-off-by: Raman Tehlan --- .github/workflows/push-sandbox-image.yml | 2 +- .github/workflows/release-chart.yml | 208 ++++++++++++++---- .github/workflows/release-image-chart.yml | 203 ----------------- .../{release.yml => release-packages.yml} | 26 +-- RELEASING.md | 100 ++++----- 5 files changed, 213 insertions(+), 326 deletions(-) delete mode 100644 .github/workflows/release-image-chart.yml rename .github/workflows/{release.yml => release-packages.yml} (90%) diff --git a/.github/workflows/push-sandbox-image.yml b/.github/workflows/push-sandbox-image.yml index 1a5e3e72c..73ce3d134 100644 --- a/.github/workflows/push-sandbox-image.yml +++ b/.github/workflows/push-sandbox-image.yml @@ -101,7 +101,7 @@ jobs: commit-message: 'chore: update sandbox image ${{ github.sha }}' title: 'chore: update sandbox image ${{ github.sha }}' body: | - Pushed `${{ steps.image.outputs.uri }}`. Merge so `release.yml` can publish the pin. + Pushed `${{ steps.image.outputs.uri }}`. Merge so `release-packages.yml` can publish the pin. Existing tenants stay on `build_metadata`. add-paths: | packages/trueforge-core/src/core/sandbox/sandboxImage.json diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index 298fad421..bd9cf7ccd 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -1,27 +1,124 @@ -name: Publish Helm chart - -# Retry / human-tag path. The prod lane publishes from release-image-chart.yml. -# Human charts/trueforge@* tags and workflow_dispatch reuse this job. +name: Build server image and publish Helm chart + +# Independent of release-packages.yml. Same Changesets select-mode: +# pending changesets skip this lane; unpublished versions build the +# from-source image, then package and publish the Helm chart. +# +# workflow_dispatch is for manual rebuilds. Do not use `gh workflow run --ref +# $GITHUB_SHA`: the dispatch API only accepts a branch or tag name (HTTP 422). on: push: - tags: - - 'charts/trueforge@*' + branches: [main] workflow_dispatch: inputs: - tag: - description: Existing tag to publish (e.g. charts/trueforge@0.1.0) - required: true + app_version: + description: App version for the image tag (empty = packages/trueforge/package.json) + required: false type: string concurrency: - group: publish-helm-chart + group: release-image-and-chart cancel-in-progress: false permissions: {} jobs: - release: - name: Publish chart + select-mode: + name: Select mode + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + mode: ${{ steps.select-mode.outputs.mode }} + permissions: + contents: read + steps: + - name: Check out repo + uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: pnpm/action-setup@v6 + + - name: Set up Node.js + uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + - name: Select Changesets mode + id: select-mode + uses: changesets/action/select-mode@v2 + + resolve: + name: Resolve versions + if: github.event_name == 'workflow_dispatch' || needs.select-mode.outputs.mode == 'publish' + needs: select-mode + runs-on: ubuntu-latest + outputs: + app_version: ${{ steps.resolve.outputs.app_version }} + image_tag: ${{ steps.resolve.outputs.image_tag }} + short_sha: ${{ steps.resolve.outputs.short_sha }} + permissions: + contents: read + steps: + - name: Check out + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Resolve app version and image tag + id: resolve + env: + INPUT_APP_VERSION: ${{ inputs.app_version }} + run: | + set -euo pipefail + PKG_VERSION=$(node -p "require('./packages/trueforge/package.json').version") + if [[ -n "${INPUT_APP_VERSION}" ]]; then + APP_VERSION="$INPUT_APP_VERSION" + else + APP_VERSION="$PKG_VERSION" + fi + if [[ -z "$APP_VERSION" || "$APP_VERSION" == "undefined" ]]; then + echo "Could not resolve APP_VERSION" >&2 + exit 1 + fi + if ! printf '%s' "$APP_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$'; then + echo "APP_VERSION '$APP_VERSION' is not a semver-like version" >&2 + exit 1 + fi + SHORT_SHA="${GITHUB_SHA:0:7}" + IMAGE_TAG="${APP_VERSION}-${SHORT_SHA}" + echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" + echo "Resolved APP_VERSION=$APP_VERSION IMAGE_TAG=$IMAGE_TAG (package.json=$PKG_VERSION)" + + build: + name: Build and push server image + needs: [resolve] + uses: truefoundry/github-workflows-public/.github/workflows/build.yml@main + permissions: + id-token: write + contents: read + with: + artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} + artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }} + image_artifact_name: trueforge + image_tag: ${{ needs.resolve.outputs.image_tag }} + image_context: . + dockerfile_path: Dockerfile + platforms: linux/amd64 + enable_jfrog: true + enable_public_ecr: false + secrets: + artifactory_username: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} + artifactory_password: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} + + publish-chart: + name: Publish Helm chart + needs: [resolve, build] runs-on: ubuntu-latest permissions: contents: write @@ -30,10 +127,9 @@ jobs: HELM_REGISTRY_URL: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} HELM_CHART_REPOSITORY: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_HELM_REPOSITORY }} steps: - - name: Check out chart commit + - name: Check out uses: actions/checkout@v7 with: - ref: ${{ inputs.tag || github.ref }} fetch-depth: 0 - name: Install yq @@ -46,44 +142,67 @@ jobs: with: version: v3.16.4 - - name: Resolve chart tag + - name: Update chart metadata + id: chart env: - INPUT_TAG: ${{ inputs.tag }} + IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + APP_VERSION: ${{ needs.resolve.outputs.app_version }} run: | set -euo pipefail + CURRENT=$(yq -r '.version' charts/trueforge/Chart.yaml) + VERSION=$(bash scripts/resolve-chart-version.sh "$CURRENT" "$APP_VERSION") + export VERSION IMAGE_TAG APP_VERSION + yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml + yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml + yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" + echo "Chart version=$VERSION appVersion=$APP_VERSION image.tag=$IMAGE_TAG" - VERSION=$(yq -r '.version' charts/trueforge/Chart.yaml) - APP_VERSION=$(yq -r '.appVersion' charts/trueforge/Chart.yaml) - IMAGE_TAG=$(yq -r '.image.tag' charts/trueforge/values.yaml) - - if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]]; then - echo "Invalid chart version: $VERSION" >&2 - exit 1 - fi + - name: Build chart dependencies + run: helm dependency build "$CHART_DIR" - if [[ -n "${INPUT_TAG:-}" ]]; then - TAG="$INPUT_TAG" - else - TAG="${GITHUB_REF#refs/tags/}" - fi + - name: Lint chart + run: helm lint "$CHART_DIR" --values charts/trueforge/ci/lint-values.yaml - case "$TAG" in - charts/trueforge@*) EXPECTED="${TAG#charts/trueforge@}" ;; - *) echo "Invalid chart tag: $TAG" >&2; exit 1 ;; - esac + - name: Commit chart metadata, tag, and GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.chart.outputs.version }} + APP_VERSION: ${{ steps.chart.outputs.app_version }} + IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + run: | + set -euo pipefail + TAG="charts/trueforge@${VERSION}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - ACTUAL=$(yq -r '.version' "$CHART_DIR/Chart.yaml") - if [[ "$ACTUAL" != "$EXPECTED" ]]; then - echo "Chart.yaml version '$ACTUAL' does not match tag version '$EXPECTED'" >&2 - exit 1 + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "Tag $TAG already exists; skipping commit and GitHub Release" + else + git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml + if git diff --staged --quiet; then + echo "Chart metadata already matches $VERSION; skipping commit" >&2 + else + git commit -m "release(chart): ${VERSION}" + git push origin HEAD:main + fi + + git tag -a "$TAG" -m "Helm chart ${VERSION} (appVersion ${APP_VERSION}, image ${IMAGE_TAG})" + git push origin "$TAG" + + NOTES_FILE="$(mktemp)" + { + echo "Chart version: ${VERSION}" + echo "appVersion: ${APP_VERSION}" + echo "image.tag: ${IMAGE_TAG}" + } >"$NOTES_FILE" + gh release create "$TAG" \ + --title "$TAG" \ + --notes-file "$NOTES_FILE" + rm -f "$NOTES_FILE" fi - echo "VERSION=$VERSION" >> "$GITHUB_ENV" - echo "Publishing chart version=$ACTUAL appVersion=$APP_VERSION image.tag=$IMAGE_TAG" - - - name: Build chart dependencies - run: helm dependency build "$CHART_DIR" - - name: Helm registry login env: HELM_REGISTRY_USERNAME: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} @@ -93,6 +212,8 @@ jobs: | helm registry login -u "$HELM_REGISTRY_USERNAME" --password-stdin "$HELM_REGISTRY_URL" - name: Package and push chart + env: + VERSION: ${{ steps.chart.outputs.version }} run: | set -euo pipefail CHART_NAME=$(yq -r '.name' "$CHART_DIR/Chart.yaml") @@ -104,4 +225,5 @@ jobs: { echo "### Helm chart" echo "Pushed \`oci://${HELM_CHART_REPOSITORY}/${CHART_NAME}:${VERSION}\`" + echo "Image tag \`${{ needs.resolve.outputs.image_tag }}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release-image-chart.yml b/.github/workflows/release-image-chart.yml deleted file mode 100644 index 6f6b4fcf0..000000000 --- a/.github/workflows/release-image-chart.yml +++ /dev/null @@ -1,203 +0,0 @@ -name: Release image and Helm chart - -# Prod lane: build the from-source Dockerfile, push {appVersion}-{shortSha}, -# then package and publish the Helm chart (tag + GitHub Release + OCI). -# -# Called from release.yml after pack/smoke (same commit; does not wait for npm). -# workflow_dispatch is for manual rebuilds. Do not use `gh workflow run --ref -# $GITHUB_SHA`: the dispatch API only accepts a branch or tag name (HTTP 422). -on: - workflow_call: - inputs: - app_version: - description: App version for the image tag (empty = packages/trueforge/package.json) - required: false - type: string - workflow_dispatch: - inputs: - app_version: - description: App version for the image tag (empty = packages/trueforge/package.json) - required: false - type: string - -concurrency: - group: release-image-and-chart - cancel-in-progress: false - -permissions: {} - -jobs: - resolve: - name: Resolve versions - runs-on: ubuntu-latest - outputs: - app_version: ${{ steps.resolve.outputs.app_version }} - image_tag: ${{ steps.resolve.outputs.image_tag }} - short_sha: ${{ steps.resolve.outputs.short_sha }} - permissions: - contents: read - steps: - - name: Check out - uses: actions/checkout@v7 - with: - persist-credentials: false - - - name: Resolve app version and image tag - id: resolve - env: - INPUT_APP_VERSION: ${{ inputs.app_version }} - run: | - set -euo pipefail - PKG_VERSION=$(node -p "require('./packages/trueforge/package.json').version") - if [[ -n "${INPUT_APP_VERSION}" ]]; then - APP_VERSION="$INPUT_APP_VERSION" - else - APP_VERSION="$PKG_VERSION" - fi - if [[ -z "$APP_VERSION" || "$APP_VERSION" == "undefined" ]]; then - echo "Could not resolve APP_VERSION" >&2 - exit 1 - fi - if ! printf '%s' "$APP_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$'; then - echo "APP_VERSION '$APP_VERSION' is not a semver-like version" >&2 - exit 1 - fi - SHORT_SHA="${GITHUB_SHA:0:7}" - IMAGE_TAG="${APP_VERSION}-${SHORT_SHA}" - echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" - echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" - echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" - echo "Resolved APP_VERSION=$APP_VERSION IMAGE_TAG=$IMAGE_TAG (package.json=$PKG_VERSION)" - - build: - name: Build and push server image - needs: [resolve] - uses: truefoundry/github-workflows-public/.github/workflows/build.yml@main - permissions: - id-token: write - contents: read - with: - artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} - artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }} - image_artifact_name: trueforge - image_tag: ${{ needs.resolve.outputs.image_tag }} - image_context: . - dockerfile_path: Dockerfile - platforms: linux/amd64 - enable_jfrog: true - enable_public_ecr: false - secrets: - artifactory_username: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} - artifactory_password: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} - - publish-chart: - name: Publish Helm chart - needs: [resolve, build] - runs-on: ubuntu-latest - permissions: - contents: write - env: - CHART_DIR: charts/trueforge - HELM_REGISTRY_URL: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} - HELM_CHART_REPOSITORY: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_HELM_REPOSITORY }} - steps: - - name: Check out - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Install yq - run: | - sudo wget -q https://github.com/mikefarah/yq/releases/download/v4.53.3/yq_linux_amd64 -O /usr/bin/yq - sudo chmod +x /usr/bin/yq - - - name: Install Helm - uses: azure/setup-helm@v5 - with: - version: v3.16.4 - - - name: Update chart metadata - id: chart - env: - IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} - APP_VERSION: ${{ needs.resolve.outputs.app_version }} - run: | - set -euo pipefail - CURRENT=$(yq -r '.version' charts/trueforge/Chart.yaml) - VERSION=$(bash scripts/resolve-chart-version.sh "$CURRENT" "$APP_VERSION") - export VERSION IMAGE_TAG APP_VERSION - yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml - yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml - yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" - echo "Chart version=$VERSION appVersion=$APP_VERSION image.tag=$IMAGE_TAG" - - - name: Build chart dependencies - run: helm dependency build "$CHART_DIR" - - - name: Lint chart - run: helm lint "$CHART_DIR" --values charts/trueforge/ci/lint-values.yaml - - - name: Commit chart metadata, tag, and GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ steps.chart.outputs.version }} - APP_VERSION: ${{ steps.chart.outputs.app_version }} - IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} - run: | - set -euo pipefail - TAG="charts/trueforge@${VERSION}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then - echo "Tag $TAG already exists; skipping commit and GitHub Release" - else - git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml - if git diff --staged --quiet; then - echo "Chart metadata already matches $VERSION; skipping commit" >&2 - else - git commit -m "release(chart): ${VERSION}" - git push origin HEAD:main - fi - - git tag -a "$TAG" -m "Helm chart ${VERSION} (appVersion ${APP_VERSION}, image ${IMAGE_TAG})" - git push origin "$TAG" - - NOTES_FILE="$(mktemp)" - { - echo "Chart version: ${VERSION}" - echo "appVersion: ${APP_VERSION}" - echo "image.tag: ${IMAGE_TAG}" - } >"$NOTES_FILE" - gh release create "$TAG" \ - --title "$TAG" \ - --notes-file "$NOTES_FILE" - rm -f "$NOTES_FILE" - fi - - - name: Helm registry login - env: - HELM_REGISTRY_USERNAME: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} - HELM_REGISTRY_PASSWORD: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} - run: | - printf '%s' "$HELM_REGISTRY_PASSWORD" \ - | helm registry login -u "$HELM_REGISTRY_USERNAME" --password-stdin "$HELM_REGISTRY_URL" - - - name: Package and push chart - env: - VERSION: ${{ steps.chart.outputs.version }} - run: | - set -euo pipefail - CHART_NAME=$(yq -r '.name' "$CHART_DIR/Chart.yaml") - mkdir -p dist - helm package "$CHART_DIR" --destination dist - PACKAGE="dist/${CHART_NAME}-${VERSION}.tgz" - echo "Pushing $PACKAGE to oci://$HELM_CHART_REPOSITORY" - helm push "$PACKAGE" "oci://$HELM_CHART_REPOSITORY" - { - echo "### Helm chart" - echo "Pushed \`oci://${HELM_CHART_REPOSITORY}/${CHART_NAME}:${VERSION}\`" - echo "Image tag \`${{ needs.resolve.outputs.image_tag }}\`" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release-packages.yml similarity index 90% rename from .github/workflows/release.yml rename to .github/workflows/release-packages.yml index a2e332911..b34fac08f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release-packages.yml @@ -1,22 +1,22 @@ # npm publish for the four public packages, plus PyPI for python/trueforge_sdk. # -# npm and PyPI trusted publishers are bound to `release.yml`. -# Do not rename this file without updating every package on npmjs.com and PyPI. +# npm and PyPI trusted publishers are bound to this filename +# (`release-packages.yml`). Update every package on npmjs.com and PyPI if you +# rename it. They were previously bound to `release.yml`. # # On every push to main, changesets/action/select-mode chooses: # - pending `.changeset/*.md` → version job opens/updates the Version Packages PR # (install only; that PR is gated by CI.yml). When @truefoundry/trueforge-sdk # moves, scripts/version.mjs mirrors that version into python/trueforge_sdk and rebakes Fern. # - none pending + unpublished versions → pack (build/test) + Windows npx smoke, -# then npm publish, PyPI publish, and image+Helm release in parallel +# then npm publish and PyPI publish in parallel # Merge the Version Packages PR to publish. There is no separate publish Action -# and no `v*` tag trigger. +# and no `v*` tag trigger. Image and Helm are a separate workflow. # # Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc` # disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN / -# NODE_AUTH_TOKEN. id-token is granted only on the npm/PyPI publish jobs (the -# image workflow requests its own). -name: Release +# NODE_AUTH_TOKEN. id-token is granted only on the publish jobs. +name: Version or publish packages on: push: @@ -266,15 +266,3 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: python/trueforge_sdk/dist - - # Same commit as this publish run (GITHUB_SHA). Does not wait for npm. - # A workflow_dispatch --ref SHA is rejected: the dispatch API only accepts - # a branch or tag name (HTTP 422). - release-image-and-chart: - name: Release image and Helm chart - needs: [pack, windows-npx-smoke] - uses: ./.github/workflows/release-image-chart.yml - permissions: - contents: write - id-token: write - secrets: inherit diff --git a/RELEASING.md b/RELEASING.md index cc58c1e2f..8304a8c0e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,14 +3,14 @@ This repo ships npm packages, a production container image, a Helm chart, a sandbox image, and optional from-source **dev** images. -| What | Trigger | Workflow | -| ----------------------- | ---------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| npm packages | Push to `main` (Changesets) | [`release.yml`](.github/workflows/release.yml) | -| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) | -| Prod image + Helm chart | Same `mode=publish` run as npm (parallel, after pack/smoke; does not wait for npm) | [`release-image-chart.yml`](.github/workflows/release-image-chart.yml) | -| Chart OCI retry | Push or dispatch of `charts/trueforge@*` | [`release-chart.yml`](.github/workflows/release-chart.yml) | -| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | -| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | +| What | Trigger | Workflow | +| ----------------------- | ---------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| npm packages | Push to `main` (Changesets) | [`release-packages.yml`](.github/workflows/release-packages.yml) | +| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release-packages.yml`](.github/workflows/release-packages.yml) | +| Prod image + Helm chart | Push to `main` (Changesets publish mode, parallel with packages) or dispatch | [`release-chart.yml`](.github/workflows/release-chart.yml) | +| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | +| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | +| PR checks | Pull request / merge group | [`ci.yml`](.github/workflows/ci.yml) | ## Versioning @@ -47,8 +47,8 @@ helm install trueforge oci://tfy.jfrog.io/tfy-helm/trueforge --version `). SDK regen already adds @@ -58,12 +58,12 @@ package publish (`select-mode` → `version` \| `pack` → npm + PyPI + image/He `scripts/version.mjs` mirrors that version into `python/trueforge_sdk` and regenerates both SDKs. Review and merge. 3. With no pending changesets, **pack** (build/test) and **Windows npx smoke** - run in parallel, then **npm publish**, **PyPI publish**, and **image+Helm - release** run in parallel. PyPI skips when that `pyproject.toml` version is - already published. The image lane does not wait for npm. + run in parallel, then **npm publish** and **PyPI publish** run in parallel. + PyPI skips when that `pyproject.toml` version is already published. + Image and Helm are a separate workflow on the same push (see below). 4. Pin dependents to exact versions during early `0.x`. -`workflow_dispatch` on **Release** re-runs the same workflow. +`workflow_dispatch` on **Version or publish packages** re-runs the same workflow. ## Prerelease mode @@ -82,12 +82,14 @@ Repo-wide via `.changeset/pre.json` (absent = publish to `latest`): Each public **npm** package must list this repo + workflow as a trusted publisher on npmjs.com: - Repository: `truefoundry/trueforge` -- Workflow: `release.yml` (exact filename) +- Workflow: `release-packages.yml` (exact filename) - No GitHub Environment name +This filename used to be `release.yml`. Update every package on npmjs.com (and +PyPI below) before the first publish from this workflow, or OIDC will 403. + Do not set `NPM_TOKEN` / `_authToken` on the npm publish job - that disables OIDC. -Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`) for -registries. The image workflow uses Artifactory credentials. +Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`). Publish attaches npm provenance (`NPM_CONFIG_PROVENANCE` on the publish job, and `publishConfig.provenance: true` on every public package). That publicly attests @@ -96,7 +98,7 @@ the source repo and commit on npmjs.com. **PyPI** `trueforge-sdk` uses the same workflow file via a trusted publisher: - Repository: `truefoundry/trueforge` -- Workflow: `release.yml` (exact filename) +- Workflow: `release-packages.yml` (exact filename) - No Environment name (unless you add one to the job and mirror it on PyPI) - Create the project once on PyPI (or publish the first version), then add the pending/trusted publisher before the first OIDC upload succeeds. @@ -111,53 +113,51 @@ pnpm clean && pnpm build && pnpm standalone:start ## Troubleshooting -- **No Version Packages PR** - no `.changeset/*.md` on `main`. Add one, or re-run **Release**. +- **No Version Packages PR** - no `.changeset/*.md` on `main`. Add one, or re-run **Version or publish packages**. - **Publish wants a tag** - RCs need the `rc` dist-tag (set automatically while `pre.json` exists). -- **403** - version already on npm, or trusted-publisher config mismatch. +- **403** - version already on npm, or trusted-publisher config mismatch (filename must be `release-packages.yml`). - **OIDC fail** - pnpm >= 11.0.7; remove registry `_authToken`. - **Missing `dist/_frontend/index.html`** - root `pnpm build` must build `frontend` first. - **SDK not regenerated on Version PR** - only when `@truefoundry/trueforge-sdk` version moved (`scripts/version.mjs`; needs Docker). That path also mirrors the version into `python/trueforge_sdk`. - **PyPI 403 / invalid-publisher** - register a trusted publisher for `trueforge-sdk` bound to - `release.yml` (and create the project if it does not exist yet). -- **Prod image / chart missing after package publish** - dispatch the image workflow on a - **branch or tag** (not a SHA): `gh workflow run release-image-chart.yml --ref main`. -- **Chart OCI retry** - `gh workflow run release-chart.yml -f tag=charts/trueforge@X.Y.Z`. + `release-packages.yml` (and create the project if it does not exist yet). +- **Prod image / chart missing after package publish** - dispatch the chart workflow on a + **branch or tag** (not a SHA): `gh workflow run release-chart.yml --ref main`. --- # Image and Helm chart ```text -pack + Windows npx smoke - → in parallel: npm publish | PyPI publish | release-image-chart +push to main (no pending changesets, unpublished versions) + → release-packages.yml: pack + smoke → npm | PyPI + → release-chart.yml (parallel, independent): → build from-source Dockerfile → push X.Y.Z- → bump Chart.yaml / values.yaml, lint, package → commit to main, tag charts/trueforge@A.B.C, GH Release, OCI push manual rebuild - → workflow_dispatch release-image-chart - → same publish path - -chart OCI retry - → human tags charts/trueforge@A.B.C (or gh workflow run release-chart.yml) - → release-chart.yml publishes OCI (no image rebuild) + → workflow_dispatch release-chart.yml + → same image/Helm path ``` ## Dockerfile -| File | Role | -| -------------------------- | ------------------------------------------------------------------------------------------------ | -| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build dev image** | +| File | Role | +| -------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build SHA-tagged from-source image** | The image is the workspace at `GITHUB_SHA`. Chart `appVersion` is that commit's `packages/trueforge/package.json` version (not a requirement that npm already advertises the package). -## Release image and Helm chart +## Build server image and publish Helm chart -[`release-image-chart.yml`](.github/workflows/release-image-chart.yml) -(`workflow_call` from **Release**, or manual `workflow_dispatch`): +[`release-chart.yml`](.github/workflows/release-chart.yml) +(push to `main`, or manual `workflow_dispatch`). Same Changesets select-mode as +packages: pending changesets skip the image/Helm jobs. Manual dispatch always +builds. | Input | Default | Meaning | | ------------- | --------------------------------- | -------------------------------------------- | @@ -169,33 +169,13 @@ commit the chart metadata to `main` with `GITHUB_TOKEN` (that push does not start another workflow). ```bash -gh workflow run release-image-chart.yml -gh workflow run release-image-chart.yml -f app_version=0.1.0 +gh workflow run release-chart.yml +gh workflow run release-chart.yml -f app_version=0.1.0 ``` Chart SemVer is chosen by [`scripts/resolve-chart-version.sh`](scripts/resolve-chart-version.sh) from the current `Chart.yaml` on the build commit. -## Publish Helm chart (retry) - -[`release-chart.yml`](.github/workflows/release-chart.yml) publishes an existing -tagged chart. It does not build an image. - -| Trigger | What it does | -| -------------------------------- | ----------------------------- | -| Push of tag `charts/trueforge@*` | OCI push (tag already exists) | -| `workflow_dispatch` with `tag=` | OCI push for an existing tag | - -`GITHUB_TOKEN` tag pushes from **Release image and Helm chart** do not start this -workflow; that job publishes OCI itself. A human PAT tag push does. - -Chart-only example (already-bumped `Chart.yaml` on a commit): - -```bash -git tag charts/trueforge@0.1.0 -git push origin charts/trueforge@0.1.0 -``` - ## Dev / floating main External deploy repo owns `truefoundry.yaml` (`git-helm-repo` @ `main`). Build a from-source image: From 9eda890ac93320fa082c7f7601ecfbbf432f6509 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 23 Sep 2026 19:08:49 +0530 Subject: [PATCH 3/7] Trigger image and Helm release from a chart/v* tag. The package workflow now pushes chart/v in parallel with npm and PyPI, and release-chart.yml builds the image and publishes the chart from that tag. Drops the chart-release bot PR and merge gate. Signed-off-by: Raman Tehlan --- .github/workflows/release-chart.yml | 102 ++++++-------------- .github/workflows/release-packages.yml | 47 ++++++++- RELEASING.md | 85 ++++++++-------- charts/trueforge/Chart.yaml | 5 +- charts/trueforge/README.md | 7 +- charts/trueforge/values.yaml | 6 +- scripts/resolve-chart-version.sh | 99 ++++++++++--------- tests/scripts/resolve-chart-version.test.sh | 22 +++-- 8 files changed, 195 insertions(+), 178 deletions(-) diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index bd9cf7ccd..1f256cc69 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -1,14 +1,14 @@ name: Build server image and publish Helm chart -# Independent of release-packages.yml. Same Changesets select-mode: -# pending changesets skip this lane; unpublished versions build the -# from-source image, then package and publish the Helm chart. +# Starts when release-packages.yml pushes chart/v (App token). +# Builds the from-source image, then packages and publishes the Helm chart. # # workflow_dispatch is for manual rebuilds. Do not use `gh workflow run --ref # $GITHUB_SHA`: the dispatch API only accepts a branch or tag name (HTTP 422). on: push: - branches: [main] + tags: + - 'chart/v[0-9]*' workflow_dispatch: inputs: app_version: @@ -23,38 +23,8 @@ concurrency: permissions: {} jobs: - select-mode: - name: Select mode - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - mode: ${{ steps.select-mode.outputs.mode }} - permissions: - contents: read - steps: - - name: Check out repo - uses: actions/checkout@v7 - with: - persist-credentials: false - - - uses: pnpm/action-setup@v6 - - - name: Set up Node.js - uses: actions/setup-node@v7 - with: - node-version: 24 - cache: pnpm - - - run: pnpm install --frozen-lockfile - - - name: Select Changesets mode - id: select-mode - uses: changesets/action/select-mode@v2 - resolve: name: Resolve versions - if: github.event_name == 'workflow_dispatch' || needs.select-mode.outputs.mode == 'publish' - needs: select-mode runs-on: ubuntu-latest outputs: app_version: ${{ steps.resolve.outputs.app_version }} @@ -72,10 +42,19 @@ jobs: id: resolve env: INPUT_APP_VERSION: ${{ inputs.app_version }} + EVENT_NAME: ${{ github.event_name }} + REF_NAME: ${{ github.ref_name }} run: | set -euo pipefail PKG_VERSION=$(node -p "require('./packages/trueforge/package.json').version") - if [[ -n "${INPUT_APP_VERSION}" ]]; then + if [[ "$EVENT_NAME" == "push" && "$REF_NAME" == chart/v* ]]; then + TAG_VERSION="${REF_NAME#chart/v}" + if [[ "$TAG_VERSION" != "$PKG_VERSION" ]]; then + echo "Tag v$TAG_VERSION does not match package.json $PKG_VERSION" >&2 + exit 1 + fi + APP_VERSION="$PKG_VERSION" + elif [[ -n "${INPUT_APP_VERSION}" ]]; then APP_VERSION="$INPUT_APP_VERSION" else APP_VERSION="$PKG_VERSION" @@ -165,44 +144,6 @@ jobs: - name: Lint chart run: helm lint "$CHART_DIR" --values charts/trueforge/ci/lint-values.yaml - - name: Commit chart metadata, tag, and GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ steps.chart.outputs.version }} - APP_VERSION: ${{ steps.chart.outputs.app_version }} - IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} - run: | - set -euo pipefail - TAG="charts/trueforge@${VERSION}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - - if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then - echo "Tag $TAG already exists; skipping commit and GitHub Release" - else - git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml - if git diff --staged --quiet; then - echo "Chart metadata already matches $VERSION; skipping commit" >&2 - else - git commit -m "release(chart): ${VERSION}" - git push origin HEAD:main - fi - - git tag -a "$TAG" -m "Helm chart ${VERSION} (appVersion ${APP_VERSION}, image ${IMAGE_TAG})" - git push origin "$TAG" - - NOTES_FILE="$(mktemp)" - { - echo "Chart version: ${VERSION}" - echo "appVersion: ${APP_VERSION}" - echo "image.tag: ${IMAGE_TAG}" - } >"$NOTES_FILE" - gh release create "$TAG" \ - --title "$TAG" \ - --notes-file "$NOTES_FILE" - rm -f "$NOTES_FILE" - fi - - name: Helm registry login env: HELM_REGISTRY_USERNAME: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} @@ -227,3 +168,18 @@ jobs: echo "Pushed \`oci://${HELM_CHART_REPOSITORY}/${CHART_NAME}:${VERSION}\`" echo "Image tag \`${{ needs.resolve.outputs.image_tag }}\`" } >> "$GITHUB_STEP_SUMMARY" + + - name: Commit chart metadata to main + env: + VERSION: ${{ steps.chart.outputs.version }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml + if git diff --staged --quiet; then + echo "Chart metadata already matches $VERSION" + exit 0 + fi + git commit -m "release(chart): ${VERSION}" + git push origin HEAD:main diff --git a/.github/workflows/release-packages.yml b/.github/workflows/release-packages.yml index b34fac08f..e7979b227 100644 --- a/.github/workflows/release-packages.yml +++ b/.github/workflows/release-packages.yml @@ -9,9 +9,9 @@ # (install only; that PR is gated by CI.yml). When @truefoundry/trueforge-sdk # moves, scripts/version.mjs mirrors that version into python/trueforge_sdk and rebakes Fern. # - none pending + unpublished versions → pack (build/test) + Windows npx smoke, -# then npm publish and PyPI publish in parallel -# Merge the Version Packages PR to publish. There is no separate publish Action -# and no `v*` tag trigger. Image and Helm are a separate workflow. +# then npm publish, PyPI publish, and a chart/v* tag in parallel. +# Merge the Version Packages PR to publish. The chart/v* tag (pushed with the +# GitHub App token) starts the image + Helm workflow. # # Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc` # disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN / @@ -266,3 +266,44 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: python/trueforge_sdk/dist + + # App token so the tag push can start release-chart.yml (GITHUB_TOKEN cannot). + create-release-tag: + name: Create release tag + needs: [pack, windows-npx-smoke] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_ID }} + private-key: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_PRIVATE_KEY }} + + - name: Check out repo + uses: actions/checkout@v7 + with: + token: ${{ steps.app-token.outputs.token }} + + - name: Tag package version + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + VERSION=$(node -p "require('./packages/trueforge/package.json').version") + if ! printf '%s' "$VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$'; then + echo "package.json version '$VERSION' is not semver-like" >&2 + exit 1 + fi + TAG="chart/v${VERSION}" + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "Tag $TAG already exists; not moving it" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git tag -a "$TAG" -m "@truefoundry/trueforge ${VERSION}" + git push origin "$TAG" + echo "Pushed $TAG" diff --git a/RELEASING.md b/RELEASING.md index 8304a8c0e..7568068fd 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,14 +3,14 @@ This repo ships npm packages, a production container image, a Helm chart, a sandbox image, and optional from-source **dev** images. -| What | Trigger | Workflow | -| ----------------------- | ---------------------------------------------------------------------------- | -------------------------------------------------------------------- | -| npm packages | Push to `main` (Changesets) | [`release-packages.yml`](.github/workflows/release-packages.yml) | -| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release-packages.yml`](.github/workflows/release-packages.yml) | -| Prod image + Helm chart | Push to `main` (Changesets publish mode, parallel with packages) or dispatch | [`release-chart.yml`](.github/workflows/release-chart.yml) | -| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | -| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | -| PR checks | Pull request / merge group | [`ci.yml`](.github/workflows/ci.yml) | +| What | Trigger | Workflow | +| ----------------------- | ------------------------------------------------------------- | -------------------------------------------------------------------- | +| npm packages | Push to `main` (Changesets) | [`release-packages.yml`](.github/workflows/release-packages.yml) | +| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release-packages.yml`](.github/workflows/release-packages.yml) | +| Prod image + Helm chart | Push of `chart/v*` tag from the package workflow, or dispatch | [`release-chart.yml`](.github/workflows/release-chart.yml) | +| Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | +| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | +| PR checks | Pull request / merge group | [`ci.yml`](.github/workflows/ci.yml) | ## Versioning @@ -20,8 +20,8 @@ sandbox image, and optional from-source **dev** images. | PyPI `trueforge-sdk` | Same SemVer as `@truefoundry/trueforge-sdk` (mirrored into `pyproject.toml` on Version Packages) | | Chart `appVersion` | `packages/trueforge/package.json` version at the build commit | | Prod image | Root [`Dockerfile`](Dockerfile): from-source workspace build | -| Prod image tag | `{appVersion}-{shortSha}` (shortSha of the build commit) | -| Chart `version` | Independent SemVer; git tag `charts/trueforge@A.B.C` must match | +| Prod image tag | `{packageVersion}-{shortSha}` | +| Chart `version` | Same major.minor as `@truefoundry/trueforge`; patch/RC may still advance | | Sandbox image | [`sandbox.Dockerfile`](packages/trueforge-core/scripts/sandbox/sandbox.Dockerfile); tag = full commit SHA | | Dev image | Same [`Dockerfile`](Dockerfile); tag = full commit SHA | @@ -47,8 +47,8 @@ helm install trueforge oci://tfy.jfrog.io/tfy-helm/trueforge --version `). SDK regen already adds @@ -58,9 +58,10 @@ is packages only (`select-mode` → `version` \| `pack` → npm + PyPI): `scripts/version.mjs` mirrors that version into `python/trueforge_sdk` and regenerates both SDKs. Review and merge. 3. With no pending changesets, **pack** (build/test) and **Windows npx smoke** - run in parallel, then **npm publish** and **PyPI publish** run in parallel. - PyPI skips when that `pyproject.toml` version is already published. - Image and Helm are a separate workflow on the same push (see below). + run in parallel, then **npm publish**, **PyPI publish**, and **Create release tag** + (`chart/v`) run in parallel. PyPI skips when that + `pyproject.toml` version is already published. The tag is pushed with the + GitHub App token so it can start the image + Helm workflow. 4. Pin dependents to exact versions during early `0.x`. `workflow_dispatch` on **Version or publish packages** re-runs the same workflow. @@ -122,8 +123,9 @@ pnpm clean && pnpm build && pnpm standalone:start (`scripts/version.mjs`; needs Docker). That path also mirrors the version into `python/trueforge_sdk`. - **PyPI 403 / invalid-publisher** - register a trusted publisher for `trueforge-sdk` bound to `release-packages.yml` (and create the project if it does not exist yet). -- **Prod image / chart missing after package publish** - dispatch the chart workflow on a - **branch or tag** (not a SHA): `gh workflow run release-chart.yml --ref main`. +- **Prod image / chart missing after package publish** - confirm `chart/v` was + pushed, or dispatch the chart workflow on a **branch or tag** (not a SHA): + `gh workflow run release-chart.yml --ref main`. --- @@ -131,50 +133,51 @@ pnpm clean && pnpm build && pnpm standalone:start ```text push to main (no pending changesets, unpublished versions) - → release-packages.yml: pack + smoke → npm | PyPI - → release-chart.yml (parallel, independent): + → release-packages.yml: pack + smoke + → npm | PyPI | chart/v tag (parallel) + → tag push starts release-chart.yml → build from-source Dockerfile → push X.Y.Z- - → bump Chart.yaml / values.yaml, lint, package - → commit to main, tag charts/trueforge@A.B.C, GH Release, OCI push + → helm lint/package/push OCI + → commit Chart.yaml + values.yaml to main manual rebuild - → workflow_dispatch release-chart.yml - → same image/Helm path + → workflow_dispatch release-chart.yml --ref main (or an existing chart/v* tag) ``` +The package workflow pushes the tag with the GitHub App token. `GITHUB_TOKEN` +tag pushes do not start other workflows. + ## Dockerfile | File | Role | | -------------------------- | ------------------------------------------------------------------------------------------------------------------- | | [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build SHA-tagged from-source image** | -The image is the workspace at `GITHUB_SHA`. Chart `appVersion` is that commit's -`packages/trueforge/package.json` version (not a requirement that npm already -advertises the package). +The image is the workspace at the tagged commit. Chart `appVersion` is that +commit's `packages/trueforge/package.json` version. A `chart/v*` tag push fails +if the version after `chart/v` does not match that package version. ## Build server image and publish Helm chart -[`release-chart.yml`](.github/workflows/release-chart.yml) -(push to `main`, or manual `workflow_dispatch`). Same Changesets select-mode as -packages: pending changesets skip the image/Helm jobs. Manual dispatch always -builds. +[`release-chart.yml`](.github/workflows/release-chart.yml) (`chart/v*` tag, or +`workflow_dispatch`). -| Input | Default | Meaning | -| ------------- | --------------------------------- | -------------------------------------------- | -| `app_version` | `packages/trueforge/package.json` | Version used in the image tag and appVersion | +| Input | Default | Meaning | +| ------------- | --------------------------------- | -------------------------------------------------------------- | +| `app_version` | `packages/trueforge/package.json` | Used on dispatch; ignored when the trigger is a `chart/v*` tag | -Always: build/push `{appVersion}-{shortSha}`, patch-bump chart `version`, set -`appVersion` and `image.tag`, lint, package, push OCI, tag, GitHub Release, and -commit the chart metadata to `main` with `GITHUB_TOKEN` (that push does not -start another workflow). +Always: build/push `{appVersion}-{shortSha}`, set chart `version` to the app +major.minor (patch/RC may still advance), set `appVersion` and `image.tag`, +lint, package, push OCI, then commit those chart files to `main`. ```bash -gh workflow run release-chart.yml -gh workflow run release-chart.yml -f app_version=0.1.0 +gh workflow run release-chart.yml --ref main +gh workflow run release-chart.yml --ref main -f app_version=0.1.0 ``` -Chart SemVer is chosen by [`scripts/resolve-chart-version.sh`](scripts/resolve-chart-version.sh) -from the current `Chart.yaml` on the build commit. +Chart major.minor is taken from [`scripts/resolve-chart-version.sh`](scripts/resolve-chart-version.sh) +so it matches `@truefoundry/trueforge` (and the docker tag prefix). Patch and RC +still advance per chart release. ## Dev / floating main diff --git a/charts/trueforge/Chart.yaml b/charts/trueforge/Chart.yaml index 8a2a4594e..67f2ae634 100644 --- a/charts/trueforge/Chart.yaml +++ b/charts/trueforge/Chart.yaml @@ -2,9 +2,8 @@ apiVersion: v2 name: trueforge description: TrueForge server (API + UI) served from a single container image. type: application -# version / appVersion / image.tag are written on main by the image+Helm release -# workflow (or by a human). Publishing is gated by git tag charts/trueforge@ -# (must match version). +# version / appVersion / image.tag are written on main by release-chart.yml +# after the chart/v* tag from release-packages.yml. version: "0.2.3-rc.0" appVersion: "0.3.0-rc.0" kubeVersion: ">=1.25.0-0" diff --git a/charts/trueforge/README.md b/charts/trueforge/README.md index 238840f0f..14eda7a17 100644 --- a/charts/trueforge/README.md +++ b/charts/trueforge/README.md @@ -8,10 +8,9 @@ and the UI. Production, Compose smoke, and Railway all build the repository-root The chart always runs the server in **distributed** mode (`STANDALONE=false`) against Postgres and Redis. -Chart `version` / `appVersion` / `image.tag` are written on `main` by the -image+Helm release workflow (or a human). Publishing is gated by git tag -`charts/trueforge@`. -See [`RELEASING.md`](../../RELEASING.md). +Chart `version` / `appVersion` / `image.tag` are written on `main` by +[`release-chart.yml`](../../.github/workflows/release-chart.yml) when a +`chart/v*` tag is pushed. See [`RELEASING.md`](../../RELEASING.md). ## Dev defaults (read before exposing) diff --git a/charts/trueforge/values.yaml b/charts/trueforge/values.yaml index 3eb81f538..6d33b72cc 100644 --- a/charts/trueforge/values.yaml +++ b/charts/trueforge/values.yaml @@ -38,9 +38,9 @@ global: sizeLimit: 10Mi image: repository: tfy.jfrog.io/tfy-images/trueforge - # Tag defaults to chart appVersion when empty. The image+Helm release workflow - # sets this to {appVersion}-{shortSha} for the from-source image already in - # the registry. + # Tag defaults to chart appVersion when empty. release-chart.yml sets this + # to {appVersion}-{shortSha} for the from-source image already in the + # registry. tag: "0.3.0-rc.0-346221b" pullPolicy: IfNotPresent imagePullSecrets: [] diff --git a/scripts/resolve-chart-version.sh b/scripts/resolve-chart-version.sh index e74dce73c..d0395aa26 100755 --- a/scripts/resolve-chart-version.sh +++ b/scripts/resolve-chart-version.sh @@ -7,73 +7,86 @@ PR_VERSION=${3:-} SEMVER='^([0-9]+)\.([0-9]+)\.([0-9]+)(-[0-9A-Za-z.-]+)?$' -if [[ ! "$CURRENT" =~ $SEMVER ]]; then +parse() { + local v=$1 + if [[ ! "$v" =~ $SEMVER ]]; then + return 1 + fi + _major=${BASH_REMATCH[1]} + _minor=${BASH_REMATCH[2]} + _patch=${BASH_REMATCH[3]} + _pre=${BASH_REMATCH[4]:-} +} + +if ! parse "$CURRENT"; then echo "Current chart version '$CURRENT' is not semver" >&2 exit 1 fi +CUR_MAJOR=$_major +CUR_MINOR=$_minor +CUR_PATCH=$_patch +CUR_PRE=$_pre -MAJOR=${BASH_REMATCH[1]} -MINOR=${BASH_REMATCH[2]} -PATCH=${BASH_REMATCH[3]} -CURRENT_PRERELEASE=${BASH_REMATCH[4]:-} -SELECTED_MAJOR=$MAJOR -SELECTED_MINOR=$MINOR -SELECTED_PATCH=$PATCH - -# A stable chart starts the next patch. A prerelease chart is already on its -# target core, so subsequent RCs and the stable release keep that core. -if [[ -z "$CURRENT_PRERELEASE" ]]; then - SELECTED_PATCH=$((PATCH + 1)) -fi - -if [[ ! "$APP_VERSION" =~ $SEMVER ]]; then +if ! parse "$APP_VERSION"; then echo "App version '$APP_VERSION' is not semver" >&2 exit 1 fi +APP_MAJOR=$_major +APP_MINOR=$_minor +APP_PRE=$_pre -APP_PRERELEASE=${BASH_REMATCH[4]:-} -PR_PRERELEASE="" -PR_CORE_SELECTED=false +# Chart major.minor follows the app package (and thus the docker tag prefix). +SELECTED_MAJOR=$APP_MAJOR +SELECTED_MINOR=$APP_MINOR -if [[ -n "$PR_VERSION" && "$PR_VERSION" =~ $SEMVER ]]; then - PR_MAJOR=${BASH_REMATCH[1]} - PR_MINOR=${BASH_REMATCH[2]} - PR_PATCH=${BASH_REMATCH[3]} - PR_PRERELEASE=${BASH_REMATCH[4]:-} +if ((CUR_MAJOR == APP_MAJOR && CUR_MINOR == APP_MINOR)); then + # Same line: a stable chart starts the next patch. A prerelease chart is + # already on its target core, so later RCs and the stable release keep it. + if [[ -z "$CUR_PRE" ]]; then + SELECTED_PATCH=$((CUR_PATCH + 1)) + else + SELECTED_PATCH=$CUR_PATCH + fi +else + SELECTED_PATCH=0 +fi - if ((PR_MAJOR > SELECTED_MAJOR || - (PR_MAJOR == SELECTED_MAJOR && PR_MINOR > SELECTED_MINOR) || - (PR_MAJOR == SELECTED_MAJOR && PR_MINOR == SELECTED_MINOR && PR_PATCH > SELECTED_PATCH))); then - SELECTED_MAJOR=$PR_MAJOR - SELECTED_MINOR=$PR_MINOR - SELECTED_PATCH=$PR_PATCH - PR_CORE_SELECTED=true - elif ((PR_MAJOR == SELECTED_MAJOR && PR_MINOR == SELECTED_MINOR && PR_PATCH == SELECTED_PATCH)); then - PR_CORE_SELECTED=true +PR_CORE_SELECTED=false +PR_PRE="" +if [[ -n "$PR_VERSION" ]] && parse "$PR_VERSION"; then + PR_MAJOR=$_major + PR_MINOR=$_minor + PR_PATCH=$_patch + PR_PRE=$_pre + # A reviewer bump on the open PR is kept only when it stays on the app line. + if ((PR_MAJOR == APP_MAJOR && PR_MINOR == APP_MINOR)); then + if ((PR_PATCH > SELECTED_PATCH)); then + SELECTED_PATCH=$PR_PATCH + PR_CORE_SELECTED=true + elif ((PR_PATCH == SELECTED_PATCH)); then + PR_CORE_SELECTED=true + fi fi fi -# The app version only selects stable or prerelease mode. Chart RC counters are -# derived from chart versions and advance independently from the app's suffix. -SELECTED_PRERELEASE="" -if [[ -n "$APP_PRERELEASE" ]]; then +SELECTED_PRE="" +if [[ -n "$APP_PRE" ]]; then RC_COUNTER='^-rc\.([0-9]+)$' HIGHEST_RC=-1 - if ((MAJOR == SELECTED_MAJOR && MINOR == SELECTED_MINOR && PATCH == SELECTED_PATCH)) && - [[ "$CURRENT_PRERELEASE" =~ $RC_COUNTER ]]; then + if ((CUR_MAJOR == SELECTED_MAJOR && CUR_MINOR == SELECTED_MINOR && CUR_PATCH == SELECTED_PATCH)) && + [[ "$CUR_PRE" =~ $RC_COUNTER ]]; then HIGHEST_RC=${BASH_REMATCH[1]} fi - if [[ "$PR_CORE_SELECTED" == true && "$PR_PRERELEASE" =~ $RC_COUNTER ]]; then + if [[ "$PR_CORE_SELECTED" == true && "$PR_PRE" =~ $RC_COUNTER ]]; then PR_COUNTER=${BASH_REMATCH[1]} if ((PR_COUNTER > HIGHEST_RC)); then HIGHEST_RC=$PR_COUNTER fi fi - SELECTED_PRERELEASE="-rc.$((HIGHEST_RC + 1))" + SELECTED_PRE="-rc.$((HIGHEST_RC + 1))" fi -VERSION="${SELECTED_MAJOR}.${SELECTED_MINOR}.${SELECTED_PATCH}${SELECTED_PRERELEASE}" -printf '%s\n' "$VERSION" +printf '%s\n' "${SELECTED_MAJOR}.${SELECTED_MINOR}.${SELECTED_PATCH}${SELECTED_PRE}" diff --git a/tests/scripts/resolve-chart-version.test.sh b/tests/scripts/resolve-chart-version.test.sh index 912cf7a3c..3514f5ab9 100755 --- a/tests/scripts/resolve-chart-version.test.sh +++ b/tests/scripts/resolve-chart-version.test.sh @@ -21,12 +21,11 @@ assert_version() { fi } -# Stable chart releases continue with patch bumps. +# Same major.minor as the app: stable charts continue with patch bumps. assert_version 0.1.6 0.1.5 0.1.5 assert_version 0.1.7 0.1.6 0.1.6 -# Entering prerelease mode starts the chart's own RC counter at zero. The app's -# prerelease tag and counter do not influence the chart version. +# Entering prerelease mode on the same line starts the chart RC counter at zero. assert_version 0.1.6-rc.0 0.1.5 0.1.5-rc.2 # Merged chart RCs and RCs in the open chart PR both advance linearly. @@ -39,11 +38,18 @@ assert_version 0.1.6-rc.5 0.1.6-rc.3 0.1.5-rc.7 0.1.6-rc.4 assert_version 0.1.6 0.1.6-rc.4 0.1.6 assert_version 0.1.6 0.1.5 0.1.6 0.1.6-rc.4 -# A reviewed higher chart core is preserved without coupling it to the app core. -assert_version 0.2.0-rc.0 0.1.5 0.1.5-rc.3 0.2.0 -assert_version 0.2.0-rc.5 0.1.5 0.1.5-rc.3 0.2.0-rc.4 -assert_version 0.2.0 0.1.5 0.1.5 0.2.0-rc.4 -assert_version 0.1.6-rc.0 0.1.5 0.1.5-rc.3 0.1.4-rc.9 +# Chart major.minor follows the app (package / docker prefix). +assert_version 0.3.0-rc.0 0.2.3-rc.0 0.3.0-rc.0 +assert_version 0.3.0 0.2.5 0.3.0 +assert_version 0.4.0-rc.0 0.3.1 0.4.0-rc.1 + +# A reviewer patch bump on the open PR is kept when it stays on the app line. +assert_version 0.3.1-rc.0 0.3.0 0.3.0-rc.1 0.3.1 +assert_version 0.3.0-rc.5 0.3.0-rc.3 0.3.0-rc.0 0.3.0-rc.4 + +# A PR version on a different minor than the app is ignored. +assert_version 0.3.0-rc.0 0.2.3-rc.0 0.3.0-rc.0 0.4.0 +assert_version 0.1.6-rc.0 0.1.5 0.1.5-rc.3 0.2.0 if ((failures > 0)); then exit 1 From 755405bf562aec810316433ab988176f23dcb943 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Wed, 23 Sep 2026 19:39:54 +0530 Subject: [PATCH 4/7] Make chart publish retry-safe and keep the pinned image runnable. Replay chart metadata onto origin/main with the App token, treat an already-published OCI version as success, and leave the npm-install controller command in values.yaml until that publish rewrites it. Signed-off-by: Raman Tehlan --- .github/workflows/release-chart.yml | 65 ++++++++++++++++++++++++----- RELEASING.md | 8 ++-- charts/trueforge/README.md | 6 +-- charts/trueforge/values.yaml | 5 ++- 4 files changed, 66 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index 1f256cc69..ada06065a 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -50,7 +50,7 @@ jobs: if [[ "$EVENT_NAME" == "push" && "$REF_NAME" == chart/v* ]]; then TAG_VERSION="${REF_NAME#chart/v}" if [[ "$TAG_VERSION" != "$PKG_VERSION" ]]; then - echo "Tag v$TAG_VERSION does not match package.json $PKG_VERSION" >&2 + echo "Tag chart/v$TAG_VERSION does not match package.json $PKG_VERSION" >&2 exit 1 fi APP_VERSION="$PKG_VERSION" @@ -106,10 +106,20 @@ jobs: HELM_REGISTRY_URL: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} HELM_CHART_REPOSITORY: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_HELM_REPOSITORY }} steps: + # App token: GITHUB_TOKEN cannot start workflows and often cannot push + # to protected main. Chart files are replayed onto origin/main so a tag + # or dispatch ref never force-updates main. + - id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_ID }} + private-key: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_PRIVATE_KEY }} + - name: Check out uses: actions/checkout@v7 with: fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} - name: Install yq run: | @@ -121,19 +131,22 @@ jobs: with: version: v3.16.4 - - name: Update chart metadata + - name: Update chart metadata on main id: chart env: IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} APP_VERSION: ${{ needs.resolve.outputs.app_version }} run: | set -euo pipefail + git fetch origin main + git checkout -B chart-release origin/main CURRENT=$(yq -r '.version' charts/trueforge/Chart.yaml) VERSION=$(bash scripts/resolve-chart-version.sh "$CURRENT" "$APP_VERSION") export VERSION IMAGE_TAG APP_VERSION yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml + yq -i '.controller.command = ["node", "dist/controller-main.js"]' charts/trueforge/values.yaml echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" echo "Chart version=$VERSION appVersion=$APP_VERSION image.tag=$IMAGE_TAG" @@ -162,7 +175,18 @@ jobs: helm package "$CHART_DIR" --destination dist PACKAGE="dist/${CHART_NAME}-${VERSION}.tgz" echo "Pushing $PACKAGE to oci://$HELM_CHART_REPOSITORY" - helm push "$PACKAGE" "oci://$HELM_CHART_REPOSITORY" + set +e + PUSH_OUT=$(helm push "$PACKAGE" "oci://$HELM_CHART_REPOSITORY" 2>&1) + PUSH_RC=$? + set -e + printf '%s\n' "$PUSH_OUT" + if [[ "$PUSH_RC" -ne 0 ]]; then + if printf '%s' "$PUSH_OUT" | grep -qiE 'already exists|409'; then + echo "Chart $VERSION already in the registry; continuing so main can catch up" + else + exit "$PUSH_RC" + fi + fi { echo "### Helm chart" echo "Pushed \`oci://${HELM_CHART_REPOSITORY}/${CHART_NAME}:${VERSION}\`" @@ -172,14 +196,35 @@ jobs: - name: Commit chart metadata to main env: VERSION: ${{ steps.chart.outputs.version }} + IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + APP_VERSION: ${{ needs.resolve.outputs.app_version }} run: | set -euo pipefail git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml - if git diff --staged --quiet; then - echo "Chart metadata already matches $VERSION" - exit 0 - fi - git commit -m "release(chart): ${VERSION}" - git push origin HEAD:main + + apply_metadata() { + export VERSION IMAGE_TAG APP_VERSION + yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml + yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml + yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml + yq -i '.controller.command = ["node", "dist/controller-main.js"]' charts/trueforge/values.yaml + } + + for attempt in 1 2 3; do + git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml + if git diff --staged --quiet; then + echo "Chart metadata already matches $VERSION" + exit 0 + fi + git commit -m "release(chart): ${VERSION}" + if git push origin HEAD:main; then + exit 0 + fi + echo "Push to main failed (attempt $attempt); replay metadata onto origin/main" + git fetch origin main + git reset --hard origin/main + apply_metadata + done + echo "Could not fast-forward chart metadata onto main" >&2 + exit 1 diff --git a/RELEASING.md b/RELEASING.md index 7568068fd..60870dbf2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -166,9 +166,11 @@ if the version after `chart/v` does not match that package version. | ------------- | --------------------------------- | -------------------------------------------------------------- | | `app_version` | `packages/trueforge/package.json` | Used on dispatch; ignored when the trigger is a `chart/v*` tag | -Always: build/push `{appVersion}-{shortSha}`, set chart `version` to the app -major.minor (patch/RC may still advance), set `appVersion` and `image.tag`, -lint, package, push OCI, then commit those chart files to `main`. +Always: build/push `{appVersion}-{shortSha}`, replay chart `version` / +`appVersion` / `image.tag` / controller command onto current `main`, lint, +package, push OCI (idempotent if that chart version is already in the registry), +then commit those files to `main`. The commit is replayed onto `origin/main` if +`main` moved during the image build. ```bash gh workflow run release-chart.yml --ref main diff --git a/charts/trueforge/README.md b/charts/trueforge/README.md index 14eda7a17..ac6028959 100644 --- a/charts/trueforge/README.md +++ b/charts/trueforge/README.md @@ -8,9 +8,9 @@ and the UI. Production, Compose smoke, and Railway all build the repository-root The chart always runs the server in **distributed** mode (`STANDALONE=false`) against Postgres and Redis. -Chart `version` / `appVersion` / `image.tag` are written on `main` by -[`release-chart.yml`](../../.github/workflows/release-chart.yml) when a -`chart/v*` tag is pushed. See [`RELEASING.md`](../../RELEASING.md). +Chart `version` / `appVersion` / `image.tag` / controller command are written +on `main` by [`release-chart.yml`](../../.github/workflows/release-chart.yml) +when a `chart/v*` tag is pushed. See [`RELEASING.md`](../../RELEASING.md). ## Dev defaults (read before exposing) diff --git a/charts/trueforge/values.yaml b/charts/trueforge/values.yaml index 6d33b72cc..4cc6e94f3 100644 --- a/charts/trueforge/values.yaml +++ b/charts/trueforge/values.yaml @@ -117,10 +117,11 @@ controller: enabled: true # Annotations on the controller Deployment object (for example, Argo CD sync waves). deploymentAnnotations: {} - # Container command. Image WORKDIR is /app/packages/trueforge (from-source Dockerfile). + # Matches the pinned image.tag. release-chart.yml rewrites this to + # dist/controller-main.js when it publishes a from-source image. command: - node - - dist/controller-main.js + - node_modules/@truefoundry/trueforge/dist/controller-main.js # Base URL the controller uses to reach the server (SERVER_URL). # Empty → the in-cluster server Service (http(s)://:). serverUrl: "" From bce80526030b36ba0a5a663f9e2d61405e84b0d0 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Thu, 24 Sep 2026 14:46:29 +0530 Subject: [PATCH 5/7] Address review comments on the split release workflows. Keep trusted publishing on release.yml, restore the npm-install Dockerfile as Dockerfile.npm, keep Dockerfile.dev for existing Railway services, and put the from-source controller command in values.yaml. The package workflow dispatches the chart workflow; charts/trueforge@* is tagged on the Helm metadata commit. Signed-off-by: Raman Tehlan --- .../{build-dev-image.yml => build-image.yml} | 6 +- .github/workflows/push-sandbox-image.yml | 2 +- .github/workflows/release-chart.yml | 48 +++---- .../{release-packages.yml => release.yml} | 49 +++----- .railway/railway.ts | 4 +- Dockerfile.dev | 119 ++++++++++++++++++ Dockerfile.npm | 33 +++++ RELEASING.md | 81 ++++++------ charts/trueforge/Chart.yaml | 4 +- charts/trueforge/README.md | 7 +- charts/trueforge/values.yaml | 5 +- 11 files changed, 251 insertions(+), 107 deletions(-) rename .github/workflows/{build-dev-image.yml => build-image.yml} (94%) rename .github/workflows/{release-packages.yml => release.yml} (84%) create mode 100644 Dockerfile.dev create mode 100644 Dockerfile.npm diff --git a/.github/workflows/build-dev-image.yml b/.github/workflows/build-image.yml similarity index 94% rename from .github/workflows/build-dev-image.yml rename to .github/workflows/build-image.yml index fdbf97863..df970db1b 100644 --- a/.github/workflows/build-dev-image.yml +++ b/.github/workflows/build-image.yml @@ -1,4 +1,4 @@ -name: Build dev image +name: Build image # From-source image for floating-main / external deploy repos. # Builds the root Dockerfile and pushes .../trueforge:. No chart publish. @@ -6,7 +6,7 @@ on: workflow_dispatch: concurrency: - group: build-dev-image + group: build-image cancel-in-progress: false permissions: @@ -47,7 +47,7 @@ jobs: set -euo pipefail echo "Pushed image URI: $IMAGE_URI" { - echo "### Dev image (from-source)" + echo "### Image (from-source)" echo "\`$IMAGE_URI\`" echo "" echo "Patch this SHA into your external \`truefoundry.yaml\` as \`image.tag\`." diff --git a/.github/workflows/push-sandbox-image.yml b/.github/workflows/push-sandbox-image.yml index 73ce3d134..1a5e3e72c 100644 --- a/.github/workflows/push-sandbox-image.yml +++ b/.github/workflows/push-sandbox-image.yml @@ -101,7 +101,7 @@ jobs: commit-message: 'chore: update sandbox image ${{ github.sha }}' title: 'chore: update sandbox image ${{ github.sha }}' body: | - Pushed `${{ steps.image.outputs.uri }}`. Merge so `release-packages.yml` can publish the pin. + Pushed `${{ steps.image.outputs.uri }}`. Merge so `release.yml` can publish the pin. Existing tenants stay on `build_metadata`. add-paths: | packages/trueforge-core/src/core/sandbox/sandboxImage.json diff --git a/.github/workflows/release-chart.yml b/.github/workflows/release-chart.yml index ada06065a..f5d8bf6b0 100644 --- a/.github/workflows/release-chart.yml +++ b/.github/workflows/release-chart.yml @@ -1,14 +1,12 @@ name: Build server image and publish Helm chart -# Starts when release-packages.yml pushes chart/v (App token). -# Builds the from-source image, then packages and publishes the Helm chart. +# Starts when release.yml dispatches this workflow (App token). Builds the +# from-source image, publishes the Helm chart, commits metadata to main, then +# tags charts/trueforge@ on that commit. # -# workflow_dispatch is for manual rebuilds. Do not use `gh workflow run --ref +# workflow_dispatch is also for manual rebuilds. Do not use `gh workflow run --ref # $GITHUB_SHA`: the dispatch API only accepts a branch or tag name (HTTP 422). on: - push: - tags: - - 'chart/v[0-9]*' workflow_dispatch: inputs: app_version: @@ -42,19 +40,10 @@ jobs: id: resolve env: INPUT_APP_VERSION: ${{ inputs.app_version }} - EVENT_NAME: ${{ github.event_name }} - REF_NAME: ${{ github.ref_name }} run: | set -euo pipefail PKG_VERSION=$(node -p "require('./packages/trueforge/package.json').version") - if [[ "$EVENT_NAME" == "push" && "$REF_NAME" == chart/v* ]]; then - TAG_VERSION="${REF_NAME#chart/v}" - if [[ "$TAG_VERSION" != "$PKG_VERSION" ]]; then - echo "Tag chart/v$TAG_VERSION does not match package.json $PKG_VERSION" >&2 - exit 1 - fi - APP_VERSION="$PKG_VERSION" - elif [[ -n "${INPUT_APP_VERSION}" ]]; then + if [[ -n "${INPUT_APP_VERSION}" ]]; then APP_VERSION="$INPUT_APP_VERSION" else APP_VERSION="$PKG_VERSION" @@ -67,12 +56,14 @@ jobs: echo "APP_VERSION '$APP_VERSION' is not a semver-like version" >&2 exit 1 fi - SHORT_SHA="${GITHUB_SHA:0:7}" + # Peel annotated tags: GITHUB_SHA is the tag object, not the commit. + COMMIT_SHA=$(git rev-parse --verify HEAD) + SHORT_SHA=$(git rev-parse --short=7 HEAD) IMAGE_TAG="${APP_VERSION}-${SHORT_SHA}" echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" - echo "Resolved APP_VERSION=$APP_VERSION IMAGE_TAG=$IMAGE_TAG (package.json=$PKG_VERSION)" + echo "Resolved APP_VERSION=$APP_VERSION IMAGE_TAG=$IMAGE_TAG commit=$COMMIT_SHA (package.json=$PKG_VERSION)" build: name: Build and push server image @@ -146,7 +137,6 @@ jobs: yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml - yq -i '.controller.command = ["node", "dist/controller-main.js"]' charts/trueforge/values.yaml echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "app_version=$APP_VERSION" >> "$GITHUB_OUTPUT" echo "Chart version=$VERSION appVersion=$APP_VERSION image.tag=$IMAGE_TAG" @@ -177,14 +167,14 @@ jobs: echo "Pushing $PACKAGE to oci://$HELM_CHART_REPOSITORY" set +e PUSH_OUT=$(helm push "$PACKAGE" "oci://$HELM_CHART_REPOSITORY" 2>&1) - PUSH_RC=$? + PUSH_STATUS=$? set -e printf '%s\n' "$PUSH_OUT" - if [[ "$PUSH_RC" -ne 0 ]]; then + if [[ "$PUSH_STATUS" -ne 0 ]]; then if printf '%s' "$PUSH_OUT" | grep -qiE 'already exists|409'; then echo "Chart $VERSION already in the registry; continuing so main can catch up" else - exit "$PUSH_RC" + exit "$PUSH_STATUS" fi fi { @@ -208,17 +198,29 @@ jobs: yq -i '.version = strenv(VERSION)' charts/trueforge/Chart.yaml yq -i '.appVersion = strenv(APP_VERSION)' charts/trueforge/Chart.yaml yq -i '.image.tag = strenv(IMAGE_TAG)' charts/trueforge/values.yaml - yq -i '.controller.command = ["node", "dist/controller-main.js"]' charts/trueforge/values.yaml + } + + tag_chart() { + TAG="charts/trueforge@${VERSION}" + if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then + echo "Tag $TAG already exists" + return 0 + fi + git tag -a "$TAG" -m "Helm chart ${VERSION}" + git push origin "$TAG" + echo "Pushed $TAG" } for attempt in 1 2 3; do git add charts/trueforge/Chart.yaml charts/trueforge/values.yaml if git diff --staged --quiet; then echo "Chart metadata already matches $VERSION" + tag_chart exit 0 fi git commit -m "release(chart): ${VERSION}" if git push origin HEAD:main; then + tag_chart exit 0 fi echo "Push to main failed (attempt $attempt); replay metadata onto origin/main" diff --git a/.github/workflows/release-packages.yml b/.github/workflows/release.yml similarity index 84% rename from .github/workflows/release-packages.yml rename to .github/workflows/release.yml index e7979b227..4fbc99341 100644 --- a/.github/workflows/release-packages.yml +++ b/.github/workflows/release.yml @@ -1,17 +1,22 @@ # npm publish for the four public packages, plus PyPI for python/trueforge_sdk. # -# npm and PyPI trusted publishers are bound to this filename -# (`release-packages.yml`). Update every package on npmjs.com and PyPI if you -# rename it. They were previously bound to `release.yml`. +# npm and PyPI trusted publishers are bound to this filename (`release.yml`). +# Do not rename it without updating every package on npmjs.com and PyPI. # # On every push to main, changesets/action/select-mode chooses: # - pending `.changeset/*.md` → version job opens/updates the Version Packages PR # (install only; that PR is gated by CI.yml). When @truefoundry/trueforge-sdk # moves, scripts/version.mjs mirrors that version into python/trueforge_sdk and rebakes Fern. # - none pending + unpublished versions → pack (build/test) + Windows npx smoke, -# then npm publish, PyPI publish, and a chart/v* tag in parallel. -# Merge the Version Packages PR to publish. The chart/v* tag (pushed with the -# GitHub App token) starts the image + Helm workflow. +# then npm publish, PyPI publish, and the image+Helm workflow in parallel. +# Merge the Version Packages PR to publish. The chart workflow is started with +# the GitHub App token (`gh workflow run`); GITHUB_TOKEN cannot dispatch it. +# After Helm publish, that workflow tags charts/trueforge@ on the +# metadata commit. +# +# Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc` +# disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN / +# NODE_AUTH_TOKEN. id-token is granted only on the publish jobs. # # Auth is GitHub OIDC (no NPM_TOKEN / PYPI_TOKEN). An `_authToken` in user `.npmrc` # disables pnpm's OIDC exchange, so the npm publish job must not set NPM_TOKEN / @@ -267,14 +272,15 @@ jobs: with: packages-dir: python/trueforge_sdk/dist - # App token so the tag push can start release-chart.yml (GITHUB_TOKEN cannot). - create-release-tag: - name: Create release tag + # App token: GITHUB_TOKEN cannot dispatch another workflow. + dispatch-chart-release: + name: Start image and Helm release needs: [pack, windows-npx-smoke] runs-on: ubuntu-latest timeout-minutes: 10 permissions: - contents: write + actions: write + contents: read steps: - id: app-token uses: actions/create-github-app-token@v3 @@ -282,28 +288,9 @@ jobs: client-id: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_ID }} private-key: ${{ secrets.TRUEFORGE_GENERATE_SDK_APP_PRIVATE_KEY }} - - name: Check out repo - uses: actions/checkout@v7 - with: - token: ${{ steps.app-token.outputs.token }} - - - name: Tag package version + - name: Dispatch release-chart.yml env: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | set -euo pipefail - VERSION=$(node -p "require('./packages/trueforge/package.json').version") - if ! printf '%s' "$VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$'; then - echo "package.json version '$VERSION' is not semver-like" >&2 - exit 1 - fi - TAG="chart/v${VERSION}" - if git ls-remote --exit-code --tags origin "refs/tags/${TAG}" >/dev/null 2>&1; then - echo "Tag $TAG already exists; not moving it" - exit 0 - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git tag -a "$TAG" -m "@truefoundry/trueforge ${VERSION}" - git push origin "$TAG" - echo "Pushed $TAG" + gh workflow run release-chart.yml --repo "$GITHUB_REPOSITORY" --ref "${GITHUB_REF_NAME}" diff --git a/.railway/railway.ts b/.railway/railway.ts index 288be706b..60678f26d 100644 --- a/.railway/railway.ts +++ b/.railway/railway.ts @@ -28,7 +28,9 @@ export default defineRailway(_ctx => { // Deploys from this repository's default branch. Forks: change owner/repo // (and optionally branch) to build your own copy. source: github('truefoundry/trueforge'), - // Railway uses the root from-source Dockerfile by default. + // Railway uses the root from-source Dockerfile by default. Existing + // services may still set RAILWAY_DOCKERFILE_PATH=Dockerfile.dev; that file + // is kept as a copy of Dockerfile until IaC is re-applied. healthcheck: '/healthz', healthcheckTimeout: 300, deploy: { diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 000000000..5114a5cf0 --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,119 @@ +# syntax=docker/dockerfile:1 +# +# Same as Dockerfile. Kept so existing Railway services that still set +# RAILWAY_DOCKERFILE_PATH=Dockerfile.dev keep building. New applies use +# the root Dockerfile by default. +# +# From-source multi-stage image for prod Helm, local smoke, Railway, and +# SHA-tagged floating-main builds. Lives at the repository root because the +# build needs the whole pnpm workspace as its context: the server depends on +# the workspace package @truefoundry/trueforge-core. +# +# Dependency install uses pnpm fetch (lockfile-only) then install --offline so +# the download layer stays cached when only package.json / scripts change. +# See https://pnpm.io/cli/fetch. BuildKit cache mounts are avoided so the same +# file builds on Railway Metal (which requires a hardcoded service id in mount ids). + +FROM node:24-slim AS base +ENV PNPM_HOME=/pnpm +ENV PATH="$PNPM_HOME:$PATH" +RUN corepack enable && pnpm config set store-dir /pnpm/store +WORKDIR /app + +# --------------------------------------------------------------------------- +# store: the pnpm store, from the lockfile only (stable when manifests churn). +# BuildKit cache mounts are omitted: Railway's Metal builder requires +# `id=s/-…` (hardcoded per deploy), which cannot live in a shared +# OSS Dockerfile. Layer cache on this stage still hits when the lockfile is +# unchanged. +# --------------------------------------------------------------------------- +FROM base AS store +COPY pnpm-lock.yaml pnpm-workspace.yaml ./ +RUN pnpm fetch + +# --------------------------------------------------------------------------- +# workspace: install inputs shared by every stage below - the manifests plus the +# sources the root postinstall hook (build:gen) inlines. +# --------------------------------------------------------------------------- +FROM store AS workspace +COPY package.json .npmrc tsconfig.base.json ./ +# Root scripts used by package build steps (rm-path.mjs, chmod-path.mjs). +COPY scripts scripts +COPY packages/trueforge-core/package.json packages/trueforge-core/package.json +COPY packages/trueforge/package.json packages/trueforge/package.json +COPY packages/trueforge-sdk/package.json packages/trueforge-sdk/package.json +COPY packages/frontend/package.json packages/frontend/package.json +COPY packages/trueforge-ui/package.json packages/trueforge-ui/package.json +COPY packages/assistant-ui-runtime/package.json packages/assistant-ui-runtime/package.json +COPY packages/trueforge-core/scripts packages/trueforge-core/scripts +COPY packages/trueforge-core/src/core/sandbox/scripts packages/trueforge-core/src/core/sandbox/scripts + +# --------------------------------------------------------------------------- +# builder: install all deps (incl. dev) and build trueforge-core + server. +# SDK dist is copied into the production runner; host-dev/typecheck resolve SDK +# from src/ via trueforge-dev. Build it here once for the image. +# --------------------------------------------------------------------------- +FROM workspace AS builder +RUN pnpm install --frozen-lockfile --offline --filter @truefoundry/trueforge... +COPY packages/trueforge-core packages/trueforge-core +COPY packages/trueforge-sdk packages/trueforge-sdk +RUN pnpm --filter @truefoundry/trueforge-sdk build +COPY packages/trueforge packages/trueforge +RUN pnpm --filter @truefoundry/trueforge-core build && pnpm --filter @truefoundry/trueforge build + +# --------------------------------------------------------------------------- +# frontend-builder: build the UI the server serves (parallel to builder above). +# Bundlers read SDK src/ (trueforge-dev); the UI prebuild emits the declarations +# tsc needs, so this stage never compiles SDK JavaScript. +# --------------------------------------------------------------------------- +FROM workspace AS frontend-builder +RUN pnpm install --frozen-lockfile --offline --filter frontend... +COPY packages/trueforge-sdk packages/trueforge-sdk +COPY packages/assistant-ui-runtime packages/assistant-ui-runtime +COPY packages/trueforge-ui packages/trueforge-ui +RUN pnpm --filter @truefoundry/trueforge-ui build +COPY packages/frontend packages/frontend +RUN pnpm --filter frontend build + +# --------------------------------------------------------------------------- +# prod-deps: production dependency tree (no dev tooling), resolved offline. +# --------------------------------------------------------------------------- +FROM workspace AS prod-deps +RUN pnpm install --frozen-lockfile --offline --prod --filter @truefoundry/trueforge... + +# --------------------------------------------------------------------------- +# runner: minimal image with prod node_modules + built artifacts. +# --------------------------------------------------------------------------- +FROM base AS runner + +ENV NODE_ENV=production \ + HOST=0.0.0.0 + +# Production dependency tree (pnpm workspace symlinks preserved). +COPY --from=prod-deps /app/node_modules ./node_modules +COPY --from=prod-deps /app/packages/trueforge-core/node_modules ./packages/trueforge-core/node_modules +COPY --from=prod-deps /app/packages/trueforge/node_modules ./packages/trueforge/node_modules + +# Built workspace dependencies (@truefoundry/trueforge-core + SDK). +COPY --from=builder /app/packages/trueforge-core/package.json ./packages/trueforge-core/package.json +COPY --from=builder /app/packages/trueforge-core/dist ./packages/trueforge-core/dist +COPY --from=builder /app/packages/trueforge-sdk/package.json ./packages/trueforge-sdk/package.json +COPY --from=builder /app/packages/trueforge-sdk/dist ./packages/trueforge-sdk/dist + +# Built server (JS). UI is copied below from the parallel frontend stage into +# dist/_frontend - same path as the npm tarball / `pnpm build` copy step. +COPY --from=builder /app/packages/trueforge/package.json ./packages/trueforge/package.json +COPY --from=builder /app/packages/trueforge/dist ./packages/trueforge/dist +# Frontend builds in a parallel stage; place it at the same path as the npm tarball. +COPY --from=frontend-builder /app/packages/frontend/dist ./packages/trueforge/dist/_frontend + +WORKDIR /app/packages/trueforge + +RUN groupadd --gid 10001 trueforge \ + && useradd --uid 10001 --gid trueforge trueforge + +EXPOSE 8790 + +# Launch-only (matches root `pnpm start` / `standalone:start`). Image already contains dist. +USER 10001:10001 +CMD ["node", "dist/main.js"] diff --git a/Dockerfile.npm b/Dockerfile.npm new file mode 100644 index 000000000..592593c88 --- /dev/null +++ b/Dockerfile.npm @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1 +# +# npm-install production image (the previous root Dockerfile). Prod Helm, +# Compose, and Railway now build the from-source root Dockerfile instead. +# +# Required build-arg: +# APP_VERSION - npm version to install, e.g. 0.1.0 +# +# Example: +# docker build -f Dockerfile.npm --build-arg APP_VERSION=0.1.0 -t trueforge:0.1.0 . + +FROM node:24-slim AS runner +WORKDIR /app +# HOST=0.0.0.0 so Kubernetes Service/probe traffic reaches the process. +ENV NODE_ENV=production \ + STANDALONE=false \ + HOST=0.0.0.0 + +ARG APP_VERSION +RUN test -n "$APP_VERSION" || (echo "APP_VERSION build-arg is required" >&2 && exit 1) + +# Fail closed if the version is not on the registry (no workspace fallback). +RUN npm install --omit=dev "@truefoundry/trueforge@${APP_VERSION}" \ + && npm cache clean --force + +RUN groupadd --gid 10001 trueforge \ + && useradd --uid 10001 --gid trueforge --shell /usr/sbin/nologin trueforge + +EXPOSE 8790 + +# Same entry as the from-source image / `pnpm start` (launch-only; dist is in the package). +USER 10001:10001 +CMD ["node", "node_modules/@truefoundry/trueforge/dist/main.js"] diff --git a/RELEASING.md b/RELEASING.md index 60870dbf2..db43bafb8 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,11 +5,11 @@ sandbox image, and optional from-source **dev** images. | What | Trigger | Workflow | | ----------------------- | ------------------------------------------------------------- | -------------------------------------------------------------------- | -| npm packages | Push to `main` (Changesets) | [`release-packages.yml`](.github/workflows/release-packages.yml) | -| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release-packages.yml`](.github/workflows/release-packages.yml) | -| Prod image + Helm chart | Push of `chart/v*` tag from the package workflow, or dispatch | [`release-chart.yml`](.github/workflows/release-chart.yml) | +| npm packages | Push to `main` (Changesets) | [`release.yml`](.github/workflows/release.yml) | +| PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) | +| Prod image + Helm chart | Dispatch from the package workflow, or manual dispatch | [`release-chart.yml`](.github/workflows/release-chart.yml) | | Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | -| Dev (from-source) image | Manual `workflow_dispatch` | [`build-dev-image.yml`](.github/workflows/build-dev-image.yml) | +| From-source image | Manual `workflow_dispatch` | [`build-image.yml`](.github/workflows/build-image.yml) | | PR checks | Pull request / merge group | [`ci.yml`](.github/workflows/ci.yml) | ## Versioning @@ -47,8 +47,8 @@ helm install trueforge oci://tfy.jfrog.io/tfy-helm/trueforge --version `). SDK regen already adds @@ -58,10 +58,9 @@ is packages plus a chart tag (`select-mode` → `version` \| `pack` → npm + Py `scripts/version.mjs` mirrors that version into `python/trueforge_sdk` and regenerates both SDKs. Review and merge. 3. With no pending changesets, **pack** (build/test) and **Windows npx smoke** - run in parallel, then **npm publish**, **PyPI publish**, and **Create release tag** - (`chart/v`) run in parallel. PyPI skips when that - `pyproject.toml` version is already published. The tag is pushed with the - GitHub App token so it can start the image + Helm workflow. + run in parallel, then **npm publish**, **PyPI publish**, and **Start image and Helm release** + run in parallel. PyPI skips when that `pyproject.toml` version is already + published. The chart workflow is dispatched with the GitHub App token. 4. Pin dependents to exact versions during early `0.x`. `workflow_dispatch` on **Version or publish packages** re-runs the same workflow. @@ -83,11 +82,11 @@ Repo-wide via `.changeset/pre.json` (absent = publish to `latest`): Each public **npm** package must list this repo + workflow as a trusted publisher on npmjs.com: - Repository: `truefoundry/trueforge` -- Workflow: `release-packages.yml` (exact filename) +- Workflow: `release.yml` (exact filename) - No GitHub Environment name -This filename used to be `release.yml`. Update every package on npmjs.com (and -PyPI below) before the first publish from this workflow, or OIDC will 403. +Do not rename this file without updating every package on npmjs.com (and PyPI +below), or OIDC will 403. Do not set `NPM_TOKEN` / `_authToken` on the npm publish job - that disables OIDC. Only the **publish** / **publish-python** jobs use OIDC (`id-token: write`). @@ -99,7 +98,7 @@ the source repo and commit on npmjs.com. **PyPI** `trueforge-sdk` uses the same workflow file via a trusted publisher: - Repository: `truefoundry/trueforge` -- Workflow: `release-packages.yml` (exact filename) +- Workflow: `release.yml` (exact filename) - No Environment name (unless you add one to the job and mirror it on PyPI) - Create the project once on PyPI (or publish the first version), then add the pending/trusted publisher before the first OIDC upload succeeds. @@ -116,15 +115,15 @@ pnpm clean && pnpm build && pnpm standalone:start - **No Version Packages PR** - no `.changeset/*.md` on `main`. Add one, or re-run **Version or publish packages**. - **Publish wants a tag** - RCs need the `rc` dist-tag (set automatically while `pre.json` exists). -- **403** - version already on npm, or trusted-publisher config mismatch (filename must be `release-packages.yml`). +- **403** - version already on npm, or trusted-publisher config mismatch (filename must be `release.yml`). - **OIDC fail** - pnpm >= 11.0.7; remove registry `_authToken`. - **Missing `dist/_frontend/index.html`** - root `pnpm build` must build `frontend` first. - **SDK not regenerated on Version PR** - only when `@truefoundry/trueforge-sdk` version moved (`scripts/version.mjs`; needs Docker). That path also mirrors the version into `python/trueforge_sdk`. - **PyPI 403 / invalid-publisher** - register a trusted publisher for `trueforge-sdk` bound to - `release-packages.yml` (and create the project if it does not exist yet). -- **Prod image / chart missing after package publish** - confirm `chart/v` was - pushed, or dispatch the chart workflow on a **branch or tag** (not a SHA): + `release.yml` (and create the project if it does not exist yet). +- **Prod image / chart missing after package publish** - dispatch the chart + workflow on a **branch or tag** (not a SHA): `gh workflow run release-chart.yml --ref main`. --- @@ -133,44 +132,46 @@ pnpm clean && pnpm build && pnpm standalone:start ```text push to main (no pending changesets, unpublished versions) - → release-packages.yml: pack + smoke - → npm | PyPI | chart/v tag (parallel) - → tag push starts release-chart.yml + → release.yml: pack + smoke + → npm | PyPI | dispatch release-chart.yml (parallel) + → release-chart.yml → build from-source Dockerfile → push X.Y.Z- → helm lint/package/push OCI → commit Chart.yaml + values.yaml to main + → tag charts/trueforge@ on that commit manual rebuild - → workflow_dispatch release-chart.yml --ref main (or an existing chart/v* tag) + → workflow_dispatch release-chart.yml --ref main ``` -The package workflow pushes the tag with the GitHub App token. `GITHUB_TOKEN` -tag pushes do not start other workflows. +The package workflow starts the chart workflow with the GitHub App token. +`GITHUB_TOKEN` cannot dispatch other workflows. ## Dockerfile -| File | Role | -| -------------------------- | ------------------------------------------------------------------------------------------------------------------- | -| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build SHA-tagged from-source image** | +| File | Role | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------- | +| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build image** | +| [`Dockerfile.dev`](Dockerfile.dev) | Same as `Dockerfile`. Kept for Railway services that still set `RAILWAY_DOCKERFILE_PATH=Dockerfile.dev` | +| [`Dockerfile.npm`](Dockerfile.npm) | Previous npm-install image (`APP_VERSION` from the registry) | -The image is the workspace at the tagged commit. Chart `appVersion` is that -commit's `packages/trueforge/package.json` version. A `chart/v*` tag push fails -if the version after `chart/v` does not match that package version. +The image is the workspace at the dispatched commit. Chart `appVersion` is that +commit's `packages/trueforge/package.json` version. Image tags use the peeled +commit SHA (`git rev-parse HEAD`), not an annotated-tag object. ## Build server image and publish Helm chart -[`release-chart.yml`](.github/workflows/release-chart.yml) (`chart/v*` tag, or -`workflow_dispatch`). +[`release-chart.yml`](.github/workflows/release-chart.yml) (`workflow_dispatch`). -| Input | Default | Meaning | -| ------------- | --------------------------------- | -------------------------------------------------------------- | -| `app_version` | `packages/trueforge/package.json` | Used on dispatch; ignored when the trigger is a `chart/v*` tag | +| Input | Default | Meaning | +| ------------- | --------------------------------- | -------------------------------------------- | +| `app_version` | `packages/trueforge/package.json` | Version used in the image tag and appVersion | Always: build/push `{appVersion}-{shortSha}`, replay chart `version` / -`appVersion` / `image.tag` / controller command onto current `main`, lint, -package, push OCI (idempotent if that chart version is already in the registry), -then commit those files to `main`. The commit is replayed onto `origin/main` if -`main` moved during the image build. +`appVersion` / `image.tag` onto current `main`, lint, package, push OCI +(idempotent if that chart version is already in the registry), commit those +files to `main`, then tag `charts/trueforge@` on that commit. +The commit is replayed onto `origin/main` if `main` moved during the image build. ```bash gh workflow run release-chart.yml --ref main @@ -186,7 +187,7 @@ still advance per chart release. External deploy repo owns `truefoundry.yaml` (`git-helm-repo` @ `main`). Build a from-source image: ```bash -gh workflow run build-dev-image.yml --ref main +gh workflow run build-image.yml --ref main # → tfy.jfrog.io/tfy-images/trueforge: ``` diff --git a/charts/trueforge/Chart.yaml b/charts/trueforge/Chart.yaml index 67f2ae634..30dd2af6c 100644 --- a/charts/trueforge/Chart.yaml +++ b/charts/trueforge/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: trueforge description: TrueForge server (API + UI) served from a single container image. type: application -# version / appVersion / image.tag are written on main by release-chart.yml -# after the chart/v* tag from release-packages.yml. +# version / appVersion / image.tag are written on main by release-chart.yml, +# which then tags charts/trueforge@ on that commit. version: "0.2.3-rc.0" appVersion: "0.3.0-rc.0" kubeVersion: ">=1.25.0-0" diff --git a/charts/trueforge/README.md b/charts/trueforge/README.md index ac6028959..97801c52d 100644 --- a/charts/trueforge/README.md +++ b/charts/trueforge/README.md @@ -8,9 +8,10 @@ and the UI. Production, Compose smoke, and Railway all build the repository-root The chart always runs the server in **distributed** mode (`STANDALONE=false`) against Postgres and Redis. -Chart `version` / `appVersion` / `image.tag` / controller command are written -on `main` by [`release-chart.yml`](../../.github/workflows/release-chart.yml) -when a `chart/v*` tag is pushed. See [`RELEASING.md`](../../RELEASING.md). +Chart `version` / `appVersion` / `image.tag` are written on `main` by +[`release-chart.yml`](../../.github/workflows/release-chart.yml). That workflow +then tags `charts/trueforge@` on the metadata commit. See +[`RELEASING.md`](../../RELEASING.md). ## Dev defaults (read before exposing) diff --git a/charts/trueforge/values.yaml b/charts/trueforge/values.yaml index 4cc6e94f3..6d33b72cc 100644 --- a/charts/trueforge/values.yaml +++ b/charts/trueforge/values.yaml @@ -117,11 +117,10 @@ controller: enabled: true # Annotations on the controller Deployment object (for example, Argo CD sync waves). deploymentAnnotations: {} - # Matches the pinned image.tag. release-chart.yml rewrites this to - # dist/controller-main.js when it publishes a from-source image. + # Container command. Image WORKDIR is /app/packages/trueforge (from-source Dockerfile). command: - node - - node_modules/@truefoundry/trueforge/dist/controller-main.js + - dist/controller-main.js # Base URL the controller uses to reach the server (SERVER_URL). # Empty → the in-cluster server Service (http(s)://:). serverUrl: "" From 8ceca3324e40d21b01471436884769650f34ab49 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Thu, 24 Sep 2026 14:50:32 +0530 Subject: [PATCH 6/7] Remove the unused SHA-tagged image workflow. Production images come from release-chart.yml; devtest pins a git SHA and builds from source. Nothing else started build-image.yml. Signed-off-by: Raman Tehlan --- .github/workflows/build-image.yml | 54 ------------------------------- Dockerfile | 4 +-- Dockerfile.dev | 4 +-- RELEASING.md | 22 ++++--------- 4 files changed, 11 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/build-image.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml deleted file mode 100644 index df970db1b..000000000 --- a/.github/workflows/build-image.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build image - -# From-source image for floating-main / external deploy repos. -# Builds the root Dockerfile and pushes .../trueforge:. No chart publish. -on: - workflow_dispatch: - -concurrency: - group: build-image - cancel-in-progress: false - -permissions: - contents: read - -jobs: - build: - name: Build and push from-source image - uses: truefoundry/github-workflows-public/.github/workflows/build.yml@main - permissions: - id-token: write - contents: read - with: - artifactory_registry_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_REGISTRY_URL }} - artifactory_repository_url: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }} - image_artifact_name: trueforge - image_tag: ${{ github.sha }} - image_context: . - dockerfile_path: Dockerfile - platforms: linux/amd64 - enable_jfrog: true - enable_public_ecr: false - secrets: - artifactory_username: ${{ secrets.TRUEFORGE_ARTIFACTORY_USERNAME }} - artifactory_password: ${{ secrets.TRUEFORGE_ARTIFACTORY_PASSWORD }} - - summary: - name: Print image URI - needs: [build] - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - name: Summary - env: - IMAGE_URI: ${{ vars.TRUEFOUNDRY_ARTIFACTORY_PUBLIC_REPOSITORY }}/trueforge:${{ github.sha }} - run: | - set -euo pipefail - echo "Pushed image URI: $IMAGE_URI" - { - echo "### Image (from-source)" - echo "\`$IMAGE_URI\`" - echo "" - echo "Patch this SHA into your external \`truefoundry.yaml\` as \`image.tag\`." - } >> "$GITHUB_STEP_SUMMARY" diff --git a/Dockerfile b/Dockerfile index 8f353fcb5..a126843d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1 # -# From-source multi-stage image for prod Helm, local smoke, Railway, and -# SHA-tagged floating-main builds. Lives at the repository root because the +# From-source multi-stage image for prod Helm, local smoke, and Railway. +# Lives at the repository root because the # build needs the whole pnpm workspace as its context: the server depends on # the workspace package @truefoundry/trueforge-core. # diff --git a/Dockerfile.dev b/Dockerfile.dev index 5114a5cf0..ed51c6482 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -4,8 +4,8 @@ # RAILWAY_DOCKERFILE_PATH=Dockerfile.dev keep building. New applies use # the root Dockerfile by default. # -# From-source multi-stage image for prod Helm, local smoke, Railway, and -# SHA-tagged floating-main builds. Lives at the repository root because the +# From-source multi-stage image for prod Helm, local smoke, and Railway. +# Lives at the repository root because the # build needs the whole pnpm workspace as its context: the server depends on # the workspace package @truefoundry/trueforge-core. # diff --git a/RELEASING.md b/RELEASING.md index db43bafb8..795c02ccf 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,7 +1,7 @@ # Releasing -This repo ships npm packages, a production container image, a Helm chart, a -sandbox image, and optional from-source **dev** images. +This repo ships npm packages, a production container image, a Helm chart, and a +sandbox image. | What | Trigger | Workflow | | ----------------------- | ------------------------------------------------------------- | -------------------------------------------------------------------- | @@ -9,7 +9,6 @@ sandbox image, and optional from-source **dev** images. | PyPI `trueforge-sdk` | Same `mode=publish` run as npm (parallel OIDC job) | [`release.yml`](.github/workflows/release.yml) | | Prod image + Helm chart | Dispatch from the package workflow, or manual dispatch | [`release-chart.yml`](.github/workflows/release-chart.yml) | | Sandbox image + pin PR | Push to `main` when `scripts/sandbox/**` changes, or dispatch | [`push-sandbox-image.yml`](.github/workflows/push-sandbox-image.yml) | -| From-source image | Manual `workflow_dispatch` | [`build-image.yml`](.github/workflows/build-image.yml) | | PR checks | Pull request / merge group | [`ci.yml`](.github/workflows/ci.yml) | ## Versioning @@ -23,7 +22,6 @@ sandbox image, and optional from-source **dev** images. | Prod image tag | `{packageVersion}-{shortSha}` | | Chart `version` | Same major.minor as `@truefoundry/trueforge`; patch/RC may still advance | | Sandbox image | [`sandbox.Dockerfile`](packages/trueforge-core/scripts/sandbox/sandbox.Dockerfile); tag = full commit SHA | -| Dev image | Same [`Dockerfile`](Dockerfile); tag = full commit SHA | Install a published chart: @@ -151,7 +149,7 @@ The package workflow starts the chart workflow with the GitHub App token. | File | Role | | ---------------------------------- | ------------------------------------------------------------------------------------------------------- | -| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway, **Build image** | +| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway | | [`Dockerfile.dev`](Dockerfile.dev) | Same as `Dockerfile`. Kept for Railway services that still set `RAILWAY_DOCKERFILE_PATH=Dockerfile.dev` | | [`Dockerfile.npm`](Dockerfile.npm) | Previous npm-install image (`APP_VERSION` from the registry) | @@ -182,17 +180,11 @@ Chart major.minor is taken from [`scripts/resolve-chart-version.sh`](scripts/res so it matches `@truefoundry/trueforge` (and the docker tag prefix). Patch and RC still advance per chart release. -## Dev / floating main +## Devtest -External deploy repo owns `truefoundry.yaml` (`git-helm-repo` @ `main`). Build a from-source image: - -```bash -gh workflow run build-image.yml --ref main -# → tfy.jfrog.io/tfy-images/trueforge: -``` - -Patch that SHA into `image.tag`. Secrets via `secretKeyRef` only - never plaintext in git. -Do not use SHA-tagged images as production chart defaults. +[`deploy-devtest.yml`](.github/workflows/deploy-devtest.yml) pins the current +`main` SHA into `truefoundry/trueforge-devtest-deployment`. That environment +builds from source. Secrets via `secretKeyRef` only - never plaintext in git. ## Bundled chart dependencies From b9914d9437dffbf1d3a72a3869f5184ffca3aac0 Mon Sep 17 00:00:00 2001 From: Raman Tehlan Date: Thu, 24 Sep 2026 15:04:15 +0530 Subject: [PATCH 7/7] Remove Dockerfile.dev. Railway now uses the root from-source Dockerfile. The duplicate copy is no longer needed. Signed-off-by: Raman Tehlan --- .railway/railway.ts | 4 +- Dockerfile.dev | 119 -------------------------------------------- RELEASING.md | 9 ++-- 3 files changed, 5 insertions(+), 127 deletions(-) delete mode 100644 Dockerfile.dev diff --git a/.railway/railway.ts b/.railway/railway.ts index 60678f26d..288be706b 100644 --- a/.railway/railway.ts +++ b/.railway/railway.ts @@ -28,9 +28,7 @@ export default defineRailway(_ctx => { // Deploys from this repository's default branch. Forks: change owner/repo // (and optionally branch) to build your own copy. source: github('truefoundry/trueforge'), - // Railway uses the root from-source Dockerfile by default. Existing - // services may still set RAILWAY_DOCKERFILE_PATH=Dockerfile.dev; that file - // is kept as a copy of Dockerfile until IaC is re-applied. + // Railway uses the root from-source Dockerfile by default. healthcheck: '/healthz', healthcheckTimeout: 300, deploy: { diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index ed51c6482..000000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,119 +0,0 @@ -# syntax=docker/dockerfile:1 -# -# Same as Dockerfile. Kept so existing Railway services that still set -# RAILWAY_DOCKERFILE_PATH=Dockerfile.dev keep building. New applies use -# the root Dockerfile by default. -# -# From-source multi-stage image for prod Helm, local smoke, and Railway. -# Lives at the repository root because the -# build needs the whole pnpm workspace as its context: the server depends on -# the workspace package @truefoundry/trueforge-core. -# -# Dependency install uses pnpm fetch (lockfile-only) then install --offline so -# the download layer stays cached when only package.json / scripts change. -# See https://pnpm.io/cli/fetch. BuildKit cache mounts are avoided so the same -# file builds on Railway Metal (which requires a hardcoded service id in mount ids). - -FROM node:24-slim AS base -ENV PNPM_HOME=/pnpm -ENV PATH="$PNPM_HOME:$PATH" -RUN corepack enable && pnpm config set store-dir /pnpm/store -WORKDIR /app - -# --------------------------------------------------------------------------- -# store: the pnpm store, from the lockfile only (stable when manifests churn). -# BuildKit cache mounts are omitted: Railway's Metal builder requires -# `id=s/-…` (hardcoded per deploy), which cannot live in a shared -# OSS Dockerfile. Layer cache on this stage still hits when the lockfile is -# unchanged. -# --------------------------------------------------------------------------- -FROM base AS store -COPY pnpm-lock.yaml pnpm-workspace.yaml ./ -RUN pnpm fetch - -# --------------------------------------------------------------------------- -# workspace: install inputs shared by every stage below - the manifests plus the -# sources the root postinstall hook (build:gen) inlines. -# --------------------------------------------------------------------------- -FROM store AS workspace -COPY package.json .npmrc tsconfig.base.json ./ -# Root scripts used by package build steps (rm-path.mjs, chmod-path.mjs). -COPY scripts scripts -COPY packages/trueforge-core/package.json packages/trueforge-core/package.json -COPY packages/trueforge/package.json packages/trueforge/package.json -COPY packages/trueforge-sdk/package.json packages/trueforge-sdk/package.json -COPY packages/frontend/package.json packages/frontend/package.json -COPY packages/trueforge-ui/package.json packages/trueforge-ui/package.json -COPY packages/assistant-ui-runtime/package.json packages/assistant-ui-runtime/package.json -COPY packages/trueforge-core/scripts packages/trueforge-core/scripts -COPY packages/trueforge-core/src/core/sandbox/scripts packages/trueforge-core/src/core/sandbox/scripts - -# --------------------------------------------------------------------------- -# builder: install all deps (incl. dev) and build trueforge-core + server. -# SDK dist is copied into the production runner; host-dev/typecheck resolve SDK -# from src/ via trueforge-dev. Build it here once for the image. -# --------------------------------------------------------------------------- -FROM workspace AS builder -RUN pnpm install --frozen-lockfile --offline --filter @truefoundry/trueforge... -COPY packages/trueforge-core packages/trueforge-core -COPY packages/trueforge-sdk packages/trueforge-sdk -RUN pnpm --filter @truefoundry/trueforge-sdk build -COPY packages/trueforge packages/trueforge -RUN pnpm --filter @truefoundry/trueforge-core build && pnpm --filter @truefoundry/trueforge build - -# --------------------------------------------------------------------------- -# frontend-builder: build the UI the server serves (parallel to builder above). -# Bundlers read SDK src/ (trueforge-dev); the UI prebuild emits the declarations -# tsc needs, so this stage never compiles SDK JavaScript. -# --------------------------------------------------------------------------- -FROM workspace AS frontend-builder -RUN pnpm install --frozen-lockfile --offline --filter frontend... -COPY packages/trueforge-sdk packages/trueforge-sdk -COPY packages/assistant-ui-runtime packages/assistant-ui-runtime -COPY packages/trueforge-ui packages/trueforge-ui -RUN pnpm --filter @truefoundry/trueforge-ui build -COPY packages/frontend packages/frontend -RUN pnpm --filter frontend build - -# --------------------------------------------------------------------------- -# prod-deps: production dependency tree (no dev tooling), resolved offline. -# --------------------------------------------------------------------------- -FROM workspace AS prod-deps -RUN pnpm install --frozen-lockfile --offline --prod --filter @truefoundry/trueforge... - -# --------------------------------------------------------------------------- -# runner: minimal image with prod node_modules + built artifacts. -# --------------------------------------------------------------------------- -FROM base AS runner - -ENV NODE_ENV=production \ - HOST=0.0.0.0 - -# Production dependency tree (pnpm workspace symlinks preserved). -COPY --from=prod-deps /app/node_modules ./node_modules -COPY --from=prod-deps /app/packages/trueforge-core/node_modules ./packages/trueforge-core/node_modules -COPY --from=prod-deps /app/packages/trueforge/node_modules ./packages/trueforge/node_modules - -# Built workspace dependencies (@truefoundry/trueforge-core + SDK). -COPY --from=builder /app/packages/trueforge-core/package.json ./packages/trueforge-core/package.json -COPY --from=builder /app/packages/trueforge-core/dist ./packages/trueforge-core/dist -COPY --from=builder /app/packages/trueforge-sdk/package.json ./packages/trueforge-sdk/package.json -COPY --from=builder /app/packages/trueforge-sdk/dist ./packages/trueforge-sdk/dist - -# Built server (JS). UI is copied below from the parallel frontend stage into -# dist/_frontend - same path as the npm tarball / `pnpm build` copy step. -COPY --from=builder /app/packages/trueforge/package.json ./packages/trueforge/package.json -COPY --from=builder /app/packages/trueforge/dist ./packages/trueforge/dist -# Frontend builds in a parallel stage; place it at the same path as the npm tarball. -COPY --from=frontend-builder /app/packages/frontend/dist ./packages/trueforge/dist/_frontend - -WORKDIR /app/packages/trueforge - -RUN groupadd --gid 10001 trueforge \ - && useradd --uid 10001 --gid trueforge trueforge - -EXPOSE 8790 - -# Launch-only (matches root `pnpm start` / `standalone:start`). Image already contains dist. -USER 10001:10001 -CMD ["node", "dist/main.js"] diff --git a/RELEASING.md b/RELEASING.md index 795c02ccf..c3248816e 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -147,11 +147,10 @@ The package workflow starts the chart workflow with the GitHub App token. ## Dockerfile -| File | Role | -| ---------------------------------- | ------------------------------------------------------------------------------------------------------- | -| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway | -| [`Dockerfile.dev`](Dockerfile.dev) | Same as `Dockerfile`. Kept for Railway services that still set `RAILWAY_DOCKERFILE_PATH=Dockerfile.dev` | -| [`Dockerfile.npm`](Dockerfile.npm) | Previous npm-install image (`APP_VERSION` from the registry) | +| File | Role | +| ---------------------------------- | --------------------------------------------------------------------------- | +| [`Dockerfile`](Dockerfile) | From-source. Prod Helm, [`docker-compose.yml`](docker-compose.yml), Railway | +| [`Dockerfile.npm`](Dockerfile.npm) | Previous npm-install image (`APP_VERSION` from the registry) | The image is the workspace at the dispatched commit. Chart `appVersion` is that commit's `packages/trueforge/package.json` version. Image tags use the peeled