diff --git a/.github/workflows/_ci-detect-changes.reusable.yml b/.github/workflows/_ci-detect-changes.reusable.yml index 765783e9..7fd17009 100644 --- a/.github/workflows/_ci-detect-changes.reusable.yml +++ b/.github/workflows/_ci-detect-changes.reusable.yml @@ -33,6 +33,9 @@ on: run_e2e: description: 'Whether to run E2E tests' value: ${{ jobs.detect.outputs.run_e2e }} + run_live: + description: 'Whether to run tests against live external services' + value: ${{ jobs.detect.outputs.run_live }} permissions: contents: read @@ -50,6 +53,7 @@ jobs: run_config: ${{ steps.determine.outputs.run_config }} run_integration: ${{ steps.determine.outputs.run_integration }} run_e2e: ${{ steps.determine.outputs.run_e2e }} + run_live: ${{ steps.determine.outputs.run_live }} steps: - name: Checkout @@ -80,6 +84,14 @@ jobs: - 'test/e2e/**' - 'scripts/test-e2e.ts' - '.github/workflows/_ci-e2e.reusable.yml' + live: + - 'packages/notes/src/**' + - 'packages/notes/package.json' + - 'packages/notes/test/integration/ollama-live.spec.ts' + - 'releasekit.config.json' + - '.github/workflows/ci.yml' + - '.github/workflows/_ci-detect-changes.reusable.yml' + - '.github/workflows/_ci-live.reusable.yml' infra: - '.github/workflows/**' - '.github/actions/**' @@ -101,6 +113,7 @@ jobs: echo "run_config=true" >> $GITHUB_OUTPUT echo "run_integration=true" >> $GITHUB_OUTPUT echo "run_e2e=true" >> $GITHUB_OUTPUT + echo "run_live=true" >> $GITHUB_OUTPUT echo "::notice::Force all mode - running all tests" exit 0 fi @@ -112,6 +125,7 @@ jobs: CONFIG="${{ steps.changes.outputs.config }}" INTEGRATION="${{ steps.changes.outputs.integration }}" E2E="${{ steps.changes.outputs.e2e }}" + LIVE="${{ steps.changes.outputs.live }}" INFRA="${{ steps.changes.outputs.infra }}" RUN_VERSION="false" @@ -163,6 +177,7 @@ jobs: echo "run_config=$RUN_CONFIG" >> $GITHUB_OUTPUT echo "run_integration=$RUN_INTEGRATION" >> $GITHUB_OUTPUT echo "run_e2e=$RUN_E2E" >> $GITHUB_OUTPUT + echo "run_live=$LIVE" >> $GITHUB_OUTPUT echo "## Change Detection" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -175,4 +190,5 @@ jobs: echo "| config/core | $CONFIG | $RUN_CONFIG |" >> $GITHUB_STEP_SUMMARY echo "| integration | $INTEGRATION | $RUN_INTEGRATION |" >> $GITHUB_STEP_SUMMARY echo "| e2e | $E2E | $RUN_E2E |" >> $GITHUB_STEP_SUMMARY + echo "| live | $LIVE | $LIVE |" >> $GITHUB_STEP_SUMMARY echo "| infra | $INFRA | - |" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/_ci-live.reusable.yml b/.github/workflows/_ci-live.reusable.yml new file mode 100644 index 00000000..e41bbc13 --- /dev/null +++ b/.github/workflows/_ci-live.reusable.yml @@ -0,0 +1,43 @@ +name: Live Tests + +on: + workflow_call: + inputs: + cache_key: + required: true + type: string + secrets: + OLLAMA_API_KEY: + required: false + +permissions: + contents: read + +jobs: + live-tests: + name: Live Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v7 + + - name: Setup Development Environment + uses: ./.github/workflows/actions/setup-workspace + + - name: Download build artifacts + uses: ./.github/workflows/actions/download-archive + with: + name: releasekit-build + cache_key: ${{ inputs.cache_key }} + + # Not turbo: its strict env mode strips RELEASEKIT_NOTES_LIVE and the suite silently skips. + - name: Notes against live Ollama + run: | + RELEASEKIT_NOTES_LIVE_MODEL=$(jq -er '.notes.releaseNotes.llm.model' releasekit.config.json) + OLLAMA_BASE_URL=$(jq -er '.notes.releaseNotes.llm.baseURL' releasekit.config.json) + export RELEASEKIT_NOTES_LIVE_MODEL OLLAMA_BASE_URL + pnpm --filter @releasekit/notes test ollama-live + env: + RELEASEKIT_NOTES_LIVE: '1' + OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69a41edc..d6d9d027 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: build: name: Build needs: [detect-changes] - if: needs.detect-changes.outputs.has_changes == 'true' + if: needs.detect-changes.outputs.has_changes == 'true' || needs.detect-changes.outputs.run_live == 'true' uses: ./.github/workflows/_ci-build.reusable.yml with: os: ubuntu-latest @@ -192,6 +192,21 @@ jobs: os: ${{ matrix.os }} cache_key: ${{ needs.build.outputs.cache_key }} + live-tests: + name: Live Tests + needs: [detect-changes, build] + # Skip forks (no secrets) and the standing release PR (version bumps only). + if: >- + needs.detect-changes.outputs.run_live == 'true' && + github.head_ref != 'release/next' && + github.repository == 'goosewobbler/releasekit' && + (github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository) + uses: ./.github/workflows/_ci-live.reusable.yml + with: + cache_key: ${{ needs.build.outputs.cache_key }} + secrets: + OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }} + ci-status: name: CI Status if: always() @@ -206,6 +221,7 @@ jobs: - integration-tests - package-tests - e2e-tests + - live-tests runs-on: ubuntu-latest steps: - name: Check CI results diff --git a/AGENTS.md b/AGENTS.md index d7481821..2d8b3c5a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ Design decisions are recorded in `docs/adr/` — check there before reworking a - **One package:** `pnpm --filter test` or `typecheck` skips Turborepo, so it runs against the dependencies' built `dist/`, which may be stale. Build them first with `pnpm turbo run build --filter=^...`. - **Lint:** a package's own `lint` script is Biome only; `pnpm lint` also runs ESLint. - **CI-only suites:** `pnpm test` doesn't run the action harness (`test:action:*`, `test:harness:*`), `scripts/test-e2e.ts` or `scripts/test-packages.ts`. Run them when you change a CLI, `action.yml` or what a package ships. +- **Live tests:** integration tests that call a real service, such as `packages/notes/test/integration/ollama-live.spec.ts`, skip unless their opt-in variable is set; CI runs them in the Live Tests job. A new one needs a step in `_ci-live.reusable.yml`, any new secret declared there and passed from `ci.yml`, and its package's paths in Detect Changes' `live` filter. ## Gotchas