ci: add a Live Tests job, starting with notes against Ollama Cloud

.github/workflows/_ci-detect-changes.reusable.yml

@@ -33,6 +33,9 @@ on:
       run_e2e:
         description: 'Whether to run E2E tests'
         value: ${{ jobs.detect.outputs.run_e2e }}
+      run_live:
+        description: 'Whether to run tests against live external services'
+        value: ${{ jobs.detect.outputs.run_live }}
 
 permissions:
   contents: read
@@ -50,6 +53,7 @@ jobs:
       run_config: ${{ steps.determine.outputs.run_config }}
       run_integration: ${{ steps.determine.outputs.run_integration }}
       run_e2e: ${{ steps.determine.outputs.run_e2e }}
+      run_live: ${{ steps.determine.outputs.run_live }}
 
     steps:
       - name: Checkout
@@ -80,6 +84,14 @@ jobs:
               - 'test/e2e/**'
               - 'scripts/test-e2e.ts'
               - '.github/workflows/_ci-e2e.reusable.yml'
+            live:
+              - 'packages/notes/src/**'
+              - 'packages/notes/package.json'
+              - 'packages/notes/test/integration/ollama-live.spec.ts'
+              - 'releasekit.config.json'
+              - '.github/workflows/ci.yml'
+              - '.github/workflows/_ci-detect-changes.reusable.yml'
+              - '.github/workflows/_ci-live.reusable.yml'
             infra:
               - '.github/workflows/**'
               - '.github/actions/**'
@@ -101,6 +113,7 @@ jobs:
             echo "run_config=true" >> $GITHUB_OUTPUT
             echo "run_integration=true" >> $GITHUB_OUTPUT
             echo "run_e2e=true" >> $GITHUB_OUTPUT
+            echo "run_live=true" >> $GITHUB_OUTPUT
             echo "::notice::Force all mode - running all tests"
             exit 0
           fi
@@ -112,6 +125,7 @@ jobs:
           CONFIG="${{ steps.changes.outputs.config }}"
           INTEGRATION="${{ steps.changes.outputs.integration }}"
           E2E="${{ steps.changes.outputs.e2e }}"
+          LIVE="${{ steps.changes.outputs.live }}"
           INFRA="${{ steps.changes.outputs.infra }}"
 
           RUN_VERSION="false"
@@ -163,6 +177,7 @@ jobs:
           echo "run_config=$RUN_CONFIG" >> $GITHUB_OUTPUT
           echo "run_integration=$RUN_INTEGRATION" >> $GITHUB_OUTPUT
           echo "run_e2e=$RUN_E2E" >> $GITHUB_OUTPUT
+          echo "run_live=$LIVE" >> $GITHUB_OUTPUT
 
           echo "## Change Detection" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
@@ -175,4 +190,5 @@ jobs:
           echo "| config/core | $CONFIG | $RUN_CONFIG |" >> $GITHUB_STEP_SUMMARY
           echo "| integration | $INTEGRATION | $RUN_INTEGRATION |" >> $GITHUB_STEP_SUMMARY
           echo "| e2e | $E2E | $RUN_E2E |" >> $GITHUB_STEP_SUMMARY
+          echo "| live | $LIVE | $LIVE |" >> $GITHUB_STEP_SUMMARY
           echo "| infra | $INFRA | - |" >> $GITHUB_STEP_SUMMARY

.github/workflows/_ci-live.reusable.yml

@@ -0,0 +1,43 @@
+name: Live Tests
+
+on:
+  workflow_call:
+    inputs:
+      cache_key:
+        required: true
+        type: string
+    secrets:
+      OLLAMA_API_KEY:
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  live-tests:
+    name: Live Tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v7
+
+      - name: Setup Development Environment
+        uses: ./.github/workflows/actions/setup-workspace
+
+      - name: Download build artifacts
+        uses: ./.github/workflows/actions/download-archive
+        with:
+          name: releasekit-build
+          cache_key: ${{ inputs.cache_key }}
+
+      # Not turbo: its strict env mode strips RELEASEKIT_NOTES_LIVE and the suite silently skips.
+      - name: Notes against live Ollama
+        run: |
+          RELEASEKIT_NOTES_LIVE_MODEL=$(jq -er '.notes.releaseNotes.llm.model' releasekit.config.json)
+          OLLAMA_BASE_URL=$(jq -er '.notes.releaseNotes.llm.baseURL' releasekit.config.json)
+          export RELEASEKIT_NOTES_LIVE_MODEL OLLAMA_BASE_URL
+          pnpm --filter @releasekit/notes test ollama-live
+        env:
+          RELEASEKIT_NOTES_LIVE: '1'
+          OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ jobs:
   build:
     name: Build
     needs: [detect-changes]
-    if: needs.detect-changes.outputs.has_changes == 'true'
+    if: needs.detect-changes.outputs.has_changes == 'true' || needs.detect-changes.outputs.run_live == 'true'
     uses: ./.github/workflows/_ci-build.reusable.yml
     with:
       os: ubuntu-latest
@@ -192,6 +192,21 @@ jobs:
       os: ${{ matrix.os }}
       cache_key: ${{ needs.build.outputs.cache_key }}
 
+  live-tests:
+    name: Live Tests
+    needs: [detect-changes, build]
+    # Skip forks (no secrets) and the standing release PR (version bumps only).
+    if: >-
+      needs.detect-changes.outputs.run_live == 'true' &&
+      github.head_ref != 'release/next' &&
+      github.repository == 'goosewobbler/releasekit' &&
+      (github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository)
+    uses: ./.github/workflows/_ci-live.reusable.yml
+    with:
+      cache_key: ${{ needs.build.outputs.cache_key }}
+    secrets:
+      OLLAMA_API_KEY: ${{ secrets.OLLAMA_API_KEY }}
+
   ci-status:
     name: CI Status
     if: always()
@@ -206,6 +221,7 @@ jobs:
       - integration-tests
       - package-tests
       - e2e-tests
+      - live-tests
     runs-on: ubuntu-latest
     steps:
       - name: Check CI results

AGENTS.md

@@ -21,6 +21,7 @@ Design decisions are recorded in `docs/adr/` — check there before reworking a
 - **One package:** `pnpm --filter <package> test` or `typecheck` skips Turborepo, so it runs against the dependencies' built `dist/`, which may be stale. Build them first with `pnpm turbo run build --filter=<package>^...`.
 - **Lint:** a package's own `lint` script is Biome only; `pnpm lint` also runs ESLint.
 - **CI-only suites:** `pnpm test` doesn't run the action harness (`test:action:*`, `test:harness:*`), `scripts/test-e2e.ts` or `scripts/test-packages.ts`. Run them when you change a CLI, `action.yml` or what a package ships.
+- **Live tests:** integration tests that call a real service, such as `packages/notes/test/integration/ollama-live.spec.ts`, skip unless their opt-in variable is set; CI runs them in the Live Tests job. A new one needs a step in `_ci-live.reusable.yml`, any new secret declared there and passed from `ci.yml`, and its package's paths in Detect Changes' `live` filter.
 
 ## Gotchas