From 1bfa966af23dde10babe9b1066efb5dc28ce88e2 Mon Sep 17 00:00:00 2001 From: Scott Kirkland Date: Fri, 16 Oct 2026 09:51:15 -0700 Subject: [PATCH 1/5] Automate test deployment and gate production with approval --- .github/workflows/ci.yml | 33 ++++++++- .github/workflows/deploy.yml | 12 ++-- docs/test-deployment.md | 129 ++++++++++++++++++++++++----------- 3 files changed, 127 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41fc2cb7..90f2f341 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,20 @@ -name: CI +name: CI/CD on: pull_request: branches: [master] push: branches: [master] + workflow_dispatch: + inputs: + environment: + description: GitHub Environment to deploy. + required: true + type: choice + default: test + options: + - test + - prod permissions: contents: read @@ -12,6 +22,7 @@ permissions: jobs: validate: name: Validate + if: github.event_name == 'pull_request' runs-on: ubuntu-22.04 timeout-minutes: 25 steps: @@ -52,3 +63,23 @@ jobs: - name: Test .NET projects run: dotnet test Payments.sln --configuration Release --no-build + + deploy_test: + name: Deploy test + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + uses: ./.github/workflows/deploy.yml + permissions: + contents: read + id-token: write + with: + environment: test + + deploy_manual: + name: Deploy ${{ inputs.environment }} + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' + uses: ./.github/workflows/deploy.yml + permissions: + contents: read + id-token: write + with: + environment: ${{ inputs.environment }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 1db3c7db..fb6d4726 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,13 +1,12 @@ name: Deploy on: - workflow_dispatch: + workflow_call: inputs: environment: description: GitHub Environment to deploy. required: true - type: environment - default: test + type: string permissions: contents: read @@ -16,8 +15,11 @@ permissions: jobs: deploy: name: Deploy ${{ inputs.environment }} - # Only test is enabled during this migration. Production stays in Azure Pipelines. - if: github.ref == 'refs/heads/master' && inputs.environment == 'test' + if: >- + github.ref == 'refs/heads/master' && + ((github.event_name == 'push' && inputs.environment == 'test') || + (github.event_name == 'workflow_dispatch' && + (inputs.environment == 'test' || inputs.environment == 'prod'))) runs-on: ubuntu-22.04 timeout-minutes: 30 environment: ${{ inputs.environment }} diff --git a/docs/test-deployment.md b/docs/test-deployment.md index 2fb27eee..9f02cf6c 100644 --- a/docs/test-deployment.md +++ b/docs/test-deployment.md @@ -1,13 +1,24 @@ # GitHub Actions deployment -The manual `Deploy` workflow follows the [web-app-template deployment workflow](https://github.com/ucdavis/web-app-template/blob/main/.github/workflows/deploy-azure-appservice.yml). -Choose a GitHub environment when running it from `master`. Only `test` is enabled -for this migration. Production continues through the existing Azure pipeline. +The `CI/CD` workflow follows the [web-app-template workflow pattern](https://github.com/ucdavis/web-app-template/blob/main/.github/workflows/ci-cd.yml) +and calls the shared `Deploy` workflow. + +| Trigger | Result | +| -------------------------------------------------- | ------------------------------------------------------------- | +| Pull request to `master` | Build and test, without deployment | +| Push to `master`, including a merge | Build, test, and deploy to `test` | +| Manual `CI/CD` run on `master`, environment `test` | Build, test, and deploy to test | +| Manual `CI/CD` run on `master`, environment `prod` | Wait for approval, then build, test, and deploy to production | + +Production is requested separately; a successful test deployment does not start +a production deployment. Both environments allow only the `master` branch. +Deployments are serialized per environment, without cancelling an active run. ## Build and publish -CI builds the frontend and .NET projects and runs their tests using .NET 6 and -Node 22. Deployment repeats those steps, then publishes directly to `publish/web`. +Pull request validation and deployment both build the frontend and .NET projects +and run their tests using .NET 6 and Node 22. On pushes and manual requests, the +deployment job performs these checks before publishing directly to `publish/web`. Payments' existing publish target includes the frontend in the web output. The deploy workflow also publishes both WebJobs into the web directory before @@ -22,10 +33,9 @@ The same job logs into Azure with OIDC, checks SCM readiness, deploys the ZIP us The health endpoint performs a read-only database connection check and returns `Healthy` or `Unhealthy`, without connection details. -## GitHub environment +## GitHub environments -The `test` environment and deployment identity already exist. The environment -restricts deployment to `master` and supplies these variables: +Each GitHub environment supplies its own values for these variables: | Variable | Purpose | | ---------------------------- | --------------------------------------------- | @@ -36,39 +46,67 @@ restricts deployment to `master` and supplies these variables: | `WEB_APP_NAME` | Existing App Service name | | `APP_HEALTH_TIMEOUT_SECONDS` | Optional health-check timeout, defaults to 60 | -The test identity is `id-payments-test-deploy`, with Website Contributor scoped to -`payments-test` only. Its federated subject is -`repo:ucdavis/payments:environment:test`. No production identity or permissions are -part of this change. +| Environment | App Service and resource group | Deployment identity | Subscription | +| ----------- | ------------------------------ | ------------------------- | -------------------------- | +| `test` | `payments-test` | `id-payments-test-deploy` | Existing test subscription | +| `prod` | `payments-production` | `id-payments-prod-deploy` | UC Davis CAES Production | + +Each identity has Website Contributor scoped only to its App Service. The +federated subjects are `repo:ucdavis/payments:environment:test` and +`repo:ucdavis/payments:environment:prod`; production does not reuse test access. + +The `prod` environment requires approval from `srkirkland`. Self-approval is +allowed, but administrator bypass is disabled. The shared deployment job binds +to this environment, so it cannot start or obtain its Azure OIDC token before +approval. A manual workflow request alone does not approve deployment. Application secrets and connection strings remain in Azure. Preserve Windows hosting, 32-bit workers, Always On, schedules, domains, certificates, and `ASPNETCORE_ENVIRONMENT=Production`. Do not change database migration behavior or enable run-from-package during this migration. -The earlier draft's `TEST_DEPLOYMENT_READY` variable is no longer used. Deployment -requires an explicit manual run; do not dispatch it until the handover below. - -## Test handover - -1. Review and merge the PR after CI passes. Coordinate other application changes so - the first deployment does not also introduce schema or financial logic changes. -2. Recheck the live test release and retain its artifact for rollback. The initial - baseline was Release-218 / build 15073 / commit - `e09c70b5d33f15f1cc3840757b3c263c2cb5ba6f`. Inspect both WebJobs and their histories. -3. In Azure classic release definition 1, change only Test's automatic trigger to - manual. Preserve the build pipeline, release artifact trigger, and Production - stage. Confirm no test release is running or queued, including older releases. -4. Check neither scheduled job is running. Between hourly runs and outside the - daily AutoApprove window, run `Deploy` on `master` with environment `test`. -5. Confirm deployment and health checks pass. Inspect deployed files and both - WebJobs, especially their schedules and Windows entry points. The old MSDeploy - task removed extra files and renamed locked files; inspect the first ZIP deploy - for obsolete files or locking issues without deleting runtime data. +The earlier draft's `TEST_DEPLOYMENT_READY` variable is unused and does not block +deployment. Azure classic release definition 1 keeps both Test and Production +manual. Leave Azure Test automatic deployment disabled to avoid competing +deployers. The existing Azure build and production release remain available for +recovery; GitHub's approval gate does not govern that separate Azure release path. + +## Test deployment + +Merging to `master` starts test deployment automatically. Review the resulting +commit and workflow run, confirm deployment and health, and inspect both +WebJobs' deployed entrypoints and schedules. After enabling this workflow, the +next test deployment includes merged application changes such as PR #564. + +For a retry, run `CI/CD` from `master` and choose `test`. Check for active Azure +releases before any manual retry or recovery. Prefer a merge or manual run +between hourly job runs and outside the daily AutoApprove window. + +## Production release + +1. Recheck the live production revision and retain its Azure build artifact and + deployment metadata for rollback. On October 16, 2026, production was + Release-217 / build 14794 / commit + `13cc9bb29a14551299022b58655b04cc909ce53e`. Build 15073 is a test rollback + artifact, not the production rollback artifact. +2. Review the full delta from production to the intended `master` commit, + including PR #556's API/auth changes and PR #564's receipt/AutoApprove changes. + Check configuration and migrations; application startup runs database + migrations. Verify test evidence for the intended revision and agree a release + window outside scheduled job activity. +3. Request a manual `CI/CD` run from `master` with environment `prod`. Confirm + the run's commit is the intended release and that it is waiting for approval. + The reviewer must approve that deployment explicitly. If the wrong revision + was selected, cancel the request and select the intended release again. +4. After approval, watch deployment and health checks to completion. Inspect + deployed files and WebJob contracts, then observe natural scheduled runs. + ZIP deployment differs from the old MSDeploy task's extra-file removal and + locked-file handling, so check for obsolete files or locking errors without + deleting runtime data. ## Read-only acceptance -For this first pass, inspect login, existing invoice pages, frontend assets, and +Inspect login, existing invoice pages, frontend assets, and safe authenticated reads where access is available. Observe the next scheduled MoneyMovement and AutoApprove runs and inspect application errors and job results. Do not manually run jobs or initiate payments, recharge, email, or webhooks. @@ -78,15 +116,24 @@ job success. Record any checks that still need credentials or the next scheduled run. The AutoApprove executable does not check `Finance:RechargeAutoApprove`, so that setting is not a job-disable switch. -Automatic GitHub deployment and production migration need a separate decision. +Workflow success and health checks do not establish the exact live revision. +Record the workflow commit separately from deployment, health, scheduled-job, +and authenticated functional evidence. ## Recovery -Stop further manual GitHub runs and let any active deployment finish. Redeploy -the retained known-good artifact through the existing Azure release's Test stage -only. Check login, deployed files, and both jobs. The older artifact does not have -`/health`, so use its existing application routes for verification. - -If abandoning the handover, restore Test's previous automatic trigger after -confirming no GitHub deployment is running or queued. Package rollback does not -undo database changes or job side effects. +For test recovery, disable the `CI/CD` workflow to prevent new automatic runs, +cancel queued deployments, and let any active deployment finish before using +Azure. Redeploy the retained known-good artifact through the existing Azure +release's Test stage. Keep GitHub deployment disabled until recovery is verified +and the deployment owner is agreed. + +For production recovery, obtain explicit release approval, cancel pending GitHub +production requests, and let any active deployment finish. Use the retained +current production artifact through the existing Azure Production stage. + +Check login, deployed files, and both jobs. Older artifacts may not have `/health`, +so use their existing application routes for verification. Restore Azure Test's +automatic trigger only if handing deployment ownership back to Azure and after +confirming GitHub cannot deploy concurrently. Package rollback does not undo +database changes or job side effects. From 0b4b3ef0df0480c5568b7729a914f15cc953e3b0 Mon Sep 17 00:00:00 2001 From: Scott Kirkland Date: Fri, 16 Oct 2026 10:06:00 -0700 Subject: [PATCH 2/5] no-mistakes(document): Correct README deployment pointer while preserving CRLF --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b018190a..09be5694 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,4 @@ Accept and process credit card payments for CA&ES ## Deployment -See [the test deployment runbook](docs/test-deployment.md) for the GitHub Actions pilot, handover checks, and rollback. Production continues through Azure Pipelines. +See [the deployment runbook](docs/test-deployment.md) for automatic test deployment, approval-gated production deployment, verification, and rollback. From 5dc153f45212e7cca709651de7f4fd1872af6d98 Mon Sep 17 00:00:00 2001 From: Scott Kirkland Date: Fri, 16 Oct 2026 10:45:47 -0700 Subject: [PATCH 3/5] Document production reviewer team --- docs/test-deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/test-deployment.md b/docs/test-deployment.md index 9f02cf6c..075deaef 100644 --- a/docs/test-deployment.md +++ b/docs/test-deployment.md @@ -55,8 +55,8 @@ Each identity has Website Contributor scoped only to its App Service. The federated subjects are `repo:ucdavis/payments:environment:test` and `repo:ucdavis/payments:environment:prod`; production does not reuse test access. -The `prod` environment requires approval from `srkirkland`. Self-approval is -allowed, but administrator bypass is disabled. The shared deployment job binds +The `prod` environment requires approval from `@ucdavis/caesdo-devs`. +Self-approval is allowed, but administrator bypass is disabled. The shared deployment job binds to this environment, so it cannot start or obtain its Azure OIDC token before approval. A manual workflow request alone does not approve deployment. From 26bee8a91baa6624054f8d7963efa0c992b45dbe Mon Sep 17 00:00:00 2001 From: Scott Kirkland Date: Fri, 16 Oct 2026 10:52:25 -0700 Subject: [PATCH 4/5] no-mistakes(test): Clarify either production reviewer can approve deployment --- docs/test-deployment.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/test-deployment.md b/docs/test-deployment.md index 075deaef..8a096f14 100644 --- a/docs/test-deployment.md +++ b/docs/test-deployment.md @@ -55,7 +55,8 @@ Each identity has Website Contributor scoped only to its App Service. The federated subjects are `repo:ucdavis/payments:environment:test` and `repo:ucdavis/payments:environment:prod`; production does not reuse test access. -The `prod` environment requires approval from `@ucdavis/caesdo-devs`. +The `prod` environment requires approval from either `srkirkland` or a member of +`@ucdavis/caesdo-devs`. Self-approval is allowed, but administrator bypass is disabled. The shared deployment job binds to this environment, so it cannot start or obtain its Azure OIDC token before approval. A manual workflow request alone does not approve deployment. From 90f9fb60ecbae1ee4bcc394f5bd671b0c560497c Mon Sep 17 00:00:00 2001 From: Scott Kirkland Date: Fri, 16 Oct 2026 11:30:59 -0700 Subject: [PATCH 5/5] Queue production approval after test deployment --- .github/workflows/ci.yml | 13 ++++++++++++ .github/workflows/deploy.yml | 5 ++--- README.md | 2 +- docs/test-deployment.md | 38 +++++++++++++++++++++++------------- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90f2f341..cea4cc17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,19 @@ jobs: with: environment: test + deploy_prod: + name: Deploy production + needs: deploy_test + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/master' && + needs.deploy_test.result == 'success' + uses: ./.github/workflows/deploy.yml + permissions: + contents: read + id-token: write + with: + environment: prod + deploy_manual: name: Deploy ${{ inputs.environment }} if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fb6d4726..fed62bd0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,9 +17,8 @@ jobs: name: Deploy ${{ inputs.environment }} if: >- github.ref == 'refs/heads/master' && - ((github.event_name == 'push' && inputs.environment == 'test') || - (github.event_name == 'workflow_dispatch' && - (inputs.environment == 'test' || inputs.environment == 'prod'))) + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && + (inputs.environment == 'test' || inputs.environment == 'prod') runs-on: ubuntu-22.04 timeout-minutes: 30 environment: ${{ inputs.environment }} diff --git a/README.md b/README.md index 09be5694..854bc9c4 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,4 @@ Accept and process credit card payments for CA&ES ## Deployment -See [the deployment runbook](docs/test-deployment.md) for automatic test deployment, approval-gated production deployment, verification, and rollback. +See [the deployment runbook](docs/test-deployment.md) for automatic test deployment followed by production approval in the same workflow run, verification, and rollback. diff --git a/docs/test-deployment.md b/docs/test-deployment.md index 8a096f14..904117ac 100644 --- a/docs/test-deployment.md +++ b/docs/test-deployment.md @@ -3,16 +3,22 @@ The `CI/CD` workflow follows the [web-app-template workflow pattern](https://github.com/ucdavis/web-app-template/blob/main/.github/workflows/ci-cd.yml) and calls the shared `Deploy` workflow. -| Trigger | Result | -| -------------------------------------------------- | ------------------------------------------------------------- | -| Pull request to `master` | Build and test, without deployment | -| Push to `master`, including a merge | Build, test, and deploy to `test` | -| Manual `CI/CD` run on `master`, environment `test` | Build, test, and deploy to test | -| Manual `CI/CD` run on `master`, environment `prod` | Wait for approval, then build, test, and deploy to production | - -Production is requested separately; a successful test deployment does not start -a production deployment. Both environments allow only the `master` branch. -Deployments are serialized per environment, without cancelling an active run. +| Trigger | Result | +| -------------------------------------------------- | -------------------------------------------------------------------------------------- | +| Pull request to `master` | Build and test, without deployment | +| Push to `master`, including a merge | Deploy to `test`; after success, wait for approval to deploy the same commit to `prod` | +| Manual `CI/CD` run on `master`, environment `test` | Build, test, and deploy to test | +| Manual `CI/CD` run on `master`, environment `prod` | Wait for approval, then build, test, and deploy to production | + +After a merge, production waits for approval in the same workflow run once test +deployment and its health checks succeed. A failed or cancelled test deployment +prevents that run from reaching production. No separate manual run is needed. +Both stages build the workflow's commit, even if `master` advances while approval +is pending. Both environments allow only the `master` branch. + +Manual runs remain available for targeted retries. A manual `test` run deploys +only test; a manual `prod` run still requires approval. Deployments are serialized +per environment, without cancelling an active deployment. ## Build and publish @@ -95,10 +101,14 @@ between hourly job runs and outside the daily AutoApprove window. Check configuration and migrations; application startup runs database migrations. Verify test evidence for the intended revision and agree a release window outside scheduled job activity. -3. Request a manual `CI/CD` run from `master` with environment `prod`. Confirm - the run's commit is the intended release and that it is waiting for approval. - The reviewer must approve that deployment explicitly. If the wrong revision - was selected, cancel the request and select the intended release again. +3. Open the merge's `CI/CD` run after test deployment succeeds. Confirm its + commit is the intended release and production is waiting for approval. + Click **Review deployments**, select `prod`, then **Approve and deploy**. + Approval starts the production build, tests, and deployment for that run's + commit. Reject obsolete pending production deployments; approving an older + run releases its older commit. For a separate production retry, manually run + `CI/CD` from `master`, choose `prod`, and approve that run after reviewing its + commit and test evidence. 4. After approval, watch deployment and health checks to completion. Inspect deployed files and WebJob contracts, then observe natural scheduled runs. ZIP deployment differs from the old MSDeploy task's extra-file removal and