diff --git a/.env.example b/.env.example index e50751c1..39e9599a 100644 --- a/.env.example +++ b/.env.example @@ -26,14 +26,41 @@ POSTIL_SESSION_SECRET= # settings page. Generate: openssl rand -hex 32 GITHUB_WEBHOOK_SECRET= -# Optional iLert outbound alert receiver. Generate a random password with -# `openssl rand -hex 32`, set it here, and configure the iLert webhook URL as -# https://postil-ilert:@/api/webhooks/ilert. -# Select alert-created, alert-assigned, alert-auto-escalated, -# alert-auto-resolved, alert-acknowledged, alert-rejected, alert-raised, and -# alert-resolved as trigger events. Leave unset to make the endpoint return 404. +# iLert webhook receiver password. Generate with `openssl rand -hex 32`. The +# reconciliation workflow loads this value from the secret record that also +# supplies the deployed receiver's runtime secret. The reconciler embeds +# URL-encoded HTTP Basic credentials for `postil-ilert` and this value in the +# webhook URL. +# Do not configure a custom `Authorization` header. Select alert-created, +# alert-acknowledged, alert-comment-added, and alert-resolved trigger events. +# Deployment preserves existing runtime secrets and stages no credential. +# Disable iLert delivery through the managed alert-action configuration, and +# revoke receiver access by rotating this credential under the applicable +# change controls. POSTIL_ILERT_WEBHOOK_SECRET= +# Canonical HTTPS origin of the deployed receiver. The reconciler accepts only +# an origin without credentials, path, query, or fragment, and appends the +# fixed `/api/webhooks/ilert` path. +POSTIL_ILERT_RECEIVER_ORIGIN=https://your-host + +# Numeric iLert alert source identity. Configure this as the +# `POSTIL_ILERT_ALERT_SOURCE_ID` GitHub Actions variable for the production +# monitor, not as a secret. It selects the source whose Webhook alert action is +# reconciled and verified. +POSTIL_ILERT_ALERT_SOURCE_ID= + +# Workflow-only identity for live reconciliation and cancellation cleanup. +# `--canary` and every `--finalize-canary` invocation require this identity and +# scoped iLert management credentials. Finalization always performs a mandatory +# terminal management sweep, including after the primary canary proves cleanup, +# so it can resolve a delayed Event API duplicate. Each rerun uses a distinct +# main alert key. The workflow carries the producer attempt to the finalizer +# after an accepted ALERT; its running attempt is the sweep ceiling. +# POSTIL_ILERT_CANARY_RUN_ID= +# POSTIL_ILERT_CANARY_RUN_ATTEMPT= +# POSTIL_ILERT_CANARY_SWEEP_ATTEMPT= + # Numeric GitHub ids allowed to use operator-only surfaces. The alert stream # also requires a renewable `postil login` access token for an allowlisted id. POSTIL_OPERATOR_GITHUB_IDS= @@ -163,6 +190,12 @@ POSTIL_CACHE_DIR=.cache # opens one private incident without stopping other monitor checks. POSTIL_MONITOR_INTERVAL_MS=300000 POSTIL_MONITOR_ALERT_STATE_PATH=/var/lib/postil-monitor/alert-state.json +# Optional iLert Event API integration credential for the monitor alert source. +# The monitor and canary use it to send ALERT and RESOLVE events; it is not a +# management API bearer credential. The monitor starts without it; each alert +# delivery attempt then fails with a configuration error and stays queued for +# retry until the key is set. +# ILERT_INTEGRATION_KEY= # OPENROUTER_MANAGEMENT_API_KEY= # POSTIL_OPENROUTER_DEVELOPMENT_KEY_NAME= # POSTIL_OPENROUTER_PRODUCTION_KEY_NAME= diff --git a/.github/actions/ilert-event/action.yml b/.github/actions/ilert-event/action.yml index 072419a6..75799a00 100644 --- a/.github/actions/ilert-event/action.yml +++ b/.github/actions/ilert-event/action.yml @@ -22,7 +22,9 @@ inputs: require-delivery: description: >- Fail when no integration key is configured instead of recording the - undelivered event. Use for alerts whose delivery must be observable. + undelivered event. Production failure alerts, recovery resolves, and + delivery tests set this to true. False is reserved for explicitly + optional events that do not describe a production failure. required: false default: "false" @@ -99,6 +101,7 @@ runs: --max-time 15 \ --retry 3 \ --retry-delay 5 \ + --retry-max-time 45 \ --retry-all-errors \ --header 'Content-Type: application/json' \ --data @"${RUNNER_TEMP}/ilert-event.json" \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6e423eb..066191de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,10 @@ jobs: POSTIL_TEST_DATABASE_URL: postgresql://postgres@localhost:5432/postgres POSTIL_SEALING_KEY: 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f - name: Verify event-driven iLert operator alerts on fresh Postgres - run: bun test --isolate tests/ilert-alert-stream.test.ts + run: >- + bun test --isolate + tests/ilert-alert-stream.test.ts + tests/ilert-canary-alert-key-migration.test.ts env: POSTIL_TEST_DATABASE_URL: postgresql://postgres@localhost:5432/postgres - run: bun test tests/model-override-migration.test.ts diff --git a/.github/workflows/production-monitor.yml b/.github/workflows/production-monitor.yml index bd328663..083a7791 100644 --- a/.github/workflows/production-monitor.yml +++ b/.github/workflows/production-monitor.yml @@ -4,6 +4,7 @@ on: workflow_run: workflows: ["deploy"] types: [completed] + branches: [main] schedule: # Requested every 15 minutes; GitHub throttles scheduled workflows, so # observed cadence is best-effort (often hourly or worse). This workflow @@ -14,26 +15,149 @@ on: workflow_dispatch: inputs: test_alert: - description: Send a test alert after the production checks pass + description: Send a HIGH-priority test alert only after production checks pass, then attempt to resolve it; it can invoke the escalation path; cannot be combined with reconcile_alert_stream + required: false + type: boolean + default: false + reconcile_alert_stream: + description: Reconcile the iLert webhook action and send then resolve a HIGH-priority production test alert; it can invoke the escalation path; cannot be combined with test_alert required: false type: boolean default: false -permissions: - contents: read +permissions: {} concurrency: # Deployment-completion monitors keep independent workflow owners, while - # scheduled checks serialize together. Every recovery attempt then joins the - # same bounded FIFO queue as deploys, so it observes an idle managed fleet. + # scheduled and manual checks serialize together. Each manual canary attempt + # owns distinct iLert keys until its independent finalizer has run. Every + # recovery attempt then joins the same bounded FIFO queue as deploys, so it + # observes an idle managed fleet. group: ${{ github.event_name == 'workflow_run' && format('production-monitor-deploy-{0}', github.event.workflow_run.id) || 'production-monitor' }} queue: max cancel-in-progress: false jobs: + deployment-trust: + name: Verify triggering deployment + permissions: + actions: read + runs-on: ubuntu-latest + # Three minutes for deployment evidence, one for the sweep-state report, + # and one minute of runner headroom. + timeout-minutes: 5 + outputs: + trusted: ${{ steps.evidence.outputs.trusted }} + conclusion: ${{ steps.evidence.outputs.conclusion }} + steps: + - name: Verify deployment job evidence + id: evidence + timeout-minutes: 3 + env: + DEPLOY_EVENT: ${{ github.event.workflow_run.event }} + DEPLOY_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + DEPLOY_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + DEPLOY_RUN_ID: ${{ github.event.workflow_run.id }} + EVENT_NAME: ${{ github.event_name }} + EVENT_REF: ${{ github.ref }} + GH_TOKEN: ${{ github.token }} + RECONCILE_ALERT_STREAM: ${{ inputs.reconcile_alert_stream }} + TEST_ALERT: ${{ inputs.test_alert }} + run: | + set -euo pipefail + echo "trusted=false" >> "${GITHUB_OUTPUT}" + echo "conclusion=" >> "${GITHUB_OUTPUT}" + + # The canary finalizer resolves the compatibility key that the + # operator test alert uses. A dispatch requesting both would let the + # finalizer close the requested test alert, so it is rejected before + # any job that holds a credential can start. + if [[ "${EVENT_NAME}" == "workflow_dispatch" && "${TEST_ALERT:-}" == "true" && "${RECONCILE_ALERT_STREAM:-}" == "true" ]]; then + echo "::error title=Conflicting manual alert inputs::test_alert and reconcile_alert_stream cannot be combined. Dispatch them as separate runs." >&2 + exit 1 + fi + + if [[ "${EVENT_NAME}" == "schedule" ]]; then + echo "trusted=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then + if [[ "${EVENT_REF}" == "refs/heads/main" ]]; then + echo "trusted=true" >> "${GITHUB_OUTPUT}" + fi + exit 0 + fi + if [[ "${EVENT_NAME}" != "workflow_run" ]]; then + exit 0 + fi + # The deploy workflow admits manual runs only. A completed run is + # trusted when it was dispatched from main and its current attempt + # carries exactly one terminal deploy job. + if [[ "${DEPLOY_EVENT}" != "workflow_dispatch" ]]; then + exit 0 + fi + if [[ "${DEPLOY_HEAD_BRANCH}" != "main" ]]; then + exit 0 + fi + + if [[ ! "${DEPLOY_RUN_ATTEMPT}" =~ ^[1-9][0-9]*$ ]]; then + echo "Triggering deployment attempt is malformed." >&2 + exit 1 + fi + + jobs="" + for api_attempt in 1 2 3; do + if jobs="$(timeout --signal=TERM --kill-after=5s 30s gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${DEPLOY_RUN_ID}/attempts/${DEPLOY_RUN_ATTEMPT}/jobs?per_page=100")"; then + break + fi + if (( api_attempt == 3 )); then + echo "Deployment job evidence remained unavailable after bounded retries." >&2 + exit 1 + fi + sleep 2 + done + jq -e ' + (.total_count | type == "number") and + (.jobs | type == "array") and + (.total_count == (.jobs | length)) and + (.total_count <= 100) + ' <<<"${jobs}" >/dev/null || { + echo "Deployment job evidence is incomplete." >&2 + exit 1 + } + jq -e ' + ([.jobs[] | select(.name == "deploy")] | length) == 1 + ' <<<"${jobs}" >/dev/null || { + echo "Deployment job evidence is missing or not unique." >&2 + exit 1 + } + deployment_conclusion="$(jq -er ' + [.jobs[] | select(.name == "deploy")] | .[0].conclusion + ' <<<"${jobs}")" + echo "conclusion=${deployment_conclusion}" >> "${GITHUB_OUTPUT}" + + case "${deployment_conclusion}" in + success|failure|cancelled|timed_out) + echo "trusted=true" >> "${GITHUB_OUTPUT}" + ;; + esac + - name: Report the orphaned iLert canary sweep state + if: ${{ steps.evidence.outputs.trusted == 'true' && github.event_name != 'workflow_run' && vars.POSTIL_ILERT_CANARY_SWEEP_ENABLED != 'true' }} + timeout-minutes: 1 + run: | + set -euo pipefail + echo "::notice title=Orphaned iLert canary sweep is disabled::POSTIL_ILERT_CANARY_SWEEP_ENABLED is not 'true', so this run does not sweep orphaned canary alerts. No iLert management request is made." + { + echo "### Orphaned iLert canary sweep is disabled" + echo + echo "\`POSTIL_ILERT_CANARY_SWEEP_ENABLED\` is not \`true\`. This run makes no iLert management request and resolves no orphaned canary alert." + } >> "${GITHUB_STEP_SUMMARY:-/dev/null}" + release-recovery: name: Recover abandoned release preparation - if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion != 'success' }} + needs: deployment-trust + if: ${{ needs.deployment-trust.outputs.trusted == 'true' && (github.event_name != 'workflow_run' || needs.deployment-trust.outputs.conclusion != 'success') && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} concurrency: group: fly-deploy queue: max @@ -171,7 +295,8 @@ jobs: smoke: name: Smoke check production - if: ${{ github.event_name != 'workflow_run' }} + needs: deployment-trust + if: ${{ needs.deployment-trust.outputs.trusted == 'true' && github.event_name != 'workflow_run' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} runs-on: ubuntu-latest timeout-minutes: 6 steps: @@ -535,23 +660,218 @@ jobs: fi fi + canary-orphan-sweep: + name: Sweep orphaned iLert canaries + # Scheduled and routine manual monitors recover deterministic HIGH canary + # alerts whose producer and two finalizers were both interrupted. This job + # has no Event API credential and never emits a new canary or resolves a + # routine production-monitor alert. + # A manual test alert uses the swept compatibility key. Its own finalizer + # resolves it after successful checks; this sweep recovers an orphan. + # The sweep needs the dedicated iLert management identity, so it runs only + # when the POSTIL_ILERT_CANARY_SWEEP_ENABLED repository variable is 'true'. + # The trust job reports the disabled state on every routine run. + needs: deployment-trust + if: ${{ needs.deployment-trust.outputs.trusted == 'true' && vars.POSTIL_ILERT_CANARY_SWEEP_ENABLED == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.reconcile_alert_stream != true && inputs.test_alert != true)) && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + # Three one-minute setup steps, fourteen minutes for bounded recovery, + # and four minutes of runner headroom. + timeout-minutes: 21 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + timeout-minutes: 1 + with: + ref: ${{ github.sha }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + timeout-minutes: 1 + with: + bun-version: 1.3.14 + - name: Load iLert management secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_ILERT_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: / + secret-name: ILERT_API_KEY + - name: Resolve and verify orphaned iLert canaries + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + run: timeout 14m bun run scripts/reconcile-ilert-alert-stream.ts --sweep-canary-orphans + + alert-stream: + name: Reconcile and verify iLert webhook delivery + needs: [deployment-trust, smoke, release-recovery] + outputs: + alert-submitted: ${{ steps.canary.outputs.alert_submitted }} + producer-attempt: ${{ steps.canary.outputs.producer_attempt }} + # Manual canaries run only after smoke succeeds; failed or cancelled + # production checks must not emit a test page. Production failures use the + # stable alert key owned by the notify and resolve jobs below. + if: ${{ always() && needs.deployment-trust.outputs.trusted == 'true' && inputs.reconcile_alert_stream == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + # The 37-minute job budget reserves five minutes for setup, seven minutes + # for preview reconciliation, thirteen for reconcile plus canary, four + # for in-job cleanup, and eight minutes of runner headroom. After sending + # ALERT, the canary reserves its final two minutes for repeated resolution + # verification. + timeout-minutes: 37 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + timeout-minutes: 1 + with: + ref: ${{ github.sha }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + timeout-minutes: 1 + with: + bun-version: 1.3.14 + - name: Load iLert management secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_ILERT_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: / + secret-name: ILERT_API_KEY + - name: Load iLert integration secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY + - name: Load Postil webhook secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: POSTIL_ILERT_WEBHOOK_SECRET + - name: Preview iLert alert-stream reconciliation + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + POSTIL_ILERT_RECEIVER_ORIGIN: https://postil.dev + run: timeout 7m bun run scripts/reconcile-ilert-alert-stream.ts --dry-run + - name: Reconcile and verify the attempt-specific iLert webhook canary + id: canary + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + POSTIL_ILERT_CANARY_RUN_ATTEMPT: ${{ github.run_attempt }} + POSTIL_ILERT_CANARY_RUN_ID: ${{ github.run_id }} + POSTIL_ILERT_RECEIVER_ORIGIN: https://postil.dev + run: timeout 13m bun run scripts/reconcile-ilert-alert-stream.ts --canary + # Cancellation does not reliably start a dependent job. This step keeps + # cleanup with the producer while the runner remains available. The + # downstream finalizer still covers job timeout and runner loss. + - name: Resolve and stabilize the current iLert canary before job exit + if: ${{ always() && steps.canary.outcome != 'skipped' }} + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + POSTIL_ILERT_CANARY_ALERT_SUBMITTED: ${{ steps.canary.outputs.alert_submitted }} + POSTIL_ILERT_CANARY_RUN_ATTEMPT: ${{ github.run_attempt }} + POSTIL_ILERT_CANARY_SWEEP_ATTEMPT: ${{ github.run_attempt }} + POSTIL_ILERT_CANARY_RUN_ID: ${{ github.run_id }} + run: timeout 4m bun run scripts/reconcile-ilert-alert-stream.ts --finalize-canary + + alert-stream-finalize: + name: Finalize iLert webhook canary cleanup + needs: [deployment-trust, alert-stream] + # This separate job runs after success, failure, cancellation, or timeout. + # The primary canary proves its receiver events before a cleaned handoff, + # but this job still sweeps iLert management inventory for a delayed Event + # API duplicate and fails closed if that inventory is incomplete. + if: ${{ always() && needs.deployment-trust.outputs.trusted == 'true' && inputs.reconcile_alert_stream == true && needs.alert-stream.result != 'skipped' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + # Four explicitly bounded setup steps take at most four minutes, followed + # by four minutes for cleanup and two minutes of runner headroom. The + # 11-minute cap is strictly larger than that complete budget. It never + # reconciles the action. + timeout-minutes: 11 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + timeout-minutes: 1 + with: + ref: ${{ github.sha }} + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + timeout-minutes: 1 + with: + bun-version: 1.3.14 + - name: Load iLert management secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_ILERT_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: / + secret-name: ILERT_API_KEY + - name: Load iLert integration secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY + - name: Resolve and stabilize the reconstructible iLert canary + env: + POSTIL_ILERT_ALERT_SOURCE_ID: ${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }} + POSTIL_ILERT_CANARY_ALERT_SUBMITTED: ${{ needs.alert-stream.outputs.alert-submitted }} + POSTIL_ILERT_CANARY_RUN_ATTEMPT: ${{ needs.alert-stream.outputs.producer-attempt || github.run_attempt }} + POSTIL_ILERT_CANARY_SWEEP_ATTEMPT: ${{ github.run_attempt }} + POSTIL_ILERT_CANARY_RUN_ID: ${{ github.run_id }} + run: timeout 4m bun run scripts/reconcile-ilert-alert-stream.ts --finalize-canary + # Alert delivery is owned by the external alerting service (ilert). The # platform and this workflow only report events; paging, escalation, and # deduplication happen there. A stable alertKey groups consecutive failing # runs into one alert, and the resolve job auto-closes it on recovery. notify: name: Raise external alert - needs: [smoke, release-recovery] - if: ${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || inputs.test_alert == true) }} + needs: [deployment-trust, smoke, release-recovery] + if: ${{ always() && needs.deployment-trust.outputs.trusted == 'true' && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || (inputs.test_alert == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success')) && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} permissions: contents: read id-token: write runs-on: ubuntu-latest - timeout-minutes: 2 + # Checkout and OIDC secret loading each have one minute. Delivery has two + # minutes so curl's retry envelope retains shell headroom, and the parent + # reserves one additional minute of runner headroom. + timeout-minutes: 5 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Load alerting secret from Infisical - uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + timeout-minutes: 1 + - name: Load required iLert integration secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 with: method: oidc identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} @@ -559,8 +879,10 @@ jobs: env-slug: prod domain: https://eu.infisical.com secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY - name: Send ilert event uses: ./.github/actions/ilert-event + timeout-minutes: 2 with: event-type: ALERT summary: ${{ (needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled') && 'Postil release recovery failed' || needs.smoke.result == 'failure' && 'Postil production monitor failed' || 'Postil production monitor test alert' }} @@ -572,30 +894,76 @@ jobs: && 'Production checks failed. Run log:' || 'Operator-requested test alert; production checks passed. Run log:' }} ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }} + # A production failure has no successful notification outcome until + # iLert accepts its alert. OIDC or Infisical failure therefore fails + # this job visibly instead of turning paging into a no-op. + require-delivery: true + + # Notification can fail after iLert accepts the test alert. Wait for notify + # to finish before resolving its dedicated key, regardless of its result. + resolve-test-alert: + name: Resolve production monitor test alert + needs: [deployment-trust, smoke, release-recovery, notify] + if: ${{ always() && needs.deployment-trust.outputs.trusted == 'true' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.test_alert == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success' }} + permissions: + contents: read + id-token: write + runs-on: ubuntu-latest + # Checkout and OIDC secret loading each have one minute. Delivery has two + # minutes, with one additional minute of runner headroom. + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + timeout-minutes: 1 + - name: Load required iLert integration secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 + with: + method: oidc + identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} + project-slug: ${{ secrets.INFISICAL_PROJECT_SLUG }} + env-slug: prod + domain: https://eu.infisical.com + secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY + - name: Resolve ilert test alert + uses: ./.github/actions/ilert-event + timeout-minutes: 2 + with: + event-type: RESOLVE + summary: Postil production monitor test alert cleared + alert-key: postil-production-monitor-test require-delivery: true resolve-release-recovery: name: Resolve release recovery alert - needs: release-recovery - if: ${{ always() && (needs.release-recovery.outputs.clear == 'true' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')) }} + needs: [deployment-trust, release-recovery] + if: ${{ always() && needs.deployment-trust.outputs.trusted == 'true' && (needs.release-recovery.outputs.clear == 'true' || (github.event_name == 'workflow_run' && needs.deployment-trust.outputs.conclusion == 'success')) && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }} permissions: contents: read id-token: write runs-on: ubuntu-latest - timeout-minutes: 3 + # Every step is independently bounded. The parent reserves one additional + # minute beyond their combined budgets for runner and shell overhead. + timeout-minutes: 9 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + timeout-minutes: 1 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + timeout-minutes: 1 with: bun-version: 1.3.14 - name: Install dependencies + timeout-minutes: 2 run: bun install --frozen-lockfile - name: Verify release state is active and clear + timeout-minutes: 1 env: DATABASE_URL: ${{ secrets.DATABASE_URL }} run: bun scripts/run-release-migrations.ts --verify-clear - - name: Load alerting secret from Infisical - uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + - name: Load iLert integration secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 with: method: oidc identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} @@ -603,39 +971,38 @@ jobs: env-slug: prod domain: https://eu.infisical.com secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY - name: Resolve ilert release recovery alert uses: ./.github/actions/ilert-event + timeout-minutes: 2 with: event-type: RESOLVE summary: Postil release recovery cleared alert-key: postil-release-recovery + require-delivery: true + # The alert-stream job resolves its attempt-specific canary before it succeeds. + # The test-alert finalizer owns its dedicated key. This resolver owns only + # the stable production-monitor key, so manual tests cannot close an incident. + # Every successful non-canary run resolves it, including one after a test. resolve: name: Resolve external alert - needs: smoke - if: ${{ needs.smoke.result == 'success' && inputs.test_alert != true }} + needs: [deployment-trust, smoke] + if: ${{ needs.deployment-trust.outputs.trusted == 'true' && needs.smoke.result == 'success' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') && (github.event_name != 'workflow_dispatch' || inputs.reconcile_alert_stream != true) && inputs.test_alert != true }} permissions: contents: read id-token: write - actions: read runs-on: ubuntu-latest - timeout-minutes: 2 + # Checkout and OIDC secret loading each have one minute. Delivery has two + # minutes so curl's retry envelope retains shell headroom, and the parent + # reserves one additional minute of runner headroom. + timeout-minutes: 5 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Check whether the previous run failed - id: previous - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - previous="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/production-monitor.yml/runs?status=completed&per_page=5" \ - --jq "[.workflow_runs[] | select(.id != ${GITHUB_RUN_ID})][0].conclusion // \"none\"")" - echo "conclusion=${previous}" >> "$GITHUB_OUTPUT" - echo "Previous completed run conclusion: ${previous}" - - name: Load alerting secret from Infisical - if: ${{ steps.previous.outputs.conclusion == 'failure' }} - uses: Infisical/secrets-action@77ab1f4ccd183a543cb5b42435fbd181189f4995 # v1.0.16 + timeout-minutes: 1 + - name: Load iLert integration secret from Infisical + uses: Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db # v1.0.17 + timeout-minutes: 1 with: method: oidc identity-id: ${{ secrets.INFISICAL_MACHINE_IDENTITY_ID }} @@ -643,10 +1010,12 @@ jobs: env-slug: prod domain: https://eu.infisical.com secret-path: /postil + secret-name: ILERT_INTEGRATION_KEY - name: Resolve ilert alert - if: ${{ steps.previous.outputs.conclusion == 'failure' }} uses: ./.github/actions/ilert-event + timeout-minutes: 2 with: event-type: RESOLVE summary: Postil production monitor recovered alert-key: postil-production-monitor + require-delivery: true diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 15cfa406..9466da4b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -54,6 +54,17 @@ entries and unapproved pending migrations stop preparation. The existing strict journal verification runs before the new release receives capabilities. `bun run scripts/run-release-migrations.ts --inspect-additive` checks the same migration identity and source compatibility in a read-only transaction. +After the provider preflight and before the new release receives capabilities, +the managed release command runs `bun run operational:indexes --managed-release`. +That mode concurrently builds only the `ilert_alert_events_canary_observation_idx` +lookup index, waits at most 60 seconds for the index release lock, and bounds +each statement at 120 seconds. It neither inspects other operational indexes +nor writes the full reconciliation's release step. A failed or cancelled build +stops the release with the running fleet unchanged, and the next run rebuilds +an invalid index. The full reconciliation rebuilds a valid index with an +unexpected definition only for that non-unique lookup index; any other +unexpected definition, and any index that enforces uniqueness or a constraint, +stays in place and fails the run. The free-tier operating profile keeps Postgres idle-capable by avoiding permanent hot polling when the private monitor is disabled. Webhook intake verifies the signature, then commits the payload and one `webhook-dispatch` job in the same transaction before acknowledging GitHub. A Next.js `after` callback claims that exact job without delaying the response, and the long-running worker remains a fallback with configurable idle backoff. Completed inbox payloads are cleared. A stopped web process leaves a retryable queue claim and retained payload instead of a completed dedupe marker with missing side effects. @@ -510,9 +521,12 @@ monitoring alert only when ilert rejects the event, so a lapsed or misconfigured alerting account cannot silence paging. After each completed pass the monitor pings an ilert heartbeat (`POSTIL_MONITOR_HEARTBEAT_URL`), so a dead monitor process or persistently failing pass raises a missed-heartbeat -alert outside the platform's failure domain. Without a configured integration -key, monitoring alerts are logged only and paging relies on the heartbeat, the -external uptime checks, and the GitHub monitor. +alert outside the platform's failure domain. `ILERT_INTEGRATION_KEY` is optional +at monitor startup, so a self-hosted monitor runs its checks without an +alerting account. Without the key every monitoring alert delivery attempt +fails with an explicit configuration error, the durable outbox retains the +incident and retries it, and paging relies on the heartbeat, the external +uptime checks, and the GitHub monitor. iLert alert actions can deliver bounded alert lifecycle events to `/api/webhooks/ilert` with HTTP Basic authentication. The receiver stores the @@ -528,6 +542,94 @@ allowlist. Transport keepalives do not query alert state. The receiver and stream make no outbound iLert API calls and cannot acknowledge, accept, or resolve an alert. +The production-monitor workflow loads the workflow-only `ILERT_API_KEY` +management bearer credential through its dedicated least-privilege Infisical +OIDC identity and sends monitor and canary events with the +`ILERT_INTEGRATION_KEY` Event API credential. The management credential is not +part of the application environment. Its +`POSTIL_ILERT_ALERT_SOURCE_ID` GitHub Actions variable selects the alert source +whose action is reconciled. `POSTIL_ILERT_WEBHOOK_SECRET` supplies the receiver +password embedded as URL-encoded HTTP Basic credentials in that action, and +`POSTIL_ILERT_RECEIVER_ORIGIN` selects its strictly validated HTTPS origin. The +reconciliation workflow loads the receiver password from the secret record that +also supplies the deployed receiver's runtime secret. Reconciliation verifies that the deployed receiver accepts the +credential before it creates or updates the action. Before any action mutation +or Event API mutation, reconciliation reads `GET /api/alert-sources/{id}` with +the configured numeric source id and the management bearer credential. The +returned numeric source id, Event API type, and Event API integration key must +match the configured source. The key, management authorization, integration +fields, and full lookup URL remain confined to the sensitive request. Manual canaries send a +HIGH-priority production test alert that can invoke the escalation path. They +use attempt-specific keys derived from the workflow run id and producer run +attempt, distinct from production monitor keys. Earlier main keys are swept +through the current attempt and every discovered trusted alert is resolved with +`PUT /api/alerts/{id}/resolve` then polled by that exact id until management +reports RESOLVED. A discovered canary retains its exact id, key, and source +before receiver evidence is checked, so cleanup can resolve it when later +receiver or inventory requests fail. A blind same-key Event API RESOLVE is +limited to an undiscovered current key. +The main alert is discovered within the configured source using the same exact +32-day report-time window as cleanup. It must remain PENDING or ACCEPTED at +HIGH priority and have an +exact persisted `alert-created` receiver event. A `cleaned` handoff means the +primary canary also proves the exact persisted `alert-resolved` receiver event. +The independent cleanup-only finalizer uses the producer attempt for +accepted-current proof and the running attempt as a sweep ceiling. It accepts +GitHub's 1 through 51 attempt range, bounds work per attempt, and sweeps an +exact 32-day run-wide report-time window with PENDING, ACCEPTED, and RESOLVED +states. Every discovery pass freezes its upper report-time bound, brackets +offset pagination with matching inventory counts, repeats the complete +inventory for continuity validation, and advances the bound before the next +delayed pass. Complete inventory fingerprints preserve opaque provider alert +ids, while a matching cleanup target requires the provider's positive-integer +alert identity before mutation. The finalizer reserves 90 seconds for delayed +discovery, 60 seconds for terminal inventory validation, and 60 seconds for +retained-id resolution. Its 210-second operation deadline runs within a +four-minute shell bound, leaving one minute of GitHub's cancellation grace. +The finalizer verifies every retained id as RESOLVED. Two complete delayed +passes must find no matching open alert before success. +A cleaned handoff still performs this +management sweep to resolve a delayed Event API duplicate. If the producer +output is unavailable, the finalizer uses its running attempt to reconstruct +the deterministic identity. It never reads receiver configuration or +reconciles the action. A manual dispatch that requests both `test_alert` and +`reconcile_alert_stream` fails in the trust job before any credentialed job +starts, because the finalizer resolves the compatibility key that the operator +test alert uses. A requested test alert is sent only when the smoke and release +recovery jobs succeed. Its own finalizer resolves the dedicated key after the +notification job settles, including when delivery reports failure after iLert +accepts the alert. When the `POSTIL_ILERT_CANARY_SWEEP_ENABLED` repository +variable is `true`, scheduled and routine manual production monitors also +run a management-only orphan sweep across every exact deterministic canary key +and the exact `postil-production-monitor-test` compatibility canary key for the +configured source. Otherwise the trust job reports the sweep as disabled and +the run makes no iLert management request. The sweep scans only PENDING and ACCEPTED alerts, +count-brackets and repeats the complete offset inventory, retains each exact +id, resolves and verifies each target, and fails closed on incomplete or +changing inventory. It has no Event API credential and cannot emit a canary +or resolve the stable `postil-production-monitor` alert or an arbitrary alert. +Management cleanup validates the source +numeric id and API type without requiring an older Event API key to remain +bound; Event API mutations and action reconciliation retain the full +integration-key binding. The command supports `--dry-run`, `--canary`, +`--finalize-canary`, and `--sweep-canary-orphans`; unflagged live reconciliation +is rejected because it has no recoverable command. The canary does not open +the authenticated operator SSE stream. + +iLert supplies offset pagination rather than a snapshot, cursor, or exact-key +filter. The bounded count and repeated-inventory checks provide evidence for a +complete scan and fail closed on observed churn; they do not claim atomic +absence proof. The reconciler revalidates the numeric source ID, API integration +type, and integration-key binding at every alert-action and Event API mutation, +then reloads the complete reserved-action inventory after an action mutation. +The alert-action API has no conditional-write precondition, so a concurrent +out-of-band action update can still occur after the final guard read. The +reconciler detects conflicting pre- and post-mutation state and fails closed; +it does not claim compare-and-swap behavior. +Deployment preserves runtime secret metadata and stages no receiver credential. +The `--validate-webhook-secret` command checks a receiver secret against the +printable-ASCII length and diversity contract used by the receiver. + The monitor and product processes share the deployment platform, network, and DNS path. The private database, the external alerting service, and the external metrics collector are separate dependencies. A platform-wide or diff --git a/drizzle/0064_ilert_canary_alert_key.sql b/drizzle/0064_ilert_canary_alert_key.sql new file mode 100644 index 00000000..1affafeb --- /dev/null +++ b/drizzle/0064_ilert_canary_alert_key.sql @@ -0,0 +1,2 @@ +ALTER TABLE "ilert_alert_events" ADD COLUMN "alert_key" text;--> statement-breakpoint +ALTER TABLE "ilert_alert_events" ADD CONSTRAINT "ilert_alert_events_alert_key_check" CHECK ("ilert_alert_events"."alert_key" IS NULL OR length("ilert_alert_events"."alert_key") BETWEEN 1 AND 512) NOT VALID; diff --git a/drizzle/README.md b/drizzle/README.md index 6b1169a1..a089b752 100644 --- a/drizzle/README.md +++ b/drizzle/README.md @@ -28,6 +28,19 @@ The schema and Drizzle snapshot describe the final released database. The transactional migration stream and records their verified state in `release_steps`. +## CHECK constraints on existing tables + +`ALTER TABLE ... ADD CONSTRAINT ... CHECK` scans every existing row while it +holds an `ACCESS EXCLUSIVE` lock, which blocks reads and writes to the table. + +- Add the constraint with `NOT VALID`. PostgreSQL skips the scan and still + enforces the constraint on every inserted or updated row. +- Run `VALIDATE CONSTRAINT` in a separate release step outside the + transactional migration stream. Its scan permits concurrent reads and writes + and needs its own time budget. +- The Drizzle snapshot records the constraint expression, not its validation + state. + ## Bulk UPDATE/DELETE dedup work Row rewrites must not run as one unbounded statement against a live table: diff --git a/drizzle/meta/0064_snapshot.json b/drizzle/meta/0064_snapshot.json new file mode 100644 index 00000000..c95985d9 --- /dev/null +++ b/drizzle/meta/0064_snapshot.json @@ -0,0 +1,8422 @@ +{ + "id": "b3afea75-bf48-4e42-a078-d21f75d1b6e5", + "prevId": "a4d88b6e-4d79-48ac-ba79-eb48c407f0b6", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.billing_author_settlements": { + "name": "billing_author_settlements", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider_subscription_id": { + "name": "provider_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "period_starts_at": { + "name": "period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "period_ends_at": { + "name": "period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "active_author_count": { + "name": "active_author_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "unit_amount_cents": { + "name": "unit_amount_cents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 600 + }, + "total_amount_cents": { + "name": "total_amount_cents", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "provider_transaction_id": { + "name": "provider_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "attempt_started_at": { + "name": "attempt_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "next_reconcile_at": { + "name": "next_reconcile_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_author_settlements_org_period_idx": { + "name": "billing_author_settlements_org_period_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "period_starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "period_ends_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_author_settlements_provider_transaction_idx": { + "name": "billing_author_settlements_provider_transaction_idx", + "columns": [ + { + "expression": "provider_transaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_author_settlements_status_reconcile_idx": { + "name": "billing_author_settlements_status_reconcile_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_reconcile_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "billing_author_settlements_org_id_organizations_id_fk": { + "name": "billing_author_settlements_org_id_organizations_id_fk", + "tableFrom": "billing_author_settlements", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_author_settlements_period_check": { + "name": "billing_author_settlements_period_check", + "value": "\"billing_author_settlements\".\"period_starts_at\" < \"billing_author_settlements\".\"period_ends_at\"" + }, + "billing_author_settlements_author_count_check": { + "name": "billing_author_settlements_author_count_check", + "value": "\"billing_author_settlements\".\"active_author_count\" >= 0" + }, + "billing_author_settlements_amount_check": { + "name": "billing_author_settlements_amount_check", + "value": "\"billing_author_settlements\".\"unit_amount_cents\" = 600 AND \"billing_author_settlements\".\"total_amount_cents\" = \"billing_author_settlements\".\"active_author_count\" * \"billing_author_settlements\".\"unit_amount_cents\"" + }, + "billing_author_settlements_status_check": { + "name": "billing_author_settlements_status_check", + "value": "\"billing_author_settlements\".\"status\" IN ('pending', 'charging', 'reconciling', 'charged', 'no_charge', 'failed')" + }, + "billing_author_settlements_attempt_count_check": { + "name": "billing_author_settlements_attempt_count_check", + "value": "\"billing_author_settlements\".\"attempt_count\" >= 0" + }, + "billing_author_settlements_subscription_nonempty": { + "name": "billing_author_settlements_subscription_nonempty", + "value": "length(btrim(\"billing_author_settlements\".\"provider_subscription_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_checkout_transactions": { + "name": "billing_checkout_transactions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "requested_by_user_id": { + "name": "requested_by_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paddle'" + }, + "provider_transaction_id": { + "name": "provider_transaction_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "checkout_url": { + "name": "checkout_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'creating'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_checkout_transactions_provider_transaction_idx": { + "name": "billing_checkout_transactions_provider_transaction_idx", + "columns": [ + { + "expression": "provider_transaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_checkout_transactions_open_org_idx": { + "name": "billing_checkout_transactions_open_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"billing_checkout_transactions\".\"status\" IN ('creating', 'pending')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_checkout_transactions_status_expiry_idx": { + "name": "billing_checkout_transactions_status_expiry_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "billing_checkout_transactions_org_id_organizations_id_fk": { + "name": "billing_checkout_transactions_org_id_organizations_id_fk", + "tableFrom": "billing_checkout_transactions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "billing_checkout_transactions_requested_by_user_id_users_id_fk": { + "name": "billing_checkout_transactions_requested_by_user_id_users_id_fk", + "tableFrom": "billing_checkout_transactions", + "tableTo": "users", + "columnsFrom": [ + "requested_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_checkout_transactions_provider_check": { + "name": "billing_checkout_transactions_provider_check", + "value": "\"billing_checkout_transactions\".\"provider\" = 'paddle'" + }, + "billing_checkout_transactions_status_check": { + "name": "billing_checkout_transactions_status_check", + "value": "\"billing_checkout_transactions\".\"status\" IN ('creating', 'pending', 'completed', 'failed', 'expired', 'canceled')" + }, + "billing_checkout_transactions_provider_transaction_nonempty": { + "name": "billing_checkout_transactions_provider_transaction_nonempty", + "value": "\"billing_checkout_transactions\".\"provider_transaction_id\" IS NULL OR length(btrim(\"billing_checkout_transactions\".\"provider_transaction_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_credit_grants": { + "name": "billing_credit_grants", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "billing_credit_grants_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "amount_cents": { + "name": "amount_cents", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor": { + "name": "actor", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'admin_script'" + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "applies_at": { + "name": "applies_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_credit_grants_org_created_idx": { + "name": "billing_credit_grants_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_credit_grants_org_idempotency_idx": { + "name": "billing_credit_grants_org_idempotency_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "billing_credit_grants_org_id_organizations_id_fk": { + "name": "billing_credit_grants_org_id_organizations_id_fk", + "tableFrom": "billing_credit_grants", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_credit_grants_amount_cents_positive": { + "name": "billing_credit_grants_amount_cents_positive", + "value": "\"billing_credit_grants\".\"amount_cents\" > 0" + }, + "billing_credit_grants_reason_nonempty": { + "name": "billing_credit_grants_reason_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"reason\")) > 0" + }, + "billing_credit_grants_actor_nonempty": { + "name": "billing_credit_grants_actor_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"actor\")) > 0" + }, + "billing_credit_grants_source_nonempty": { + "name": "billing_credit_grants_source_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"source\")) > 0" + }, + "billing_credit_grants_idempotency_key_nonempty": { + "name": "billing_credit_grants_idempotency_key_nonempty", + "value": "length(btrim(\"billing_credit_grants\".\"idempotency_key\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_provider_events": { + "name": "billing_provider_events", + "schema": "", + "columns": { + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paddle'" + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_object_id": { + "name": "provider_object_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_provider_events_org_occurred_idx": { + "name": "billing_provider_events_org_occurred_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_provider_events_type_occurred_idx": { + "name": "billing_provider_events_type_occurred_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "billing_provider_events_org_id_organizations_id_fk": { + "name": "billing_provider_events_org_id_organizations_id_fk", + "tableFrom": "billing_provider_events", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_provider_events_provider_check": { + "name": "billing_provider_events_provider_check", + "value": "\"billing_provider_events\".\"provider\" = 'paddle'" + }, + "billing_provider_events_outcome_check": { + "name": "billing_provider_events_outcome_check", + "value": "\"billing_provider_events\".\"outcome\" IN ('processing', 'applied', 'stale', 'ignored', 'unmatched')" + }, + "billing_provider_events_event_type_nonempty": { + "name": "billing_provider_events_event_type_nonempty", + "value": "length(btrim(\"billing_provider_events\".\"event_type\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.billing_provider_subscriptions": { + "name": "billing_provider_subscriptions", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'paddle'" + }, + "provider_subscription_id": { + "name": "provider_subscription_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_customer_id": { + "name": "provider_customer_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_period_starts_at": { + "name": "current_period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "current_period_ends_at": { + "name": "current_period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "latest_event_occurred_at": { + "name": "latest_event_occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "latest_event_id": { + "name": "latest_event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "billing_provider_subscriptions_provider_id_idx": { + "name": "billing_provider_subscriptions_provider_id_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_subscription_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "billing_provider_subscriptions_status_period_idx": { + "name": "billing_provider_subscriptions_status_period_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "current_period_ends_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "billing_provider_subscriptions_org_id_organizations_id_fk": { + "name": "billing_provider_subscriptions_org_id_organizations_id_fk", + "tableFrom": "billing_provider_subscriptions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "billing_provider_subscriptions_provider_check": { + "name": "billing_provider_subscriptions_provider_check", + "value": "\"billing_provider_subscriptions\".\"provider\" = 'paddle'" + }, + "billing_provider_subscriptions_status_check": { + "name": "billing_provider_subscriptions_status_check", + "value": "\"billing_provider_subscriptions\".\"status\" IN ('active', 'trialing', 'past_due', 'paused', 'canceled')" + }, + "billing_provider_subscriptions_provider_subscription_nonempty": { + "name": "billing_provider_subscriptions_provider_subscription_nonempty", + "value": "length(btrim(\"billing_provider_subscriptions\".\"provider_subscription_id\")) > 0" + }, + "billing_provider_subscriptions_provider_customer_nonempty": { + "name": "billing_provider_subscriptions_provider_customer_nonempty", + "value": "length(btrim(\"billing_provider_subscriptions\".\"provider_customer_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.cli_device_authorizations": { + "name": "cli_device_authorizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "cli_device_authorizations_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "device_code_sha256": { + "name": "device_code_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "token_id": { + "name": "token_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "poll_count": { + "name": "poll_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "cli_device_authorizations_device_code_sha256_idx": { + "name": "cli_device_authorizations_device_code_sha256_idx", + "columns": [ + { + "expression": "device_code_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_device_authorizations_user_code_idx": { + "name": "cli_device_authorizations_user_code_idx", + "columns": [ + { + "expression": "user_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "cli_device_authorizations_user_id_users_id_fk": { + "name": "cli_device_authorizations_user_id_users_id_fk", + "tableFrom": "cli_device_authorizations", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "cli_device_authorizations_org_id_organizations_id_fk": { + "name": "cli_device_authorizations_org_id_organizations_id_fk", + "tableFrom": "cli_device_authorizations", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "cli_device_authorizations_token_id_cli_tokens_id_fk": { + "name": "cli_device_authorizations_token_id_cli_tokens_id_fk", + "tableFrom": "cli_device_authorizations", + "tableTo": "cli_tokens", + "columnsFrom": [ + "token_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_device_authorizations_status_check": { + "name": "cli_device_authorizations_status_check", + "value": "\"cli_device_authorizations\".\"status\" IN ('pending', 'approved', 'denied', 'claimed')" + } + }, + "isRLSEnabled": false + }, + "public.cli_refresh_sessions": { + "name": "cli_refresh_sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "cli_refresh_sessions_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cli_refresh_sessions_expiry_idx": { + "name": "cli_refresh_sessions_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "cli_refresh_sessions_user_id_users_id_fk": { + "name": "cli_refresh_sessions_user_id_users_id_fk", + "tableFrom": "cli_refresh_sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "cli_refresh_sessions_org_id_organizations_id_fk": { + "name": "cli_refresh_sessions_org_id_organizations_id_fk", + "tableFrom": "cli_refresh_sessions", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_refresh_sessions_expiry_check": { + "name": "cli_refresh_sessions_expiry_check", + "value": "\"cli_refresh_sessions\".\"expires_at\" > \"cli_refresh_sessions\".\"created_at\"" + } + }, + "isRLSEnabled": false + }, + "public.cli_refresh_tokens": { + "name": "cli_refresh_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "cli_refresh_tokens_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "token_sha256": { + "name": "token_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cli_refresh_tokens_token_sha256_idx": { + "name": "cli_refresh_tokens_token_sha256_idx", + "columns": [ + { + "expression": "token_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_refresh_tokens_current_session_idx": { + "name": "cli_refresh_tokens_current_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"cli_refresh_tokens\".\"consumed_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_refresh_tokens_session_idx": { + "name": "cli_refresh_tokens_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "cli_refresh_tokens_session_id_cli_refresh_sessions_id_fk": { + "name": "cli_refresh_tokens_session_id_cli_refresh_sessions_id_fk", + "tableFrom": "cli_refresh_tokens", + "tableTo": "cli_refresh_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_refresh_tokens_expiry_check": { + "name": "cli_refresh_tokens_expiry_check", + "value": "\"cli_refresh_tokens\".\"expires_at\" > \"cli_refresh_tokens\".\"created_at\"" + }, + "cli_refresh_tokens_consumed_after_created_check": { + "name": "cli_refresh_tokens_consumed_after_created_check", + "value": "\"cli_refresh_tokens\".\"consumed_at\" IS NULL OR \"cli_refresh_tokens\".\"consumed_at\" >= \"cli_refresh_tokens\".\"created_at\"" + } + }, + "isRLSEnabled": false + }, + "public.cli_tokens": { + "name": "cli_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "cli_tokens_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "token_sha256": { + "name": "token_sha256", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "refresh_session_id": { + "name": "refresh_session_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cli_tokens_token_sha256_idx": { + "name": "cli_tokens_token_sha256_idx", + "columns": [ + { + "expression": "token_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_tokens_org_created_idx": { + "name": "cli_tokens_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_tokens_refresh_session_idx": { + "name": "cli_tokens_refresh_session_idx", + "columns": [ + { + "expression": "refresh_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "cli_tokens_user_id_users_id_fk": { + "name": "cli_tokens_user_id_users_id_fk", + "tableFrom": "cli_tokens", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "cli_tokens_org_id_organizations_id_fk": { + "name": "cli_tokens_org_id_organizations_id_fk", + "tableFrom": "cli_tokens", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "cli_tokens_refresh_session_id_cli_refresh_sessions_id_fk": { + "name": "cli_tokens_refresh_session_id_cli_refresh_sessions_id_fk", + "tableFrom": "cli_tokens", + "tableTo": "cli_refresh_sessions", + "columnsFrom": [ + "refresh_session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "cli_tokens_scope_check": { + "name": "cli_tokens_scope_check", + "value": "\"cli_tokens\".\"scope\" IN ('inference')" + } + }, + "isRLSEnabled": false + }, + "public.customer_notification_email_deliveries": { + "name": "customer_notification_email_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "email_category": { + "name": "email_category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_count": { + "name": "event_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "customer_notification_email_deliveries_status_created_idx": { + "name": "customer_notification_email_deliveries_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "customer_notification_email_deliveries_org_created_idx": { + "name": "customer_notification_email_deliveries_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "customer_notification_email_deliveries_org_id_organizations_id_fk": { + "name": "customer_notification_email_deliveries_org_id_organizations_id_fk", + "tableFrom": "customer_notification_email_deliveries", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "customer_notification_email_deliveries_category_check": { + "name": "customer_notification_email_deliveries_category_check", + "value": "\"customer_notification_email_deliveries\".\"email_category\" IN ('security', 'payment_failure', 'trial_expiry', 'service_incident', 'billing_summary')" + }, + "customer_notification_email_deliveries_status_check": { + "name": "customer_notification_email_deliveries_status_check", + "value": "\"customer_notification_email_deliveries\".\"status\" IN ('queued', 'retrying', 'sending', 'delivered', 'suppressed', 'failed')" + }, + "customer_notification_email_deliveries_event_count_check": { + "name": "customer_notification_email_deliveries_event_count_check", + "value": "\"customer_notification_email_deliveries\".\"event_count\" BETWEEN 1 AND 20" + } + }, + "isRLSEnabled": false + }, + "public.customer_notification_email_delivery_events": { + "name": "customer_notification_email_delivery_events", + "schema": "", + "columns": { + "event_id": { + "name": "event_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "delivery_id": { + "name": "delivery_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "customer_notification_email_delivery_events_delivery_idx": { + "name": "customer_notification_email_delivery_events_delivery_idx", + "columns": [ + { + "expression": "delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "customer_notification_email_delivery_events_delivery_id_customer_notification_email_deliveries_id_fk": { + "name": "customer_notification_email_delivery_events_delivery_id_customer_notification_email_deliveries_id_fk", + "tableFrom": "customer_notification_email_delivery_events", + "tableTo": "customer_notification_email_deliveries", + "columnsFrom": [ + "delivery_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.customer_notification_events": { + "name": "customer_notification_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "customer_notification_events_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action_label": { + "name": "action_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "action_href": { + "name": "action_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "customer_notification_events_org_key_idx": { + "name": "customer_notification_events_org_key_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "idempotency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "customer_notification_events_org_created_idx": { + "name": "customer_notification_events_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "customer_notification_events_expiry_idx": { + "name": "customer_notification_events_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "customer_notification_events_org_id_organizations_id_fk": { + "name": "customer_notification_events_org_id_organizations_id_fk", + "tableFrom": "customer_notification_events", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "customer_notification_events_severity_check": { + "name": "customer_notification_events_severity_check", + "value": "\"customer_notification_events\".\"severity\" IN ('info', 'warning', 'critical')" + }, + "customer_notification_events_category_check": { + "name": "customer_notification_events_category_check", + "value": "\"customer_notification_events\".\"category\" IN ('trial', 'billing', 'service', 'security')" + }, + "customer_notification_events_visibility_check": { + "name": "customer_notification_events_visibility_check", + "value": "\"customer_notification_events\".\"visibility\" IN ('members', 'admins')" + }, + "customer_notification_events_content_check": { + "name": "customer_notification_events_content_check", + "value": "length(btrim(\"customer_notification_events\".\"idempotency_key\")) BETWEEN 1 AND 200 AND length(btrim(\"customer_notification_events\".\"title\")) BETWEEN 1 AND 120 AND length(btrim(\"customer_notification_events\".\"body\")) BETWEEN 1 AND 500" + }, + "customer_notification_events_action_check": { + "name": "customer_notification_events_action_check", + "value": "(\"customer_notification_events\".\"action_label\" IS NULL AND \"customer_notification_events\".\"action_href\" IS NULL) OR (\"customer_notification_events\".\"action_label\" IS NOT NULL AND \"customer_notification_events\".\"action_href\" IS NOT NULL AND length(btrim(\"customer_notification_events\".\"action_label\")) BETWEEN 1 AND 60 AND \"customer_notification_events\".\"action_href\" ~ '^/orgs/')" + }, + "customer_notification_events_expiry_check": { + "name": "customer_notification_events_expiry_check", + "value": "\"customer_notification_events\".\"expires_at\" > \"customer_notification_events\".\"created_at\"" + } + }, + "isRLSEnabled": false + }, + "public.customer_notification_reads": { + "name": "customer_notification_reads", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "customer_notification_reads_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "event_id": { + "name": "event_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "read_at": { + "name": "read_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "customer_notification_reads_event_user_idx": { + "name": "customer_notification_reads_event_user_idx", + "columns": [ + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "customer_notification_reads_user_event_idx": { + "name": "customer_notification_reads_user_event_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "customer_notification_reads_event_id_customer_notification_events_id_fk": { + "name": "customer_notification_reads_event_id_customer_notification_events_id_fk", + "tableFrom": "customer_notification_reads", + "tableTo": "customer_notification_events", + "columnsFrom": [ + "event_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "customer_notification_reads_user_id_users_id_fk": { + "name": "customer_notification_reads_user_id_users_id_fk", + "tableFrom": "customer_notification_reads", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.finding_approvals": { + "name": "finding_approvals", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "finding_id": { + "name": "finding_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "actor_github_id": { + "name": "actor_github_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_login_snapshot": { + "name": "actor_login_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_role_snapshot": { + "name": "actor_role_snapshot", + "type": "finding_approval_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "verb": { + "name": "verb", + "type": "finding_approval_verb", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'approve'" + }, + "reason_tag": { + "name": "reason_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_self_dismissal": { + "name": "author_self_dismissal", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "finding_kind": { + "name": "finding_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finding_severity": { + "name": "finding_severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finding_confidence": { + "name": "finding_confidence", + "type": "real", + "primaryKey": false, + "notNull": false + }, + "finding_model": { + "name": "finding_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finding_scorer_model": { + "name": "finding_scorer_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rationale": { + "name": "rationale", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "finding_approval_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "source_comment_id": { + "name": "source_comment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_org_id": { + "name": "source_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_repository_id": { + "name": "source_repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_installation_id": { + "name": "source_github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_pr_number": { + "name": "source_pr_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "source_head_sha": { + "name": "source_head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_webhook_delivery_id": { + "name": "source_webhook_delivery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_github_comment_id": { + "name": "source_github_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_comment_kind": { + "name": "source_comment_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_binding_state": { + "name": "source_binding_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'exact'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_by_user_id": { + "name": "revoked_by_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "finding_approvals_active_idx": { + "name": "finding_approvals_active_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"finding_approvals\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_approvals_github_comment_idx": { + "name": "finding_approvals_github_comment_idx", + "columns": [ + { + "expression": "source_github_installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_github_repo_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_comment_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "source_github_comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"finding_approvals\".\"source\" = 'github'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_approvals_github_delivery_idx": { + "name": "finding_approvals_github_delivery_idx", + "columns": [ + { + "expression": "source_webhook_delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"finding_approvals\".\"source\" = 'github'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_approvals_review_idx": { + "name": "finding_approvals_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "finding_approvals_review_id_reviews_id_fk": { + "name": "finding_approvals_review_id_reviews_id_fk", + "tableFrom": "finding_approvals", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "finding_approvals_actor_user_id_users_id_fk": { + "name": "finding_approvals_actor_user_id_users_id_fk", + "tableFrom": "finding_approvals", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "finding_approvals_revoked_by_user_id_users_id_fk": { + "name": "finding_approvals_revoked_by_user_id_users_id_fk", + "tableFrom": "finding_approvals", + "tableTo": "users", + "columnsFrom": [ + "revoked_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_approvals_rationale_nonempty": { + "name": "finding_approvals_rationale_nonempty", + "value": "length(btrim(\"finding_approvals\".\"rationale\")) > 0" + }, + "finding_approvals_dismissal_check": { + "name": "finding_approvals_dismissal_check", + "value": "(\"finding_approvals\".\"verb\" = 'approve' AND \"finding_approvals\".\"reason_tag\" IS NULL AND \"finding_approvals\".\"author_self_dismissal\" = false AND \"finding_approvals\".\"finding_kind\" IS NULL AND \"finding_approvals\".\"finding_severity\" IS NULL AND \"finding_approvals\".\"finding_confidence\" IS NULL AND \"finding_approvals\".\"finding_model\" IS NULL AND \"finding_approvals\".\"finding_scorer_model\" IS NULL) OR (\"finding_approvals\".\"verb\" = 'dismiss' AND \"finding_approvals\".\"reason_tag\" IS NOT NULL AND \"finding_approvals\".\"reason_tag\" IN ('false-positive', 'accepted-risk', 'out-of-scope') AND \"finding_approvals\".\"finding_kind\" IS NOT NULL AND \"finding_approvals\".\"finding_severity\" IS NOT NULL AND \"finding_approvals\".\"finding_confidence\" IS NOT NULL AND \"finding_approvals\".\"finding_confidence\" BETWEEN 0 AND 1 AND \"finding_approvals\".\"finding_model\" IS NOT NULL)" + }, + "finding_approvals_binding_check": { + "name": "finding_approvals_binding_check", + "value": "(\"finding_approvals\".\"source_binding_state\" = 'legacy' AND \"finding_approvals\".\"source_org_id\" IS NULL AND \"finding_approvals\".\"source_repository_id\" IS NULL AND \"finding_approvals\".\"source_github_installation_id\" IS NULL AND \"finding_approvals\".\"source_github_repo_id\" IS NULL AND \"finding_approvals\".\"source_pr_number\" IS NULL AND \"finding_approvals\".\"source_head_sha\" IS NULL) OR (\"finding_approvals\".\"source_binding_state\" = 'exact' AND \"finding_approvals\".\"source_org_id\" > 0 AND \"finding_approvals\".\"source_repository_id\" > 0 AND \"finding_approvals\".\"source_github_installation_id\" > 0 AND \"finding_approvals\".\"source_github_repo_id\" > 0 AND \"finding_approvals\".\"source_pr_number\" > 0 AND length(btrim(\"finding_approvals\".\"source_head_sha\")) BETWEEN 1 AND 200)" + }, + "finding_approvals_github_source_check": { + "name": "finding_approvals_github_source_check", + "value": "\"finding_approvals\".\"source\" <> 'github' OR (\"finding_approvals\".\"source_webhook_delivery_id\" IS NULL AND \"finding_approvals\".\"source_github_comment_id\" IS NULL AND \"finding_approvals\".\"source_comment_kind\" IS NULL) OR (length(btrim(\"finding_approvals\".\"source_webhook_delivery_id\")) BETWEEN 1 AND 200 AND \"finding_approvals\".\"source_github_comment_id\" > 0 AND \"finding_approvals\".\"source_comment_kind\" IN ('issue_comment', 'pull_request_review_comment'))" + } + }, + "isRLSEnabled": false + }, + "public.finding_feedback": { + "name": "finding_feedback", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "finding_feedback_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "finding_publication_id": { + "name": "finding_publication_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_github_comment_id": { + "name": "source_github_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_reaction_id": { + "name": "source_github_reaction_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "reaction_content": { + "name": "reaction_content", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "actor_github_id": { + "name": "actor_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "actor_login_snapshot": { + "name": "actor_login_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pr_author_github_id": { + "name": "pr_author_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "pr_author_login_snapshot": { + "name": "pr_author_login_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_is_pr_author": { + "name": "actor_is_pr_author", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "source_delivery_id": { + "name": "source_delivery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suggested_reason_tag": { + "name": "suggested_reason_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "finding_feedback_publication_observed_idx": { + "name": "finding_feedback_publication_observed_idx", + "columns": [ + { + "expression": "finding_publication_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "observed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_feedback_github_reply_idx": { + "name": "finding_feedback_github_reply_idx", + "columns": [ + { + "expression": "source_github_comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"finding_feedback\".\"source\" = 'reply'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_feedback_github_reaction_idx": { + "name": "finding_feedback_github_reaction_idx", + "columns": [ + { + "expression": "source_github_reaction_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"finding_feedback\".\"source\" = 'reaction'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "finding_feedback_finding_publication_id_finding_publications_id_fk": { + "name": "finding_feedback_finding_publication_id_finding_publications_id_fk", + "tableFrom": "finding_feedback", + "tableTo": "finding_publications", + "columnsFrom": [ + "finding_publication_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_feedback_source_check": { + "name": "finding_feedback_source_check", + "value": "\"finding_feedback\".\"source\" IN ('reply', 'reaction')" + }, + "finding_feedback_identity_check": { + "name": "finding_feedback_identity_check", + "value": "(\"finding_feedback\".\"source\" = 'reply' AND \"finding_feedback\".\"source_github_comment_id\" IS NOT NULL AND \"finding_feedback\".\"source_github_comment_id\" BETWEEN 1 AND 9007199254740991 AND \"finding_feedback\".\"source_github_reaction_id\" IS NULL AND \"finding_feedback\".\"reaction_content\" IS NULL AND \"finding_feedback\".\"body\" IS NOT NULL AND length(btrim(\"finding_feedback\".\"body\")) BETWEEN 1 AND 65535 AND length(btrim(\"finding_feedback\".\"source_delivery_id\")) BETWEEN 1 AND 200) OR (\"finding_feedback\".\"source\" = 'reaction' AND \"finding_feedback\".\"source_github_comment_id\" IS NOT NULL AND \"finding_feedback\".\"source_github_comment_id\" BETWEEN 1 AND 9007199254740991 AND \"finding_feedback\".\"source_github_reaction_id\" IS NOT NULL AND \"finding_feedback\".\"source_github_reaction_id\" BETWEEN 1 AND 9007199254740991 AND \"finding_feedback\".\"reaction_content\" IS NOT NULL AND \"finding_feedback\".\"reaction_content\" IN ('+1', '-1', 'unknown') AND \"finding_feedback\".\"body\" IS NULL AND \"finding_feedback\".\"source_delivery_id\" IS NULL)" + }, + "finding_feedback_actor_check": { + "name": "finding_feedback_actor_check", + "value": "\"finding_feedback\".\"actor_github_id\" BETWEEN 1 AND 9007199254740991 AND length(btrim(\"finding_feedback\".\"actor_login_snapshot\")) BETWEEN 1 AND 100 AND \"finding_feedback\".\"pr_author_github_id\" BETWEEN 1 AND 9007199254740991 AND length(btrim(\"finding_feedback\".\"pr_author_login_snapshot\")) BETWEEN 1 AND 100 AND \"finding_feedback\".\"actor_is_pr_author\" = (\"finding_feedback\".\"actor_github_id\" = \"finding_feedback\".\"pr_author_github_id\")" + }, + "finding_feedback_suggested_reason_check": { + "name": "finding_feedback_suggested_reason_check", + "value": "\"finding_feedback\".\"suggested_reason_tag\" IS NULL OR \"finding_feedback\".\"suggested_reason_tag\" IN ('false-positive', 'accepted-risk', 'out-of-scope')" + } + }, + "isRLSEnabled": false + }, + "public.finding_feedback_reconciliations": { + "name": "finding_feedback_reconciliations", + "schema": "", + "columns": { + "finding_publication_id": { + "name": "finding_publication_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_reconcile_at": { + "name": "next_reconcile_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_successful_at": { + "name": "last_successful_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "finding_feedback_reconcile_due_idx": { + "name": "finding_feedback_reconcile_due_idx", + "columns": [ + { + "expression": "next_reconcile_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "finding_feedback_reconciliations_finding_publication_id_finding_publications_id_fk": { + "name": "finding_feedback_reconciliations_finding_publication_id_finding_publications_id_fk", + "tableFrom": "finding_feedback_reconciliations", + "tableTo": "finding_publications", + "columnsFrom": [ + "finding_publication_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_feedback_reconcile_attempt_count_check": { + "name": "finding_feedback_reconcile_attempt_count_check", + "value": "\"finding_feedback_reconciliations\".\"attempt_count\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.finding_publications": { + "name": "finding_publications", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "finding_publications_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "finding_id": { + "name": "finding_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "stable_identity": { + "name": "stable_identity", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "initial_state": { + "name": "initial_state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_state": { + "name": "current_state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_comment_id": { + "name": "github_comment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lifecycle_observed_at": { + "name": "lifecycle_observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "finding_publications_review_finding_idx": { + "name": "finding_publications_review_finding_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_publications_comment_idx": { + "name": "finding_publications_comment_idx", + "columns": [ + { + "expression": "github_comment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "finding_publications_stable_finding_idx": { + "name": "finding_publications_stable_finding_idx", + "columns": [ + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stable_identity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "finding_publications_review_id_reviews_id_fk": { + "name": "finding_publications_review_id_reviews_id_fk", + "tableFrom": "finding_publications", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "finding_publications_finding_id_check": { + "name": "finding_publications_finding_id_check", + "value": "length(btrim(\"finding_publications\".\"finding_id\")) BETWEEN 1 AND 500" + }, + "finding_publications_initial_state_check": { + "name": "finding_publications_initial_state_check", + "value": "\"finding_publications\".\"initial_state\" IN ('inline', 'fileComment', 'checkAnnotation', 'summaryOnly', 'carried', 'resolved', 'suppressed', 'inlineRejected', 'unknown')" + }, + "finding_publications_current_state_check": { + "name": "finding_publications_current_state_check", + "value": "\"finding_publications\".\"current_state\" IN ('inline', 'fileComment', 'checkAnnotation', 'summaryOnly', 'carried', 'resolved', 'suppressed', 'inlineRejected', 'outdated', 'deleted', 'unknown')" + }, + "finding_publications_github_comment_id_check": { + "name": "finding_publications_github_comment_id_check", + "value": "\"finding_publications\".\"github_comment_id\" IS NULL OR \"finding_publications\".\"github_comment_id\" ~ '^[1-9][0-9]{0,19}$'" + }, + "finding_publications_file_comment_identity_check": { + "name": "finding_publications_file_comment_identity_check", + "value": "(\"finding_publications\".\"initial_state\" <> 'fileComment' AND \"finding_publications\".\"current_state\" <> 'fileComment') OR \"finding_publications\".\"github_comment_id\" IS NOT NULL" + } + }, + "isRLSEnabled": false + }, + "public.github_webhook_delivery_recoveries": { + "name": "github_webhook_delivery_recoveries", + "schema": "", + "columns": { + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "delivery_guid": { + "name": "delivery_guid", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redelivery": { + "name": "redelivery", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "request_state": { + "name": "request_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_attempts": { + "name": "request_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_requested_at": { + "name": "last_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "request_status_code": { + "name": "request_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "recovery_delivery_id": { + "name": "recovery_delivery_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "github_webhook_delivery_recoveries_guid_idx": { + "name": "github_webhook_delivery_recoveries_guid_idx", + "columns": [ + { + "expression": "delivery_guid", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "github_webhook_delivery_recoveries_retry_idx": { + "name": "github_webhook_delivery_recoveries_retry_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"github_webhook_delivery_recoveries\".\"outcome\" = 'failure' AND \"github_webhook_delivery_recoveries\".\"recovery_delivery_id\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "github_webhook_delivery_recoveries_outcome_check": { + "name": "github_webhook_delivery_recoveries_outcome_check", + "value": "\"github_webhook_delivery_recoveries\".\"outcome\" IN ('success', 'failure', 'pending')" + }, + "github_webhook_delivery_recoveries_request_state_check": { + "name": "github_webhook_delivery_recoveries_request_state_check", + "value": "\"github_webhook_delivery_recoveries\".\"request_state\" IS NULL OR \"github_webhook_delivery_recoveries\".\"request_state\" IN ('requesting', 'retryable', 'accepted', 'terminal', 'exhausted', 'recovered')" + }, + "github_webhook_delivery_recoveries_attempts_check": { + "name": "github_webhook_delivery_recoveries_attempts_check", + "value": "\"github_webhook_delivery_recoveries\".\"request_attempts\" >= 0 AND \"github_webhook_delivery_recoveries\".\"request_attempts\" <= 2" + } + }, + "isRLSEnabled": false + }, + "public.github_webhook_redelivery_state": { + "name": "github_webhook_redelivery_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sweep_started_at": { + "name": "sweep_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_page_at": { + "name": "last_page_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sweep_completed_at": { + "name": "last_sweep_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "rate_limited_until": { + "name": "rate_limited_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error_category": { + "name": "last_error_category", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "github_webhook_redelivery_state_singleton_check": { + "name": "github_webhook_redelivery_state_singleton_check", + "value": "\"github_webhook_redelivery_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "public.hosted_provider_keys": { + "name": "hosted_provider_keys", + "schema": "", + "columns": { + "create_intent_id": { + "name": "create_intent_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_key_name": { + "name": "provider_key_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_key_hash": { + "name": "provider_key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "conflicting_provider_key_hash": { + "name": "conflicting_provider_key_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sealed_runtime_key": { + "name": "sealed_runtime_key", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "entitlement_period_starts_at": { + "name": "entitlement_period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "entitlement_period_ends_at": { + "name": "entitlement_period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "entitlement_updated_at": { + "name": "entitlement_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "limit_micros": { + "name": "limit_micros", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "create_attempted_at": { + "name": "create_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "create_outcome": { + "name": "create_outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revocation_requested_at": { + "name": "revocation_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoke_attempted_at": { + "name": "revoke_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoke_outcome": { + "name": "revoke_outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reconciliation_required_at": { + "name": "reconciliation_required_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "lease_kind": { + "name": "lease_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "clock_timestamp()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "clock_timestamp()" + } + }, + "indexes": { + "hosted_provider_keys_provider_key_hash_unique": { + "name": "hosted_provider_keys_provider_key_hash_unique", + "columns": [ + { + "expression": "provider_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "hosted_provider_keys_entitlement_binding_unique": { + "name": "hosted_provider_keys_entitlement_binding_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entitlement_period_starts_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entitlement_period_ends_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "limit_micros", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"hosted_provider_keys\".\"state\" NOT IN ('revoked', 'cancelled')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "hosted_provider_keys_active_org_unique": { + "name": "hosted_provider_keys_active_org_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"hosted_provider_keys\".\"state\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "hosted_provider_keys_runtime_org_unique": { + "name": "hosted_provider_keys_runtime_org_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"hosted_provider_keys\".\"sealed_runtime_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "hosted_provider_keys_reconciliation_idx": { + "name": "hosted_provider_keys_reconciliation_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "reconciliation_required_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "hosted_provider_keys_org_id_organizations_id_fk": { + "name": "hosted_provider_keys_org_id_organizations_id_fk", + "tableFrom": "hosted_provider_keys", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "hosted_provider_keys_provider_key_name_unique": { + "name": "hosted_provider_keys_provider_key_name_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_key_name" + ] + } + }, + "policies": {}, + "checkConstraints": { + "hosted_provider_keys_state_check": { + "name": "hosted_provider_keys_state_check", + "value": "\"hosted_provider_keys\".\"state\" IN ('provisioning', 'activating', 'active', 'rejected', 'orphaned', 'revocation_pending', 'revoked', 'cancelled')" + }, + "hosted_provider_keys_provider_key_name_nonempty": { + "name": "hosted_provider_keys_provider_key_name_nonempty", + "value": "length(btrim(\"hosted_provider_keys\".\"provider_key_name\")) > 0" + }, + "hosted_provider_keys_provider_key_hash_nonempty": { + "name": "hosted_provider_keys_provider_key_hash_nonempty", + "value": "\"hosted_provider_keys\".\"provider_key_hash\" IS NULL OR length(btrim(\"hosted_provider_keys\".\"provider_key_hash\")) > 0" + }, + "hosted_provider_keys_conflicting_hash_nonempty": { + "name": "hosted_provider_keys_conflicting_hash_nonempty", + "value": "\"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL OR length(btrim(\"hosted_provider_keys\".\"conflicting_provider_key_hash\")) > 0" + }, + "hosted_provider_keys_entitlement_period_check": { + "name": "hosted_provider_keys_entitlement_period_check", + "value": "\"hosted_provider_keys\".\"entitlement_period_ends_at\" > \"hosted_provider_keys\".\"entitlement_period_starts_at\"" + }, + "hosted_provider_keys_limit_exact_range": { + "name": "hosted_provider_keys_limit_exact_range", + "value": "\"hosted_provider_keys\".\"limit_micros\" > 0 AND \"hosted_provider_keys\".\"limit_micros\" <= 2251799813685247" + }, + "hosted_provider_keys_create_outcome_check": { + "name": "hosted_provider_keys_create_outcome_check", + "value": "\"hosted_provider_keys\".\"create_outcome\" IS NULL OR \"hosted_provider_keys\".\"create_outcome\" IN ('created', 'rejected', 'rate_limited', 'ambiguous', 'name_present', 'name_not_unique', 'credential_persistence_failed', 'intent_changed', 'ownership_conflict')" + }, + "hosted_provider_keys_revoke_outcome_check": { + "name": "hosted_provider_keys_revoke_outcome_check", + "value": "\"hosted_provider_keys\".\"revoke_outcome\" IS NULL OR \"hosted_provider_keys\".\"revoke_outcome\" IN ('ambiguous', 'rejected', 'disabled', 'absent')" + }, + "hosted_provider_keys_lease_shape": { + "name": "hosted_provider_keys_lease_shape", + "value": "(\n \"hosted_provider_keys\".\"lease_id\" IS NULL\n AND \"hosted_provider_keys\".\"lease_kind\" IS NULL\n AND \"hosted_provider_keys\".\"lease_expires_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"lease_id\" IS NOT NULL\n AND \"hosted_provider_keys\".\"lease_kind\" IN ('create', 'revoke')\n AND \"hosted_provider_keys\".\"lease_expires_at\" IS NOT NULL\n )" + }, + "hosted_provider_keys_lease_state": { + "name": "hosted_provider_keys_lease_state", + "value": "\"hosted_provider_keys\".\"lease_id\" IS NULL OR (\n (\"hosted_provider_keys\".\"lease_kind\" = 'create' AND \"hosted_provider_keys\".\"state\" IN ('provisioning', 'activating', 'orphaned'))\n OR (\"hosted_provider_keys\".\"lease_kind\" = 'revoke' AND \"hosted_provider_keys\".\"state\" = 'revocation_pending')\n )" + }, + "hosted_provider_keys_lifecycle_shape": { + "name": "hosted_provider_keys_lifecycle_shape", + "value": "(\n \"hosted_provider_keys\".\"state\" = 'provisioning'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'activating'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" = 'created'\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'active'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NOT NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" = 'created'\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'rejected'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" = 'rejected'\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'orphaned'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IN ('ambiguous', 'name_present', 'name_not_unique', 'credential_persistence_failed', 'intent_changed', 'ownership_conflict')\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n AND (\n (\"hosted_provider_keys\".\"create_outcome\" = 'ownership_conflict' AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NOT NULL)\n OR (\"hosted_provider_keys\".\"create_outcome\" <> 'ownership_conflict' AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL)\n )\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'revocation_pending'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IN ('created', 'ambiguous', 'credential_persistence_failed')\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'revoked'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NOT NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"create_outcome\" IN ('created', 'ambiguous', 'credential_persistence_failed')\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IN ('disabled', 'absent')\n AND \"hosted_provider_keys\".\"revoked_at\" IS NOT NULL\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"lease_id\" IS NULL\n ) OR (\n \"hosted_provider_keys\".\"state\" = 'cancelled'\n AND \"hosted_provider_keys\".\"sealed_runtime_key\" IS NULL\n AND \"hosted_provider_keys\".\"provider_key_hash\" IS NULL\n AND \"hosted_provider_keys\".\"conflicting_provider_key_hash\" IS NULL\n AND (\n (\"hosted_provider_keys\".\"create_attempted_at\" IS NULL AND \"hosted_provider_keys\".\"create_outcome\" IS NULL)\n OR (\"hosted_provider_keys\".\"create_attempted_at\" IS NOT NULL AND \"hosted_provider_keys\".\"create_outcome\" = 'rate_limited')\n )\n AND \"hosted_provider_keys\".\"revocation_requested_at\" IS NULL\n AND \"hosted_provider_keys\".\"revoke_outcome\" IS NULL\n AND \"hosted_provider_keys\".\"revoked_at\" IS NULL\n AND \"hosted_provider_keys\".\"reconciliation_required_at\" IS NULL\n AND \"hosted_provider_keys\".\"lease_id\" IS NULL\n )" + } + }, + "isRLSEnabled": false + }, + "public.hosted_usage_reservations": { + "name": "hosted_usage_reservations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'review'" + }, + "reserved_micros": { + "name": "reserved_micros", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "actual_micros": { + "name": "actual_micros", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "hosted_usage_reservations_review_idx": { + "name": "hosted_usage_reservations_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "hosted_usage_reservations_active_org_expiry_idx": { + "name": "hosted_usage_reservations_active_org_expiry_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"hosted_usage_reservations\".\"status\" = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "hosted_usage_reservations_org_id_organizations_id_fk": { + "name": "hosted_usage_reservations_org_id_organizations_id_fk", + "tableFrom": "hosted_usage_reservations", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "hosted_usage_reservations_review_id_reviews_id_fk": { + "name": "hosted_usage_reservations_review_id_reviews_id_fk", + "tableFrom": "hosted_usage_reservations", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "hosted_usage_reservations_status_check": { + "name": "hosted_usage_reservations_status_check", + "value": "\"hosted_usage_reservations\".\"status\" IN ('active', 'reconciled', 'released')" + }, + "hosted_usage_reservations_operation_check": { + "name": "hosted_usage_reservations_operation_check", + "value": "\"hosted_usage_reservations\".\"operation\" IN ('review', 'respond', 'cli_gateway')" + }, + "hosted_usage_reservations_operation_reference_check": { + "name": "hosted_usage_reservations_operation_reference_check", + "value": "(\"hosted_usage_reservations\".\"operation\" = 'review' AND \"hosted_usage_reservations\".\"review_id\" IS NOT NULL) OR (\"hosted_usage_reservations\".\"operation\" IN ('respond', 'cli_gateway') AND \"hosted_usage_reservations\".\"review_id\" IS NULL)" + }, + "hosted_usage_reservations_reserved_positive": { + "name": "hosted_usage_reservations_reserved_positive", + "value": "\"hosted_usage_reservations\".\"reserved_micros\" > 0" + }, + "hosted_usage_reservations_actual_nonnegative": { + "name": "hosted_usage_reservations_actual_nonnegative", + "value": "\"hosted_usage_reservations\".\"actual_micros\" IS NULL OR \"hosted_usage_reservations\".\"actual_micros\" >= 0" + } + }, + "isRLSEnabled": false + }, + "public.ilert_alert_events": { + "name": "ilert_alert_events", + "schema": "", + "columns": { + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "ilert_alert_events_sequence_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "event_id": { + "name": "event_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "alert_id": { + "name": "alert_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "alert_key": { + "name": "alert_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "priority": { + "name": "priority", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "alert_source_id": { + "name": "alert_source_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "alert_source_name": { + "name": "alert_source_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "report_time": { + "name": "report_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "payload_sha256": { + "name": "payload_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "ilert_alert_events_event_id_idx": { + "name": "ilert_alert_events_event_id_idx", + "columns": [ + { + "expression": "event_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ilert_alert_events_alert_sequence_idx": { + "name": "ilert_alert_events_alert_sequence_idx", + "columns": [ + { + "expression": "alert_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "ilert_alert_events_canary_observation_idx": { + "name": "ilert_alert_events_canary_observation_idx", + "columns": [ + { + "expression": "alert_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "alert_source_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"ilert_alert_events\".\"alert_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "ilert_alert_events_alert_id_check": { + "name": "ilert_alert_events_alert_id_check", + "value": "\"ilert_alert_events\".\"alert_id\" ~ '^[1-9][0-9]{0,63}$'" + }, + "ilert_alert_events_alert_key_check": { + "name": "ilert_alert_events_alert_key_check", + "value": "\"ilert_alert_events\".\"alert_key\" IS NULL OR length(\"ilert_alert_events\".\"alert_key\") BETWEEN 1 AND 512" + }, + "ilert_alert_events_event_type_check": { + "name": "ilert_alert_events_event_type_check", + "value": "length(\"ilert_alert_events\".\"event_type\") <= 64 AND \"ilert_alert_events\".\"event_type\" ~ '^alert-[a-z]+(-[a-z]+)*$'" + }, + "ilert_alert_events_status_check": { + "name": "ilert_alert_events_status_check", + "value": "\"ilert_alert_events\".\"status\" IN ('PENDING', 'ACCEPTED', 'RESOLVED')" + }, + "ilert_alert_events_priority_check": { + "name": "ilert_alert_events_priority_check", + "value": "\"ilert_alert_events\".\"priority\" IN ('HIGH', 'LOW')" + }, + "ilert_alert_events_summary_check": { + "name": "ilert_alert_events_summary_check", + "value": "length(\"ilert_alert_events\".\"summary\") BETWEEN 1 AND 512" + }, + "ilert_alert_events_details_check": { + "name": "ilert_alert_events_details_check", + "value": "length(\"ilert_alert_events\".\"details\") BETWEEN 0 AND 8192" + }, + "ilert_alert_events_source_name_check": { + "name": "ilert_alert_events_source_name_check", + "value": "length(\"ilert_alert_events\".\"alert_source_name\") BETWEEN 1 AND 256" + }, + "ilert_alert_events_payload_sha256_check": { + "name": "ilert_alert_events_payload_sha256_check", + "value": "\"ilert_alert_events\".\"payload_sha256\" ~ '^[0-9a-f]{64}$'" + } + }, + "isRLSEnabled": false + }, + "public.installations": { + "name": "installations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "installations_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "github_installation_id": { + "name": "github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suspended": { + "name": "suspended", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "installations_org_idx": { + "name": "installations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "installations_org_id_organizations_id_fk": { + "name": "installations_org_id_organizations_id_fk", + "tableFrom": "installations", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "installations_github_installation_id_unique": { + "name": "installations_github_installation_id_unique", + "nullsNotDistinct": false, + "columns": [ + "github_installation_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "jobs_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "max_attempts": { + "name": "max_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "run_after": { + "name": "run_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "locked_at": { + "name": "locked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "locked_by": { + "name": "locked_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "jobs_claim_idx": { + "name": "jobs_claim_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_after", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "jobs_running_locked_at_idx": { + "name": "jobs_running_locked_at_idx", + "columns": [ + { + "expression": "locked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"jobs\".\"status\" = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "jobs_running_org_concurrency_idx": { + "name": "jobs_running_org_concurrency_idx", + "columns": [ + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "(\"payload\"->>'sourceOrgId')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"jobs\".\"status\" = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.large_review_attempts": { + "name": "large_review_attempts", + "schema": "", + "columns": { + "attempt_key": { + "name": "attempt_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "run_key": { + "name": "run_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_sha256": { + "name": "request_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "batch_identity": { + "name": "batch_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "response_status": { + "name": "response_status", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "response_headers": { + "name": "response_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "response_body": { + "name": "response_body", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "large_review_attempts_run_request_attempt_idx": { + "name": "large_review_attempts_run_request_attempt_idx", + "columns": [ + { + "expression": "run_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "large_review_attempts_pending_request_idx": { + "name": "large_review_attempts_pending_request_idx", + "columns": [ + { + "expression": "run_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"large_review_attempts\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "large_review_attempts_run_idx": { + "name": "large_review_attempts_run_idx", + "columns": [ + { + "expression": "run_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "large_review_attempts_run_key_large_review_runs_run_key_fk": { + "name": "large_review_attempts_run_key_large_review_runs_run_key_fk", + "tableFrom": "large_review_attempts", + "tableTo": "large_review_runs", + "columnsFrom": [ + "run_key" + ], + "columnsTo": [ + "run_key" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "large_review_attempts_key_check": { + "name": "large_review_attempts_key_check", + "value": "\"large_review_attempts\".\"attempt_key\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_attempts_request_check": { + "name": "large_review_attempts_request_check", + "value": "\"large_review_attempts\".\"request_sha256\" ~ '^[0-9a-f]{64}$' AND \"large_review_attempts\".\"batch_identity\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_attempts_attempt_check": { + "name": "large_review_attempts_attempt_check", + "value": "\"large_review_attempts\".\"attempt\" BETWEEN 1 AND 10" + }, + "large_review_attempts_state_check": { + "name": "large_review_attempts_state_check", + "value": "\"large_review_attempts\".\"state\" IN ('pending', 'completed')" + }, + "large_review_attempts_response_check": { + "name": "large_review_attempts_response_check", + "value": "(\"large_review_attempts\".\"state\" = 'pending' AND \"large_review_attempts\".\"response_status\" IS NULL AND \"large_review_attempts\".\"response_headers\" IS NULL AND \"large_review_attempts\".\"response_body\" IS NULL AND \"large_review_attempts\".\"completed_at\" IS NULL) OR (\"large_review_attempts\".\"state\" = 'completed' AND \"large_review_attempts\".\"response_status\" BETWEEN 200 AND 299 AND \"large_review_attempts\".\"response_headers\" IS NOT NULL AND \"large_review_attempts\".\"response_body\" IS NOT NULL AND \"large_review_attempts\".\"completed_at\" IS NOT NULL)" + }, + "large_review_attempts_model_check": { + "name": "large_review_attempts_model_check", + "value": "length(btrim(\"large_review_attempts\".\"model\")) BETWEEN 1 AND 500" + } + }, + "isRLSEnabled": false + }, + "public.large_review_runs": { + "name": "large_review_runs", + "schema": "", + "columns": { + "run_key": { + "name": "run_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "current_review_id": { + "name": "current_review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "cli_version": { + "name": "cli_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "configuration_sha256": { + "name": "configuration_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_identity": { + "name": "provider_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_sha": { + "name": "base_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "retry_lineage": { + "name": "retry_lineage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plan_sha256": { + "name": "plan_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hosted_reservation_id": { + "name": "hosted_reservation_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "billing_state": { + "name": "billing_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "conservatively_settled_at": { + "name": "conservatively_settled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "large_review_runs_expiry_idx": { + "name": "large_review_runs_expiry_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "large_review_runs_resume_identity_idx": { + "name": "large_review_runs_resume_identity_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pr_number", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "head_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "base_sha", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cli_version", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "configuration_sha256", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "retry_lineage", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "large_review_runs_current_review_id_reviews_id_fk": { + "name": "large_review_runs_current_review_id_reviews_id_fk", + "tableFrom": "large_review_runs", + "tableTo": "reviews", + "columnsFrom": [ + "current_review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "large_review_runs_repository_id_repositories_id_fk": { + "name": "large_review_runs_repository_id_repositories_id_fk", + "tableFrom": "large_review_runs", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "large_review_runs_key_check": { + "name": "large_review_runs_key_check", + "value": "\"large_review_runs\".\"run_key\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_runs_configuration_check": { + "name": "large_review_runs_configuration_check", + "value": "\"large_review_runs\".\"configuration_sha256\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_runs_plan_check": { + "name": "large_review_runs_plan_check", + "value": "\"large_review_runs\".\"plan_sha256\" ~ '^[0-9a-f]{64}$'" + }, + "large_review_runs_identity_lengths_check": { + "name": "large_review_runs_identity_lengths_check", + "value": "\"large_review_runs\".\"pr_number\" > 0 AND length(btrim(\"large_review_runs\".\"cli_version\")) BETWEEN 1 AND 100 AND length(btrim(\"large_review_runs\".\"provider_identity\")) BETWEEN 1 AND 2048 AND length(btrim(\"large_review_runs\".\"head_sha\")) BETWEEN 1 AND 200 AND length(btrim(\"large_review_runs\".\"base_sha\")) BETWEEN 1 AND 200 AND length(btrim(\"large_review_runs\".\"retry_lineage\")) BETWEEN 1 AND 200" + }, + "large_review_runs_billing_state_check": { + "name": "large_review_runs_billing_state_check", + "value": "(\"large_review_runs\".\"billing_state\" = 'active' AND \"large_review_runs\".\"conservatively_settled_at\" IS NULL) OR (\"large_review_runs\".\"billing_state\" = 'conservative' AND \"large_review_runs\".\"conservatively_settled_at\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.operator_alert_deliveries": { + "name": "operator_alert_deliveries", + "schema": "", + "columns": { + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "github_installation_id": { + "name": "github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "message_id": { + "name": "message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "operator_alert_deliveries_status_created_idx": { + "name": "operator_alert_deliveries_status_created_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "operator_alert_deliveries_org_created_idx": { + "name": "operator_alert_deliveries_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "operator_alert_deliveries_org_id_organizations_id_fk": { + "name": "operator_alert_deliveries_org_id_organizations_id_fk", + "tableFrom": "operator_alert_deliveries", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "operator_alert_deliveries_event_check": { + "name": "operator_alert_deliveries_event_check", + "value": "\"operator_alert_deliveries\".\"event\" IN ('trial_started', 'trial_expired', 'installation_removed', 'subscription_started', 'subscription_past_due', 'subscription_paused', 'subscription_canceled', 'billing_anomaly', 'finding_feedback_digest')" + }, + "operator_alert_deliveries_status_check": { + "name": "operator_alert_deliveries_status_check", + "value": "\"operator_alert_deliveries\".\"status\" IN ('queued', 'retrying', 'delivered', 'failed')" + }, + "operator_alert_deliveries_event_key_nonempty": { + "name": "operator_alert_deliveries_event_key_nonempty", + "value": "length(btrim(\"operator_alert_deliveries\".\"event_key\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.org_config_probe_refreshes": { + "name": "org_config_probe_refreshes", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "org_config_probe_refreshes_org_id_organizations_id_fk": { + "name": "org_config_probe_refreshes_org_id_organizations_id_fk", + "tableFrom": "org_config_probe_refreshes", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_config_snapshots": { + "name": "org_config_snapshots", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "source_repository_id": { + "name": "source_repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source_full_name": { + "name": "source_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "guardrails_md": { + "name": "guardrails_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_policy_md": { + "name": "content_policy_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "files": { + "name": "files", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "loaded_files": { + "name": "loaded_files", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "stale": { + "name": "stale", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "org_config_snapshots_org_id_organizations_id_fk": { + "name": "org_config_snapshots_org_id_organizations_id_fk", + "tableFrom": "org_config_snapshots", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "org_config_snapshots_source_repository_id_repositories_id_fk": { + "name": "org_config_snapshots_source_repository_id_repositories_id_fk", + "tableFrom": "org_config_snapshots", + "tableTo": "repositories", + "columnsFrom": [ + "source_repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_members": { + "name": "org_members", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "org_members_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + } + }, + "indexes": { + "org_members_org_user_idx": { + "name": "org_members_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_members_user_idx": { + "name": "org_members_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "org_members_org_id_organizations_id_fk": { + "name": "org_members_org_id_organizations_id_fk", + "tableFrom": "org_members", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "org_members_user_id_users_id_fk": { + "name": "org_members_user_id_users_id_fk", + "tableFrom": "org_members", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_settings": { + "name": "org_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "api_base": { + "name": "api_base", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_ciphertext": { + "name": "api_key_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "api_format": { + "name": "api_format", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'openai-compatible'" + }, + "api_auth_header_ciphertext": { + "name": "api_auth_header_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "api_auth_value_ciphertext": { + "name": "api_auth_value_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_cascade": { + "name": "model_cascade", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config_yaml": { + "name": "config_yaml", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "guardrails_md": { + "name": "guardrails_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_policy_md": { + "name": "content_policy_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "shared_config_enabled": { + "name": "shared_config_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "gate_enabled": { + "name": "gate_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "escalation_email": { + "name": "escalation_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "escalation_email_pending": { + "name": "escalation_email_pending", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verified_at": { + "name": "escalation_email_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_token_digest": { + "name": "escalation_email_verification_token_digest", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_token_ciphertext": { + "name": "escalation_email_verification_token_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_expires_at": { + "name": "escalation_email_verification_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_requested_at": { + "name": "escalation_email_verification_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_sent_at": { + "name": "escalation_email_verification_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "escalation_email_verification_message_id": { + "name": "escalation_email_verification_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "org_settings_org_id_organizations_id_fk": { + "name": "org_settings_org_id_organizations_id_fk", + "tableFrom": "org_settings", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_entitlements": { + "name": "organization_entitlements", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "subscription_mode": { + "name": "subscription_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "trial_ends_at": { + "name": "trial_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "past_due_grace_ends_at": { + "name": "past_due_grace_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "period_starts_at": { + "name": "period_starts_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "period_ends_at": { + "name": "period_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "included_usage_micros": { + "name": "included_usage_micros", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "overage_hard_cap_micros": { + "name": "overage_hard_cap_micros", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "0" + }, + "included_usage_cents": { + "name": "included_usage_cents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "overage_hard_cap_cents": { + "name": "overage_hard_cap_cents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "billing_contact_email": { + "name": "billing_contact_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verified_at": { + "name": "billing_contact_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_pending": { + "name": "billing_contact_pending", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_token_digest": { + "name": "billing_contact_verification_token_digest", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_token_ciphertext": { + "name": "billing_contact_verification_token_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_expires_at": { + "name": "billing_contact_verification_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_requested_at": { + "name": "billing_contact_verification_requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_sent_at": { + "name": "billing_contact_verification_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "billing_contact_verification_message_id": { + "name": "billing_contact_verification_message_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "promotional_eligible": { + "name": "promotional_eligible", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "promotional_ends_at": { + "name": "promotional_ends_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "organization_entitlements_org_id_organizations_id_fk": { + "name": "organization_entitlements_org_id_organizations_id_fk", + "tableFrom": "organization_entitlements", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "organization_entitlements_subscription_mode_check": { + "name": "organization_entitlements_subscription_mode_check", + "value": "\"organization_entitlements\".\"subscription_mode\" IN ('hosted', 'byok')" + }, + "organization_entitlements_status_check": { + "name": "organization_entitlements_status_check", + "value": "\"organization_entitlements\".\"status\" IN ('active', 'trialing', 'past_due', 'suspended')" + }, + "organization_entitlements_included_usage_micros_nonnegative": { + "name": "organization_entitlements_included_usage_micros_nonnegative", + "value": "\"organization_entitlements\".\"included_usage_micros\" >= 0" + }, + "organization_entitlements_overage_cap_micros_nonnegative": { + "name": "organization_entitlements_overage_cap_micros_nonnegative", + "value": "\"organization_entitlements\".\"overage_hard_cap_micros\" IS NULL OR \"organization_entitlements\".\"overage_hard_cap_micros\" >= 0" + }, + "organization_entitlements_included_usage_nonnegative": { + "name": "organization_entitlements_included_usage_nonnegative", + "value": "\"organization_entitlements\".\"included_usage_cents\" >= 0" + }, + "organization_entitlements_overage_cap_nonnegative": { + "name": "organization_entitlements_overage_cap_nonnegative", + "value": "\"organization_entitlements\".\"overage_hard_cap_cents\" IS NULL OR \"organization_entitlements\".\"overage_hard_cap_cents\" >= 0" + }, + "organization_entitlements_updated_by_nonempty": { + "name": "organization_entitlements_updated_by_nonempty", + "value": "length(btrim(\"organization_entitlements\".\"updated_by\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.organization_notification_preferences": { + "name": "organization_notification_preferences", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "billing_summary_email": { + "name": "billing_summary_email", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "service_summary_email": { + "name": "service_summary_email", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "organization_notification_preferences_org_id_organizations_id_fk": { + "name": "organization_notification_preferences_org_id_organizations_id_fk", + "tableFrom": "organization_notification_preferences", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization_setting_events": { + "name": "organization_setting_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "organization_setting_events_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "setting": { + "name": "setting", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'dashboard'" + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organization_setting_events_org_time_idx": { + "name": "organization_setting_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "organization_setting_events_org_id_organizations_id_fk": { + "name": "organization_setting_events_org_id_organizations_id_fk", + "tableFrom": "organization_setting_events", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "organization_setting_events_actor_user_id_users_id_fk": { + "name": "organization_setting_events_actor_user_id_users_id_fk", + "tableFrom": "organization_setting_events", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "organization_setting_events_setting_check": { + "name": "organization_setting_events_setting_check", + "value": "\"organization_setting_events\".\"setting\" IN ('gate_enabled', 'billing_summary_email', 'service_summary_email')" + }, + "organization_setting_events_value_check": { + "name": "organization_setting_events_value_check", + "value": "\"organization_setting_events\".\"value\" IN ('enabled', 'disabled', 'advisory')" + }, + "organization_setting_events_source_check": { + "name": "organization_setting_events_source_check", + "value": "\"organization_setting_events\".\"source\" IN ('dashboard')" + } + }, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "organizations_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_org_id": { + "name": "github_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'beta'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_github_org_id_idx": { + "name": "organizations_github_org_id_idx", + "columns": [ + { + "expression": "github_org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_monitor_check_failures": { + "name": "private_monitor_check_failures", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "private_monitor_check_failures_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "run_id": { + "name": "run_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "recovered": { + "name": "recovered", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "private_monitor_check_failures_observed_idx": { + "name": "private_monitor_check_failures_observed_idx", + "columns": [ + { + "expression": "observed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_check_failures_attempt_check": { + "name": "private_monitor_check_failures_attempt_check", + "value": "\"private_monitor_check_failures\".\"attempt\" > 0" + }, + "private_monitor_check_failures_key_nonempty": { + "name": "private_monitor_check_failures_key_nonempty", + "value": "length(btrim(\"private_monitor_check_failures\".\"key\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.private_monitor_incident_events": { + "name": "private_monitor_incident_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "private_monitor_incident_events_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group": { + "name": "group", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "transition": { + "name": "transition", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "first_detected_at": { + "name": "first_detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "private_monitor_incident_events_occurred_idx": { + "name": "private_monitor_incident_events_occurred_idx", + "columns": [ + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_incident_events_transition_check": { + "name": "private_monitor_incident_events_transition_check", + "value": "\"private_monitor_incident_events\".\"transition\" IN ('opened', 'resolved')" + }, + "private_monitor_incident_events_severity_check": { + "name": "private_monitor_incident_events_severity_check", + "value": "\"private_monitor_incident_events\".\"severity\" IN ('warning', 'critical')" + }, + "private_monitor_incident_events_text_nonempty": { + "name": "private_monitor_incident_events_text_nonempty", + "value": "length(btrim(\"private_monitor_incident_events\".\"key\")) > 0 AND length(btrim(\"private_monitor_incident_events\".\"group\")) > 0 AND length(btrim(\"private_monitor_incident_events\".\"summary\")) > 0 AND \"private_monitor_incident_events\".\"occurrence_count\" > 0" + } + }, + "isRLSEnabled": false + }, + "public.private_monitor_incidents": { + "name": "private_monitor_incidents", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "group": { + "name": "group", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "opened_detail": { + "name": "opened_detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "first_detected_at": { + "name": "first_detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_detected_at": { + "name": "last_detected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "pending_notification_key": { + "name": "pending_notification_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pending_notification_kind": { + "name": "pending_notification_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_attempts": { + "name": "notification_attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "notification_available_at": { + "name": "notification_available_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "notification_lease_owner": { + "name": "notification_lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_lease_expires_at": { + "name": "notification_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_notification_error": { + "name": "last_notification_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_notification_key": { + "name": "last_notification_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_delivery_receipt": { + "name": "last_delivery_receipt", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "private_monitor_incidents_state_updated_idx": { + "name": "private_monitor_incidents_state_updated_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "private_monitor_incidents_notification_idx": { + "name": "private_monitor_incidents_notification_idx", + "columns": [ + { + "expression": "notification_available_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "notification_lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"private_monitor_incidents\".\"pending_notification_key\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_delivery_receipt_bound": { + "name": "private_monitor_delivery_receipt_bound", + "value": "(\"last_delivery_receipt\" IS NULL AND \"last_notification_key\" IS NULL) OR (\"last_delivery_receipt\" IS NOT NULL AND \"last_notification_key\" IS NOT NULL AND length(\"last_notification_key\") BETWEEN 1 AND 512 AND jsonb_typeof(\"last_delivery_receipt\") = 'object' AND octet_length(\"last_delivery_receipt\"::text) <= 2048)" + }, + "private_monitor_incidents_state_check": { + "name": "private_monitor_incidents_state_check", + "value": "\"private_monitor_incidents\".\"state\" IN ('open', 'resolved')" + }, + "private_monitor_incidents_severity_check": { + "name": "private_monitor_incidents_severity_check", + "value": "\"private_monitor_incidents\".\"severity\" IN ('warning', 'critical')" + }, + "private_monitor_incidents_notification_kind_check": { + "name": "private_monitor_incidents_notification_kind_check", + "value": "\"private_monitor_incidents\".\"pending_notification_kind\" IS NULL OR \"private_monitor_incidents\".\"pending_notification_kind\" IN ('opened', 'reminder', 'resolved')" + }, + "private_monitor_incidents_notification_pair_check": { + "name": "private_monitor_incidents_notification_pair_check", + "value": "(\"private_monitor_incidents\".\"pending_notification_key\" IS NULL) = (\"private_monitor_incidents\".\"pending_notification_kind\" IS NULL)" + }, + "private_monitor_incidents_occurrence_count_check": { + "name": "private_monitor_incidents_occurrence_count_check", + "value": "\"private_monitor_incidents\".\"occurrence_count\" > 0 AND \"private_monitor_incidents\".\"notification_attempts\" >= 0 AND \"private_monitor_incidents\".\"notification_attempts\" <= 5" + }, + "private_monitor_incidents_text_nonempty": { + "name": "private_monitor_incidents_text_nonempty", + "value": "length(btrim(\"private_monitor_incidents\".\"key\")) > 0 AND length(btrim(\"private_monitor_incidents\".\"group\")) > 0 AND length(btrim(\"private_monitor_incidents\".\"summary\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.private_monitor_runs": { + "name": "private_monitor_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "private_monitor_runs_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "scheduled_for": { + "name": "scheduled_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "check_count": { + "name": "check_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "failure_count": { + "name": "failure_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "private_monitor_runs_scheduled_idx": { + "name": "private_monitor_runs_scheduled_idx", + "columns": [ + { + "expression": "scheduled_for", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "private_monitor_runs_started_idx": { + "name": "private_monitor_runs_started_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_runs_status_check": { + "name": "private_monitor_runs_status_check", + "value": "\"private_monitor_runs\".\"status\" IN ('running', 'completed', 'failed')" + }, + "private_monitor_runs_counts_check": { + "name": "private_monitor_runs_counts_check", + "value": "\"private_monitor_runs\".\"check_count\" >= 0 AND \"private_monitor_runs\".\"failure_count\" >= 0 AND \"private_monitor_runs\".\"failure_count\" <= \"private_monitor_runs\".\"check_count\"" + }, + "private_monitor_runs_owner_nonempty": { + "name": "private_monitor_runs_owner_nonempty", + "value": "length(btrim(\"private_monitor_runs\".\"owner\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.private_monitor_state": { + "name": "private_monitor_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_started_at": { + "name": "last_started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_completed_at": { + "name": "last_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "heartbeat_delivery_error": { + "name": "heartbeat_delivery_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_monitor_state_singleton_check": { + "name": "private_monitor_state_singleton_check", + "value": "\"private_monitor_state\".\"id\" = 1" + } + }, + "isRLSEnabled": false + }, + "public.private_worker_rehearsals": { + "name": "private_worker_rehearsals", + "schema": "", + "columns": { + "nonce": { + "name": "nonce", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'armed'" + }, + "operator_github_id": { + "name": "operator_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "job_id": { + "name": "job_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "org_slug": { + "name": "org_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repo_full_name": { + "name": "repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "review_public_id": { + "name": "review_public_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "armed_at": { + "name": "armed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "interrupted_worker_instance": { + "name": "interrupted_worker_instance", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "replacement_worker_instance": { + "name": "replacement_worker_instance", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "replacement_observed_at": { + "name": "replacement_observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "before_review_count": { + "name": "before_review_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "before_usage_count": { + "name": "before_usage_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "before_check_count": { + "name": "before_check_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "before_publication_count": { + "name": "before_publication_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_review_count": { + "name": "after_review_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_usage_count": { + "name": "after_usage_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_check_count": { + "name": "after_check_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "after_publication_count": { + "name": "after_publication_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "private_worker_rehearsals_review_idx": { + "name": "private_worker_rehearsals_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "private_worker_rehearsals_job_idx": { + "name": "private_worker_rehearsals_job_idx", + "columns": [ + { + "expression": "job_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "private_worker_rehearsals_state_idx": { + "name": "private_worker_rehearsals_state_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "private_worker_rehearsals_org_id_organizations_id_fk": { + "name": "private_worker_rehearsals_org_id_organizations_id_fk", + "tableFrom": "private_worker_rehearsals", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "private_worker_rehearsals_repository_id_repositories_id_fk": { + "name": "private_worker_rehearsals_repository_id_repositories_id_fk", + "tableFrom": "private_worker_rehearsals", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "private_worker_rehearsals_review_id_reviews_id_fk": { + "name": "private_worker_rehearsals_review_id_reviews_id_fk", + "tableFrom": "private_worker_rehearsals", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, + "private_worker_rehearsals_job_id_jobs_id_fk": { + "name": "private_worker_rehearsals_job_id_jobs_id_fk", + "tableFrom": "private_worker_rehearsals", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "private_worker_rehearsals_state_check": { + "name": "private_worker_rehearsals_state_check", + "value": "\"private_worker_rehearsals\".\"state\" IN ('armed', 'awaiting_replacement', 'replacement_verified', 'completed', 'expired', 'failed')" + }, + "private_worker_rehearsals_identity_check": { + "name": "private_worker_rehearsals_identity_check", + "value": "length(btrim(\"private_worker_rehearsals\".\"org_slug\")) > 0 AND length(btrim(\"private_worker_rehearsals\".\"repo_full_name\")) > 0 AND \"private_worker_rehearsals\".\"pr_number\" > 0 AND \"private_worker_rehearsals\".\"head_sha\" ~ '^[0-9a-f]{40}$'" + }, + "private_worker_rehearsals_arming_window_check": { + "name": "private_worker_rehearsals_arming_window_check", + "value": "\"private_worker_rehearsals\".\"expires_at\" > \"private_worker_rehearsals\".\"armed_at\" AND \"private_worker_rehearsals\".\"expires_at\" <= \"private_worker_rehearsals\".\"armed_at\" + interval '10 minutes'" + }, + "private_worker_rehearsals_before_counts_check": { + "name": "private_worker_rehearsals_before_counts_check", + "value": "(\"private_worker_rehearsals\".\"before_review_count\" IS NULL AND \"private_worker_rehearsals\".\"before_usage_count\" IS NULL AND \"private_worker_rehearsals\".\"before_check_count\" IS NULL AND \"private_worker_rehearsals\".\"before_publication_count\" IS NULL) OR (\"private_worker_rehearsals\".\"before_review_count\" >= 0 AND \"private_worker_rehearsals\".\"before_usage_count\" >= 0 AND \"private_worker_rehearsals\".\"before_check_count\" >= 0 AND \"private_worker_rehearsals\".\"before_publication_count\" >= 0)" + }, + "private_worker_rehearsals_after_counts_check": { + "name": "private_worker_rehearsals_after_counts_check", + "value": "(\"private_worker_rehearsals\".\"after_review_count\" IS NULL AND \"private_worker_rehearsals\".\"after_usage_count\" IS NULL AND \"private_worker_rehearsals\".\"after_check_count\" IS NULL AND \"private_worker_rehearsals\".\"after_publication_count\" IS NULL) OR (\"private_worker_rehearsals\".\"after_review_count\" >= 0 AND \"private_worker_rehearsals\".\"after_usage_count\" >= 0 AND \"private_worker_rehearsals\".\"after_check_count\" >= 0 AND \"private_worker_rehearsals\".\"after_publication_count\" >= 0)" + }, + "private_worker_rehearsals_replacement_pair_check": { + "name": "private_worker_rehearsals_replacement_pair_check", + "value": "(\"private_worker_rehearsals\".\"replacement_worker_instance\" IS NULL) = (\"private_worker_rehearsals\".\"replacement_observed_at\" IS NULL)" + }, + "private_worker_rehearsals_consumed_state_check": { + "name": "private_worker_rehearsals_consumed_state_check", + "value": "(\"private_worker_rehearsals\".\"state\" IN ('armed', 'expired') AND \"private_worker_rehearsals\".\"consumed_at\" IS NULL AND \"private_worker_rehearsals\".\"interrupted_worker_instance\" IS NULL AND \"private_worker_rehearsals\".\"before_review_count\" IS NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('awaiting_replacement', 'replacement_verified', 'completed', 'failed') AND \"private_worker_rehearsals\".\"consumed_at\" IS NOT NULL AND \"private_worker_rehearsals\".\"interrupted_worker_instance\" IS NOT NULL AND \"private_worker_rehearsals\".\"before_review_count\" IS NOT NULL)" + }, + "private_worker_rehearsals_replacement_state_check": { + "name": "private_worker_rehearsals_replacement_state_check", + "value": "(\"private_worker_rehearsals\".\"state\" IN ('armed', 'awaiting_replacement', 'expired') AND \"private_worker_rehearsals\".\"replacement_worker_instance\" IS NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('replacement_verified', 'completed') AND \"private_worker_rehearsals\".\"replacement_worker_instance\" IS NOT NULL) OR \"private_worker_rehearsals\".\"state\" = 'failed'" + }, + "private_worker_rehearsals_completion_state_check": { + "name": "private_worker_rehearsals_completion_state_check", + "value": "(\"private_worker_rehearsals\".\"state\" = 'completed' AND \"private_worker_rehearsals\".\"after_review_count\" IS NOT NULL AND \"private_worker_rehearsals\".\"completed_at\" IS NOT NULL AND \"private_worker_rehearsals\".\"failure_reason\" IS NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('expired', 'failed') AND \"private_worker_rehearsals\".\"after_review_count\" IS NULL AND \"private_worker_rehearsals\".\"completed_at\" IS NOT NULL AND \"private_worker_rehearsals\".\"failure_reason\" IS NOT NULL) OR (\"private_worker_rehearsals\".\"state\" IN ('armed', 'awaiting_replacement', 'replacement_verified') AND \"private_worker_rehearsals\".\"after_review_count\" IS NULL AND \"private_worker_rehearsals\".\"completed_at\" IS NULL AND \"private_worker_rehearsals\".\"failure_reason\" IS NULL)" + } + }, + "isRLSEnabled": false + }, + "public.release_steps": { + "name": "release_steps", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "details": { + "name": "details", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repo_config_probes": { + "name": "repo_config_probes", + "schema": "", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "probed_at": { + "name": "probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "files": { + "name": "files", + "type": "text[]", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "repo_config_probes_repository_id_repositories_id_fk": { + "name": "repo_config_probes_repository_id_repositories_id_fk", + "tableFrom": "repo_config_probes", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repositories": { + "name": "repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "repositories_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "installation_id": { + "name": "installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private": { + "name": "private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "repositories_installation_id_installations_id_fk": { + "name": "repositories_installation_id_installations_id_fk", + "tableFrom": "repositories", + "tableTo": "installations", + "columnsFrom": [ + "installation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "repositories_github_repo_id_unique": { + "name": "repositories_github_repo_id_unique", + "nullsNotDistinct": false, + "columns": [ + "github_repo_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repository_enablement_events": { + "name": "repository_enablement_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "repository_enablement_events_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "repository_full_name": { + "name": "repository_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_private": { + "name": "repository_private", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'dashboard'" + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "repository_enablement_events_org_time_idx": { + "name": "repository_enablement_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "repository_enablement_events_repo_time_idx": { + "name": "repository_enablement_events_repo_time_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "repository_enablement_events_org_id_organizations_id_fk": { + "name": "repository_enablement_events_org_id_organizations_id_fk", + "tableFrom": "repository_enablement_events", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "repository_enablement_events_repository_id_repositories_id_fk": { + "name": "repository_enablement_events_repository_id_repositories_id_fk", + "tableFrom": "repository_enablement_events", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "repository_enablement_events_actor_user_id_users_id_fk": { + "name": "repository_enablement_events_actor_user_id_users_id_fk", + "tableFrom": "repository_enablement_events", + "tableTo": "users", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "repository_enablement_events_action_check": { + "name": "repository_enablement_events_action_check", + "value": "\"repository_enablement_events\".\"action\" IN ('enable', 'disable')" + }, + "repository_enablement_events_source_check": { + "name": "repository_enablement_events_source_check", + "value": "\"repository_enablement_events\".\"source\" IN ('dashboard', 'github_installation', 'github_pull_request', 'github_transfer', 'github_uninstall', 'migration_baseline')" + } + }, + "isRLSEnabled": false + }, + "public.repository_gate_enforcement": { + "name": "repository_gate_enforcement", + "schema": "", + "columns": { + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "branch_protection": { + "name": "branch_protection", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "evidence": { + "name": "evidence", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_successful_at": { + "name": "last_successful_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "repository_gate_enforcement_status_checked_idx": { + "name": "repository_gate_enforcement_status_checked_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "repository_gate_enforcement_repository_id_repositories_id_fk": { + "name": "repository_gate_enforcement_repository_id_repositories_id_fk", + "tableFrom": "repository_gate_enforcement", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "repository_gate_enforcement_status_check": { + "name": "repository_gate_enforcement_status_check", + "value": "\"repository_gate_enforcement\".\"status\" IN ('required', 'not_required', 'unknown')" + }, + "repository_gate_enforcement_branch_protection_check": { + "name": "repository_gate_enforcement_branch_protection_check", + "value": "\"repository_gate_enforcement\".\"branch_protection\" IN ('protected', 'unprotected', 'unknown')" + } + }, + "isRLSEnabled": false + }, + "public.respond_deliveries": { + "name": "respond_deliveries", + "schema": "", + "columns": { + "job_id": { + "name": "job_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "reservation_id": { + "name": "reservation_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "source_org_id": { + "name": "source_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_installation_id": { + "name": "source_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_installation_id": { + "name": "source_github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "repo_full_name": { + "name": "repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_pr": { + "name": "is_pr", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "source_head_sha": { + "name": "source_head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "marker_nonce": { + "name": "marker_nonce", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "reply_to_review_comment_id": { + "name": "reply_to_review_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "publication_identity_state": { + "name": "publication_identity_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'complete'" + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prepared'" + }, + "delivery_lease_expires_at": { + "name": "delivery_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "github_comment_id": { + "name": "github_comment_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "publication_lease_id": { + "name": "publication_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "publication_lease_expires_at": { + "name": "publication_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "cancelled_at": { + "name": "cancelled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "respond_deliveries_pending_idx": { + "name": "respond_deliveries_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivery_lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "respond_deliveries_job_id_jobs_id_fk": { + "name": "respond_deliveries_job_id_jobs_id_fk", + "tableFrom": "respond_deliveries", + "tableTo": "jobs", + "columnsFrom": [ + "job_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "respond_deliveries_repository_id_repositories_id_fk": { + "name": "respond_deliveries_repository_id_repositories_id_fk", + "tableFrom": "respond_deliveries", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "respond_deliveries_reservation_id_hosted_usage_reservations_id_fk": { + "name": "respond_deliveries_reservation_id_hosted_usage_reservations_id_fk", + "tableFrom": "respond_deliveries", + "tableTo": "hosted_usage_reservations", + "columnsFrom": [ + "reservation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "respond_deliveries_state_check": { + "name": "respond_deliveries_state_check", + "value": "\"respond_deliveries\".\"state\" IN ('prepared', 'delivering', 'delivered', 'cancelled')" + }, + "respond_deliveries_issue_number_positive": { + "name": "respond_deliveries_issue_number_positive", + "value": "\"respond_deliveries\".\"issue_number\" > 0" + }, + "respond_deliveries_body_nonempty": { + "name": "respond_deliveries_body_nonempty", + "value": "length(btrim(\"respond_deliveries\".\"body\")) > 0" + }, + "respond_deliveries_publication_identity_state_check": { + "name": "respond_deliveries_publication_identity_state_check", + "value": "\"respond_deliveries\".\"publication_identity_state\" IN ('complete', 'legacy_delivered', 'cancelled_incomplete')" + }, + "respond_deliveries_publication_identity_check": { + "name": "respond_deliveries_publication_identity_check", + "value": "(\n \"respond_deliveries\".\"source_org_id\" IS NOT NULL\n AND \"respond_deliveries\".\"source_installation_id\" IS NOT NULL\n AND \"respond_deliveries\".\"source_github_installation_id\" IS NOT NULL\n AND \"respond_deliveries\".\"source_github_repo_id\" IS NOT NULL\n AND (NOT \"respond_deliveries\".\"is_pr\" OR \"respond_deliveries\".\"source_head_sha\" IS NOT NULL)\n )" + }, + "respond_deliveries_publication_identity_state_matches_row_check": { + "name": "respond_deliveries_publication_identity_state_matches_row_check", + "value": "(\n \"respond_deliveries\".\"publication_identity_state\" = 'complete'\n OR (\"respond_deliveries\".\"publication_identity_state\" = 'legacy_delivered' AND \"respond_deliveries\".\"state\" = 'delivered')\n OR (\"respond_deliveries\".\"publication_identity_state\" = 'cancelled_incomplete' AND \"respond_deliveries\".\"state\" = 'cancelled')\n )" + } + }, + "isRLSEnabled": false + }, + "public.review_feedback_control": { + "name": "review_feedback_control", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "default": 1 + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'disabled'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "review_feedback_control_singleton": { + "name": "review_feedback_control_singleton", + "value": "\"review_feedback_control\".\"id\" = 1" + }, + "review_feedback_control_mode": { + "name": "review_feedback_control_mode", + "value": "\"review_feedback_control\".\"mode\" IN ('inherit', 'enabled', 'disabled')" + } + }, + "isRLSEnabled": false + }, + "public.review_feedback_polls": { + "name": "review_feedback_polls", + "schema": "", + "columns": { + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "next_poll_at": { + "name": "next_poll_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "review_feedback_polls_github_repo_id_pr_number_pk": { + "name": "review_feedback_polls_github_repo_id_pr_number_pk", + "columns": [ + "github_repo_id", + "pr_number" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_feedback_requests": { + "name": "review_feedback_requests", + "schema": "", + "columns": { + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "digest": { + "name": "digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_feedback_requests_org_created_idx": { + "name": "review_feedback_requests_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "review_feedback_requests_github_repo_id_pr_number_head_sha_digest_pk": { + "name": "review_feedback_requests_github_repo_id_pr_number_head_sha_digest_pk", + "columns": [ + "github_repo_id", + "pr_number", + "head_sha", + "digest" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_logs": { + "name": "review_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "review_logs_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "at": { + "name": "at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "line": { + "name": "line", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "review_logs_review_seq_idx": { + "name": "review_logs_review_seq_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "review_logs_review_id_reviews_id_fk": { + "name": "review_logs_review_id_reviews_id_fk", + "tableFrom": "review_logs", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_publication_receipts": { + "name": "review_publication_receipts", + "schema": "", + "columns": { + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "receipt_version": { + "name": "receipt_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "receipt_id": { + "name": "receipt_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "publication_channel": { + "name": "publication_channel", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_review_id": { + "name": "github_review_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "review_publication_receipts_review_id_reviews_id_fk": { + "name": "review_publication_receipts_review_id_reviews_id_fk", + "tableFrom": "review_publication_receipts", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "review_publication_receipts_identity_check": { + "name": "review_publication_receipts_identity_check", + "value": "(\"review_publication_receipts\".\"receipt_version\" IS NULL AND \"review_publication_receipts\".\"receipt_id\" IS NULL AND \"review_publication_receipts\".\"publication_channel\" IS NULL) OR (\"review_publication_receipts\".\"receipt_version\" = 1 AND length(btrim(\"review_publication_receipts\".\"receipt_id\")) BETWEEN 1 AND 200 AND (\"review_publication_receipts\".\"publication_channel\" IS NULL OR \"review_publication_receipts\".\"publication_channel\" = 'reviewComments')) OR (\"review_publication_receipts\".\"receipt_version\" = 2 AND length(btrim(\"review_publication_receipts\".\"receipt_id\")) BETWEEN 1 AND 200 AND \"review_publication_receipts\".\"publication_channel\" IS NOT NULL AND \"review_publication_receipts\".\"publication_channel\" IN ('reviewComments', 'checkAnnotations'))" + }, + "review_publication_receipts_github_review_id_check": { + "name": "review_publication_receipts_github_review_id_check", + "value": "\"review_publication_receipts\".\"github_review_id\" IS NULL OR \"review_publication_receipts\".\"github_review_id\" ~ '^[1-9][0-9]{0,19}$'" + } + }, + "isRLSEnabled": false + }, + "public.reviews": { + "name": "reviews", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "reviews_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "public_id": { + "name": "public_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "source_org_id": { + "name": "source_org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_installation_id": { + "name": "source_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_installation_id": { + "name": "source_github_installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_github_repo_id": { + "name": "source_github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "source_repo_full_name": { + "name": "source_repo_full_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "author_github_id": { + "name": "author_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_sha": { + "name": "base_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "since_sha": { + "name": "since_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trigger_source": { + "name": "trigger_source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "trigger_context": { + "name": "trigger_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "review_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "envelope": { + "name": "envelope", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "config_files": { + "name": "config_files", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "config_provenance": { + "name": "config_provenance", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "silent": { + "name": "silent", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "engine_gate_failing": { + "name": "engine_gate_failing", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "gate_failing": { + "name": "gate_failing", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "advisory_check_run_id": { + "name": "advisory_check_run_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "gate_check_run_id": { + "name": "gate_check_run_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "gate_sync_lease_id": { + "name": "gate_sync_lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "gate_sync_lease_expires_at": { + "name": "gate_sync_lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "publication_lifecycle_reconciled_at": { + "name": "publication_lifecycle_reconciled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "publication_lifecycle_required_at": { + "name": "publication_lifecycle_required_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "reviews_public_id_idx": { + "name": "reviews_public_id_idx", + "columns": [ + { + "expression": "public_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "reviews_repo_pr_idx": { + "name": "reviews_repo_pr_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pr_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "reviews_status_idx": { + "name": "reviews_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "reviews_running_started_at_idx": { + "name": "reviews_running_started_at_idx", + "columns": [ + { + "expression": "started_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"reviews\".\"status\" = 'running'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "reviews_publication_lifecycle_pending_idx": { + "name": "reviews_publication_lifecycle_pending_idx", + "columns": [ + { + "expression": "finished_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"reviews\".\"status\" = 'completed' AND \"reviews\".\"publication_lifecycle_required_at\" IS NOT NULL AND \"reviews\".\"publication_lifecycle_reconciled_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "reviews_repository_id_repositories_id_fk": { + "name": "reviews_repository_id_repositories_id_fk", + "tableFrom": "reviews", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "reviews_trigger_source_check": { + "name": "reviews_trigger_source_check", + "value": "\"reviews\".\"trigger_source\" IN ('unknown', 'automatic_pull_request', 'requested_review', 'github_check_rerun', 'finding_feedback')" + }, + "reviews_trigger_context_check": { + "name": "reviews_trigger_context_check", + "value": "(\"reviews\".\"trigger_source\" = 'finding_feedback' AND \"reviews\".\"trigger_context\" IS NOT NULL AND jsonb_typeof(\"reviews\".\"trigger_context\") = 'object' AND \"reviews\".\"trigger_context\" - ARRAY['source', 'feedbackDigest']::text[] = '{}'::jsonb AND COALESCE(\"reviews\".\"trigger_context\"->>'source', '') = 'finding_feedback' AND jsonb_typeof(\"reviews\".\"trigger_context\"->'feedbackDigest') = 'string' AND COALESCE(\"reviews\".\"trigger_context\"->>'feedbackDigest', '') ~ '^[a-f0-9]{64}$') OR (\"reviews\".\"trigger_source\" = 'unknown' AND (\"reviews\".\"trigger_context\" IS NULL OR \"reviews\".\"trigger_context\" = '{\"source\":\"unknown\"}'::jsonb)) OR (\"reviews\".\"trigger_source\" <> 'unknown' AND \"reviews\".\"trigger_context\" IS NOT NULL AND jsonb_typeof(\"reviews\".\"trigger_context\") = 'object' AND \"reviews\".\"trigger_context\" - ARRAY['source', 'webhookDeliveryId', 'webhookEvent', 'webhookAction', 'sourceCommentId', 'sourceUrl', 'requestedByGithubId', 'requestedByLogin', 'checkName']::text[] = '{}'::jsonb AND \"reviews\".\"trigger_context\"->>'source' = \"reviews\".\"trigger_source\" AND jsonb_typeof(\"reviews\".\"trigger_context\"->'webhookDeliveryId') = 'string' AND COALESCE(length(btrim(\"reviews\".\"trigger_context\"->>'webhookDeliveryId')), 0) > 0 AND length(\"reviews\".\"trigger_context\"->>'webhookDeliveryId') <= 200 AND ((\"reviews\".\"trigger_source\" = 'automatic_pull_request' AND \"reviews\".\"trigger_context\"->>'webhookEvent' = 'pull_request') OR (\"reviews\".\"trigger_source\" = 'requested_review' AND \"reviews\".\"trigger_context\"->>'webhookEvent' IN ('issue_comment', 'pull_request_review_comment')) OR (\"reviews\".\"trigger_source\" = 'github_check_rerun' AND \"reviews\".\"trigger_context\"->>'webhookEvent' IN ('check_run', 'check_suite'))) AND (NOT \"reviews\".\"trigger_context\" ? 'webhookAction' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'webhookAction') = 'string' AND length(\"reviews\".\"trigger_context\"->>'webhookAction') <= 100)) AND (NOT \"reviews\".\"trigger_context\" ? 'sourceCommentId' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'sourceCommentId') = 'number' AND (\"reviews\".\"trigger_context\"->>'sourceCommentId')::numeric = trunc((\"reviews\".\"trigger_context\"->>'sourceCommentId')::numeric) AND (\"reviews\".\"trigger_context\"->>'sourceCommentId')::numeric BETWEEN 1 AND 9007199254740991)) AND (NOT \"reviews\".\"trigger_context\" ? 'sourceUrl' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'sourceUrl') = 'string' AND length(\"reviews\".\"trigger_context\"->>'sourceUrl') <= 2048 AND \"reviews\".\"trigger_context\"->>'sourceUrl' ~* '^https://github[.]com([/?#]|$)')) AND (NOT \"reviews\".\"trigger_context\" ? 'requestedByGithubId' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'requestedByGithubId') = 'number' AND (\"reviews\".\"trigger_context\"->>'requestedByGithubId')::numeric = trunc((\"reviews\".\"trigger_context\"->>'requestedByGithubId')::numeric) AND (\"reviews\".\"trigger_context\"->>'requestedByGithubId')::numeric BETWEEN 1 AND 9007199254740991)) AND (NOT \"reviews\".\"trigger_context\" ? 'requestedByLogin' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'requestedByLogin') = 'string' AND length(\"reviews\".\"trigger_context\"->>'requestedByLogin') <= 100)) AND (NOT \"reviews\".\"trigger_context\" ? 'checkName' OR (jsonb_typeof(\"reviews\".\"trigger_context\"->'checkName') = 'string' AND length(\"reviews\".\"trigger_context\"->>'checkName') <= 200)))" + } + }, + "isRLSEnabled": false + }, + "public.self_service_trial_grants": { + "name": "self_service_trial_grants", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "initiated_by_github_id": { + "name": "initiated_by_github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "requested_mode": { + "name": "requested_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "granted_mode": { + "name": "granted_mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "self_service_trial_grants_actor_created_idx": { + "name": "self_service_trial_grants_actor_created_idx", + "columns": [ + { + "expression": "initiated_by_github_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "self_service_trial_grants_requested_mode_check": { + "name": "self_service_trial_grants_requested_mode_check", + "value": "\"self_service_trial_grants\".\"requested_mode\" IN ('hosted', 'byok')" + }, + "self_service_trial_grants_granted_mode_check": { + "name": "self_service_trial_grants_granted_mode_check", + "value": "\"self_service_trial_grants\".\"granted_mode\" IN ('hosted', 'byok')" + } + }, + "isRLSEnabled": false + }, + "public.service_heartbeats": { + "name": "service_heartbeats", + "schema": "", + "columns": { + "component": { + "name": "component", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "service_heartbeats_component_check": { + "name": "service_heartbeats_component_check", + "value": "\"service_heartbeats\".\"component\" IN ('worker', 'monitor', 'monitor-heartbeat-delivery')" + }, + "service_heartbeats_instance_nonempty": { + "name": "service_heartbeats_instance_nonempty", + "value": "length(btrim(\"service_heartbeats\".\"instance_id\")) > 0" + } + }, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "github_access_token_ciphertext": { + "name": "github_access_token_ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "membership_checked_at": { + "name": "membership_checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "membership_check_available_at": { + "name": "membership_check_available_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_events": { + "name": "usage_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "usage_events_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "org_id": { + "name": "org_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "repository_id": { + "name": "repository_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "review_id": { + "name": "review_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "trigger_source": { + "name": "trigger_source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "prompt_tokens": { + "name": "prompt_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "completion_tokens": { + "name": "completion_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "model_used": { + "name": "model_used", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cost_micros": { + "name": "cost_micros", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cost_cents": { + "name": "cost_cents", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "billing_scope": { + "name": "billing_scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "usage_events_org_id_organizations_id_fk": { + "name": "usage_events_org_id_organizations_id_fk", + "tableFrom": "usage_events", + "tableTo": "organizations", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_events_repository_id_repositories_id_fk": { + "name": "usage_events_repository_id_repositories_id_fk", + "tableFrom": "usage_events", + "tableTo": "repositories", + "columnsFrom": [ + "repository_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "usage_events_review_id_reviews_id_fk": { + "name": "usage_events_review_id_reviews_id_fk", + "tableFrom": "usage_events", + "tableTo": "reviews", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "usage_events_cost_micros_nonnegative": { + "name": "usage_events_cost_micros_nonnegative", + "value": "\"usage_events\".\"cost_micros\" IS NULL OR \"usage_events\".\"cost_micros\" >= 0" + }, + "usage_events_cost_cents_nonnegative": { + "name": "usage_events_cost_cents_nonnegative", + "value": "\"usage_events\".\"cost_cents\" IS NULL OR \"usage_events\".\"cost_cents\" >= 0" + }, + "usage_events_billing_scope_check": { + "name": "usage_events_billing_scope_check", + "value": "\"usage_events\".\"billing_scope\" IN ('analytics', 'private_hosted')" + }, + "usage_events_trigger_source_check": { + "name": "usage_events_trigger_source_check", + "value": "\"usage_events\".\"trigger_source\" IN ('unknown', 'automatic_pull_request', 'requested_review', 'github_check_rerun', 'finding_feedback', 'github_mention')" + } + }, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "users_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "github_id": { + "name": "github_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "login": { + "name": "login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_github_id_unique": { + "name": "users_github_id_unique", + "nullsNotDistinct": false, + "columns": [ + "github_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "webhook_deliveries_completed_at_idx": { + "name": "webhook_deliveries_completed_at_idx", + "columns": [ + { + "expression": "completed_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"webhook_deliveries\".\"completed_at\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "webhook_deliveries_payload_completion_check": { + "name": "webhook_deliveries_payload_completion_check", + "value": "(\"webhook_deliveries\".\"payload\" IS NULL) = (\"webhook_deliveries\".\"completed_at\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + } + }, + "enums": { + "public.finding_approval_role": { + "name": "finding_approval_role", + "schema": "public", + "values": [ + "member", + "admin" + ] + }, + "public.finding_approval_source": { + "name": "finding_approval_source", + "schema": "public", + "values": [ + "github", + "dashboard" + ] + }, + "public.finding_approval_verb": { + "name": "finding_approval_verb", + "schema": "public", + "values": [ + "approve", + "dismiss" + ] + }, + "public.job_status": { + "name": "job_status", + "schema": "public", + "values": [ + "queued", + "running", + "done", + "failed" + ] + }, + "public.review_status": { + "name": "review_status", + "schema": "public", + "values": [ + "queued", + "running", + "completed", + "failed", + "stale" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 53d30b9f..ec59c99f 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -442,6 +442,13 @@ "when": 1790186892052, "tag": "0063_review_feedback_control", "breakpoints": true + }, + { + "idx": 63, + "version": "7", + "when": 1791572108139, + "tag": "0064_ilert_canary_alert_key", + "breakpoints": true } ] } diff --git a/scripts/ensure-operational-indexes.ts b/scripts/ensure-operational-indexes.ts index 60aab10c..ff1b9da6 100644 --- a/scripts/ensure-operational-indexes.ts +++ b/scripts/ensure-operational-indexes.ts @@ -12,6 +12,16 @@ interface OperationalIndex { name: string; createSql: string; definitionFragments: string[]; + requiredColumns?: { + relation: string; + columns: string[]; + }; + /** + * Rebuild a valid index whose definition differs. Only a non-unique index + * that enforces no data invariant may opt in; every other index with an + * unexpected definition fails closed and stays in place. + */ + repairDefinitionDrift?: boolean; } const OPERATIONAL_INDEXES: OperationalIndex[] = [ @@ -124,25 +134,89 @@ const OPERATIONAL_INDEXES: OperationalIndex[] = [ "github", ], }, + { + name: "ilert_alert_events_canary_observation_idx", + createSql: + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "ilert_alert_events_canary_observation_idx" ON "ilert_alert_events" USING btree ("alert_key", "event_type", "alert_source_id") WHERE "alert_key" IS NOT NULL', + definitionFragments: [ + "public.ilert_alert_events", + "using btree", + "(alert_key, event_type, alert_source_id)", + "where", + "alert_key is not null", + ], + requiredColumns: { + relation: "public.ilert_alert_events", + columns: ["alert_key", "event_type", "alert_source_id"], + }, + repairDefinitionDrift: true, + }, +]; + +/** + * Indexes the managed release command builds itself. The managed path does + * not run the full reconciliation, so it creates only indexes introduced + * together with a reviewed migration and leaves every other index untouched. + */ +export const MANAGED_RELEASE_OPERATIONAL_INDEXES: readonly string[] = [ + "ilert_alert_events_canary_observation_idx", ]; +const MANAGED_RELEASE_LOCK_WAIT_MS = 60_000; +const MANAGED_RELEASE_STATEMENT_TIMEOUT_MS = 120_000; + +export interface EnsureOperationalIndexesOptions { + /** + * Reconcile only MANAGED_RELEASE_OPERATIONAL_INDEXES under bounded lock and + * statement waits. The release step row describes the full reconciliation + * and is not written. + */ + managedRelease?: boolean; +} interface IndexState { indisvalid: boolean; indisready: boolean; + enforcesInvariant: boolean; definition: string; } -export async function ensureOperationalIndexes(pool: Pool): Promise { +export async function ensureOperationalIndexes( + pool: Pool, + options: EnsureOperationalIndexesOptions = {}, +): Promise { + const managedRelease = options.managedRelease === true; + const indexes = managedRelease + ? OPERATIONAL_INDEXES.filter((index) => + MANAGED_RELEASE_OPERATIONAL_INDEXES.includes(index.name) + ) + : OPERATIONAL_INDEXES; + if (managedRelease && indexes.length !== MANAGED_RELEASE_OPERATIONAL_INDEXES.length) { + throw new Error("managed release operational indexes are not all defined"); + } const client = await pool.connect(); let locked = false; + let statementTimeoutSet = false; try { - await acquireReleaseLock(client); + await acquireReleaseLock( + client, + managedRelease ? MANAGED_RELEASE_LOCK_WAIT_MS : RELEASE_LOCK_WAIT_MS, + ); locked = true; - await ensureReleaseStepsTable(client); + if (managedRelease) { + // A concurrent build waits for older transactions; the bound keeps a + // blocked build from holding the release command. A cancelled build + // leaves an invalid index that the next run drops and rebuilds. + await client.query( + `SET statement_timeout = ${MANAGED_RELEASE_STATEMENT_TIMEOUT_MS}`, + ); + statementTimeoutSet = true; + } + await assertOperationalIndexDependencies(client, indexes); + if (!managedRelease) await ensureReleaseStepsTable(client); const completed: string[] = []; - for (const index of OPERATIONAL_INDEXES) { + for (const index of indexes) { let state = await loadIndexState(client, index.name); - if (state && (!state.indisvalid || !state.indisready)) { + if (state && shouldRebuildIndex(index, state)) { await client.query( `DROP INDEX CONCURRENTLY IF EXISTS "public"."${index.name}"`, ); @@ -156,6 +230,7 @@ export async function ensureOperationalIndexes(pool: Pool): Promise { completed.push(index.name); } + if (managedRelease) return completed; await client.query( `INSERT INTO release_steps (name, completed_at, details) VALUES ($1, now(), $2::jsonb) @@ -166,6 +241,9 @@ export async function ensureOperationalIndexes(pool: Pool): Promise { ); return completed; } finally { + if (statementTimeoutSet) { + await client.query("RESET statement_timeout").catch(() => undefined); + } if (locked) { await client .query("SELECT pg_advisory_unlock($1, $2)", [ @@ -178,6 +256,34 @@ export async function ensureOperationalIndexes(pool: Pool): Promise { } } +async function assertOperationalIndexDependencies( + client: PoolClient, + indexes: readonly OperationalIndex[], +): Promise { + for (const index of indexes) { + const required = index.requiredColumns; + if (!required) continue; + const result = await client.query<{ attname: string }>( + `SELECT attribute.attname + FROM pg_attribute AS attribute + WHERE attribute.attrelid = to_regclass($1) + AND attribute.attnum > 0 + AND NOT attribute.attisdropped + AND attribute.attname = ANY($2::text[])`, + [required.relation, required.columns], + ); + const present = new Set(result.rows.map((row) => row.attname)); + const missing = required.columns.filter((column) => !present.has(column)); + if (missing.length > 0) { + throw new Error( + `operational index ${index.name} requires migrated columns ${required.relation}.${missing.join( + `, ${required.relation}.`, + )}`, + ); + } + } +} + async function ensureReleaseStepsTable(client: PoolClient): Promise { await client.query( `CREATE TABLE IF NOT EXISTS "release_steps" ( @@ -188,8 +294,11 @@ async function ensureReleaseStepsTable(client: PoolClient): Promise { ); } -async function acquireReleaseLock(client: PoolClient): Promise { - const deadline = Date.now() + RELEASE_LOCK_WAIT_MS; +async function acquireReleaseLock( + client: PoolClient, + waitMs: number, +): Promise { + const deadline = Date.now() + waitMs; while (true) { const result = await client.query<{ acquired: boolean }>( "SELECT pg_try_advisory_lock($1, $2) AS acquired", @@ -212,6 +321,13 @@ async function loadIndexState( const result = await client.query( `SELECT pg_index.indisvalid, pg_index.indisready, + (pg_index.indisunique + OR pg_index.indisprimary + OR pg_index.indisexclusion + OR EXISTS ( + SELECT 1 FROM pg_constraint + WHERE pg_constraint.conindid = pg_index.indexrelid + )) AS "enforcesInvariant", pg_get_indexdef(pg_index.indexrelid) AS definition FROM pg_index WHERE pg_index.indexrelid = to_regclass($1)`, @@ -227,7 +343,33 @@ function assertExpectedIndex( if (!state || !state.indisvalid || !state.indisready) { throw new Error(`operational index ${index.name} is not valid and ready`); } - const normalized = state.definition + if (!hasExpectedDefinition(index, state.definition)) { + throw new Error( + `operational index ${index.name} has an unexpected definition`, + ); + } +} + +/** + * An interrupted concurrent build leaves an invalid index that is always + * rebuilt. A valid index with an unexpected definition is rebuilt only for an + * opted-in index that enforces no invariant; otherwise it is left in place + * and the definition check fails the run. + */ +function shouldRebuildIndex( + index: OperationalIndex, + state: IndexState, +): boolean { + if (!state.indisvalid || !state.indisready) return true; + if (hasExpectedDefinition(index, state.definition)) return false; + return index.repairDefinitionDrift === true && !state.enforcesInvariant; +} + +function hasExpectedDefinition( + index: OperationalIndex, + definition: string, +): boolean { + const normalized = definition .toLowerCase() .replace(/["()]/gu, "") .replace(/\s+/gu, " "); @@ -236,17 +378,20 @@ function assertExpectedIndex( .toLowerCase() .replace(/["()]/gu, "") .replace(/\s+/gu, " "); - if (!normalized.includes(expected)) { - throw new Error( - `operational index ${index.name} has an unexpected definition`, - ); - } + if (!normalized.includes(expected)) return false; } + return true; } async function main(): Promise { try { - const indexes = await ensureOperationalIndexes(getPool()); + const args = process.argv.slice(2); + if (args.some((arg) => arg !== "--managed-release")) { + throw new Error("usage: ensure-operational-indexes.ts [--managed-release]"); + } + const indexes = await ensureOperationalIndexes(getPool(), { + managedRelease: args.includes("--managed-release"), + }); console.log(`operational indexes ready: ${indexes.join(", ")}`); } finally { await closeDb(); diff --git a/scripts/reconcile-ilert-alert-stream.ts b/scripts/reconcile-ilert-alert-stream.ts new file mode 100644 index 00000000..805b893c --- /dev/null +++ b/scripts/reconcile-ilert-alert-stream.ts @@ -0,0 +1,2700 @@ +#!/usr/bin/env bun + +import { appendFile } from "node:fs/promises"; +import { timingSafeEqual } from "node:crypto"; + +const API_BASE = "https://api.ilert.com/api"; +const ACTION_NAME = "Postil operator alert stream"; +const REQUEST_TIMEOUT_MS = 7_500; +const REQUEST_ATTEMPTS = 3; +const REQUEST_RETRY_MS = 500; +const MAX_RETRY_AFTER_MS = 10_000; +const RECONCILE_DEADLINE_MS = 180_000; +const CANARY_ATTEMPTS = 12; +const CANARY_DISCOVERY_ATTEMPTS = 30; +const CANARY_RETRY_MS = 2_000; +const CANARY_CLEANUP_RETRY_MS = 5_000; +const CANARY_DEADLINE_MS = 360_000; +const CANARY_CLEANUP_RESERVE_MS = 120_000; +export const CANARY_FINALIZER_BUDGETS = { + cancellationGraceMs: 300_000, + cleanupMs: 60_000, + discoveryMs: 90_000, + operationMs: 210_000, + shellMs: 240_000, + terminalValidationMs: 60_000, +} as const; +const CANARY_FINALIZER_DEADLINE_MS = CANARY_FINALIZER_BUDGETS.operationMs; +const CANARY_FINALIZER_DISCOVERY_DEADLINE_MS = CANARY_FINALIZER_BUDGETS.discoveryMs; +const CANARY_FINALIZER_DISCOVERY_RETRY_MS = 10_000; +const CANARY_FINALIZER_TERMINAL_INVENTORY_MS = 30_000; +const CANARY_FINALIZER_CLEANUP_RESERVE_MS = CANARY_FINALIZER_BUDGETS.cleanupMs; +const CANARY_FINALIZER_TERMINAL_INVENTORY_MAX_PAGES = 20; +const CANARY_ORPHAN_SWEEP_DEADLINE_MS = 12 * 60 * 1_000; +const CANARY_ORPHAN_SWEEP_MAX_PAGES = 20; +const ALERT_ACTION_INVENTORY_MAX_PAGES = 50; +const CANARY_CLEANUP_ATTEMPTS = 4; +// GitHub permits reruns for 30 days. Keep two additional days for queueing, +// setup, smoke checks, and clock skew before cleanup establishes its window. +export const CANARY_RERUN_LOOKBACK_MS = 32 * 24 * 60 * 60 * 1_000; +const ALERT_PAGE_SIZE = 100; +// GitHub permits the initial workflow run plus 50 reruns. +export const MAX_CANARY_RUN_ATTEMPT = 51; +const FINALIZER_STABLE_EMPTY_SCANS = 2; +const CANARY_KEY_PREFIX = "postil-ilert-webhook-canary"; +const COMPATIBILITY_CANARY_KEY = "postil-production-monitor-test"; + +export const ALERT_TRIGGER_TYPES = [ + "alert-created", + "alert-acknowledged", + "alert-comment-added", + "alert-resolved", +] as const; + +type Json = Record; +export type Fetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; +type Sleep = (milliseconds: number) => Promise; +type Clock = () => number; +type Operation = "create" | "update" | "unchanged"; +type CanaryEventType = "alert-created" | "alert-resolved"; +type CanaryHandoff = "true" | "unknown" | "cleaned"; +type BeforeMutationAttempt = () => Promise; + +interface Deadline { + expiresAt: number; + now: Clock; +} + +interface ReconcileOptions { + apiKey: string; + integrationKey: string; + receiverOrigin: string; + runId?: string; + runAttempt?: string; + sourceId: number; + webhookSecret: string; + dryRun?: boolean; + fetchFn?: Fetch; + sleep?: Sleep; + now?: Clock; +} + +interface CanaryOptions { + alertSubmitted?: CanaryHandoff; + apiKey: string; + integrationKey: string; + onAlertAttempted?: () => void | Promise; + onAlertSubmitted?: () => void | Promise; + receiverOrigin: string; + runId: string; + runAttempt: string; + sourceId: number; + webhookSecret: string; + fetchFn?: Fetch; + sleep?: Sleep; + now?: Clock; +} + +interface FinalizerOptions { + alertSubmitted?: CanaryHandoff; + apiKey: string; + integrationKey: string; + runId: string; + runAttempt: string; + sweepAttempt?: string; + sourceId: number; + fetchFn?: Fetch; + sleep?: Sleep; + now?: Clock; +} + +interface OrphanSweepOptions { + apiKey: string; + sourceId: number; + fetchFn?: Fetch; + sleep?: Sleep; + now?: Clock; +} + +export interface ReconcileResult { + actionId: string | null; + operation: Operation; +} + +export function canaryAlertKey(runId: string, runAttempt: string): string { + const identity = canaryRunIdentity(runId, runAttempt); + return `${CANARY_KEY_PREFIX}-${identity.runId}-${identity.runAttempt}`; +} + +export function parseReceiverOrigin(value: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new Error("POSTIL_ILERT_RECEIVER_ORIGIN must be an HTTPS origin"); + } + if ( + parsed.protocol !== "https:" || + parsed.username !== "" || + parsed.password !== "" || + (parsed.pathname !== "" && parsed.pathname !== "/") || + parsed.search !== "" || + parsed.hash !== "" || + !parsed.hostname || + value.length > 2_048 + ) { + throw new Error( + "POSTIL_ILERT_RECEIVER_ORIGIN must be an HTTPS origin without credentials, path, query, or fragment", + ); + } + return parsed.origin; +} + +export function desiredAlertAction( + source: Json, + secret: string, + receiverOrigin: string, +): Json { + return { + alertSources: [source], + connectorType: "webhook", + name: ACTION_NAME, + triggerMode: "AUTOMATIC", + triggerTypes: [...ALERT_TRIGGER_TYPES], + alertFilter: null, + conditions: "", + params: { + bodyTemplate: "", + webhookUrl: webhookUrl(secret, receiverOrigin), + }, + }; +} + +export function equivalentAlertAction(actual: Json, desired: Json): boolean { + const left = object(actual.params); + const right = object(desired.params); + const actualRelationIds = relationIds(actual.alertSources); + const desiredRelationIds = relationIds(desired.alertSources); + return ( + actual.name === desired.name && + actual.connectorType === desired.connectorType && + actual.triggerMode === desired.triggerMode && + sameSet(strings(actual.triggerTypes), strings(desired.triggerTypes)) && + actualRelationIds !== null && + desiredRelationIds !== null && + sameSet(actualRelationIds, desiredRelationIds) && + emptyAlertFilter(actual.alertFilter) && + emptyText(actual.conditions) && + emptyText(desired.conditions) && + left?.webhookUrl === right?.webhookUrl && + emptyText(left?.bodyTemplate) && + emptyText(right?.bodyTemplate) && + hasNoHeaders(left?.headers) && + hasNoHeaders(right?.headers) + ); +} + +function sameJson(left: unknown, right: unknown): boolean { + return canonicalJson(left) === canonicalJson(right); +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + const entries = Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)); + return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`).join(",")}}`; + } + return JSON.stringify(value); +} + +// The detail response includes read-only and expanded relations. Only these +// independently writable action fields belong in an existing-action PUT. +// A nonzero delay also requires its corresponding trigger in the PUT body. +function carriedAlertActionFields(action: Json, triggerTypes?: readonly string[]): Json { + const carried: Json = {}; + if (Array.isArray(action.teams)) { + const ids = relationIds(action.teams); + if (ids === null) throw new Error("iLert returned invalid alert-action teams"); + carried.teams = ids.sort((left, right) => left - right).map((id) => ({ id })); + } else if (action.teams !== undefined && action.teams !== null) { + throw new Error("iLert returned invalid alert-action teams"); + } + for (const [key, maximum, triggerType] of [ + ["escalationEndedDelaySec", 7_200, "alert-escalation-ended"], + ["notResolvedDelaySec", 172_800, "v-alert-not-resolved"], + ] as const) { + const value = action[key]; + if (typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= maximum) { + if (value > 0 && triggerTypes && !triggerTypes.includes(triggerType)) { + throw new Error(`iLert alert-action ${key} requires its trigger type`); + } + carried[key] = value; + } else if (value !== undefined && value !== null) { + throw new Error(`iLert returned an invalid alert-action ${key}`); + } + } + if (typeof action.connectorId === "string") { + carried.connectorId = action.connectorId; + } else if (action.connectorId !== undefined && action.connectorId !== null) { + throw new Error("iLert returned an invalid alert-action connectorId"); + } + return carried; +} + +function normalizedCarriedAlertActionFields(fields: Json): Json { + return { + teams: fields.teams ?? [], + connectorId: fields.connectorId ?? "", + escalationEndedDelaySec: fields.escalationEndedDelaySec ?? 0, + notResolvedDelaySec: fields.notResolvedDelaySec ?? 0, + }; +} + +export async function reconcileIlertAlertAction( + options: ReconcileOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const sleep = options.sleep ?? Bun.sleep; + const now = options.now ?? Date.now; + const deadline = { expiresAt: now() + RECONCILE_DEADLINE_MS, now }; + const receiverOrigin = parseReceiverOrigin(options.receiverOrigin); + requireWebhookSecret(options.webhookSecret); + const source = await loadIntegrationBoundAlertSource( + fetchFn, + options.apiKey, + options.integrationKey, + options.sourceId, + deadline, + sleep, + ); + const desired = desiredAlertAction( + source, + options.webhookSecret, + receiverOrigin, + ); + const actions = await loadCandidateActions( + fetchFn, + options.apiKey, + desired, + deadline, + sleep, + ); + const existing = reservedCandidate(actions, desired, options.sourceId); + const operation: Operation = !existing + ? "create" + : equivalentAlertAction(existing, desired) + ? "unchanged" + : "update"; + if (options.dryRun) { + return { actionId: existing ? actionId(existing) : null, operation }; + } + const identity = requireCanaryRunIdentity(options.runId, options.runAttempt); + await precleanPriorAttemptKeys({ + apiKey: options.apiKey, + deadline, + fetchFn, + runAttempt: identity.runAttempt, + runId: identity.runId, + sleep, + sourceId: options.sourceId, + }); + + await preflightReceiver( + fetchFn, + receiverOrigin, + options.webhookSecret, + deadline, + sleep, + ); + // The cleanup and receiver preflight can take most of the reconciliation + // budget. Re-read both identities at the mutation boundary so a concurrent + // action or source change observed before dispatch rejects the stale plan. + const boundarySource = await loadIntegrationBoundAlertSource( + fetchFn, + options.apiKey, + options.integrationKey, + options.sourceId, + deadline, + sleep, + ); + const boundaryDesired = desiredAlertAction( + boundarySource, + options.webhookSecret, + receiverOrigin, + ); + if (!sameJson(desired, boundaryDesired)) { + throw new Error("iLert alert-action source binding changed before mutation"); + } + const boundaryActions = await loadCandidateActions( + fetchFn, + options.apiKey, + boundaryDesired, + deadline, + sleep, + ); + const boundaryExisting = reservedCandidate( + boundaryActions, + boundaryDesired, + options.sourceId, + ); + if (!existing && boundaryExisting) { + throw new Error("A Postil alert action appeared before creation; refusing to submit POST"); + } + if ( + existing && + (!boundaryExisting || + actionId(existing) !== actionId(boundaryExisting) || + !sameJson(existing, boundaryExisting)) + ) { + throw new Error("iLert reserved alert action changed before mutation"); + } + if (operation === "unchanged") { + return { actionId: actionId(existing), operation }; + } + const id = boundaryExisting ? actionId(boundaryExisting) : null; + const carried = boundaryExisting + ? carriedAlertActionFields(boundaryExisting, strings(boundaryDesired.triggerTypes)) + : {}; + // The guard runs immediately before every action mutation attempt, including + // a retry after a transient response. Receiver acceptance comes first; the + // source binding and complete continuity-safe action inventory then form the + // final reads before the request deadline is recomputed and the mutation is + // dispatched. The provider offers no conditional write, so an out-of-band + // change after these reads remains a detected postcondition risk. + const beforeMutationAttempt: BeforeMutationAttempt = async () => { + await preflightReceiver( + fetchFn, + receiverOrigin, + options.webhookSecret, + deadline, + sleep, + ); + const attemptSource = await loadIntegrationBoundAlertSource( + fetchFn, + options.apiKey, + options.integrationKey, + options.sourceId, + deadline, + sleep, + ); + const attemptDesired = desiredAlertAction( + attemptSource, + options.webhookSecret, + receiverOrigin, + ); + if (!sameJson(boundaryDesired, attemptDesired)) { + throw new Error("iLert alert-action source binding changed before mutation"); + } + const attemptActions = await loadCandidateActions( + fetchFn, + options.apiKey, + attemptDesired, + deadline, + sleep, + ); + const attemptExisting = reservedCandidate( + attemptActions, + attemptDesired, + options.sourceId, + ); + if (!id && attemptExisting) { + throw new Error("A Postil alert action appeared before creation; refusing to submit POST"); + } + if ( + id && + (!attemptExisting || + actionId(attemptExisting) !== id || + !sameJson(attemptExisting, boundaryExisting)) + ) { + throw new Error("iLert reserved alert action changed before mutation"); + } + }; + const result = id + ? requireObject( + await management( + fetchFn, + options.apiKey, + `/alert-actions/${encodeURIComponent(id)}?include=conditions`, + { + method: "PUT", + body: JSON.stringify({ ...carried, ...boundaryDesired, id }), + }, + deadline, + sleep, + false, + beforeMutationAttempt, + ), + "iLert returned an invalid alert action", + ) + : await createAlertActionSafely({ + apiKey: options.apiKey, + deadline, + desired: boundaryDesired, + fetchFn, + beforeMutationAttempt, + sleep, + sourceId: options.sourceId, + }); + const resultId = actionId(result); + if (id && resultId !== id) { + throw new Error("iLert returned a different alert action after update"); + } + await confirmUniqueMutatedAlertAction({ + apiKey: options.apiKey, + beforeMutationAttempt, + deadline, + desired: boundaryDesired, + expectedId: resultId, + carried: id ? carried : undefined, + fetchFn, + sleep, + sourceId: options.sourceId, + }); + return { actionId: resultId, operation }; +} + +export async function verifyIlertWebhookCanary( + options: CanaryOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const sleep = options.sleep ?? Bun.sleep; + const now = options.now ?? Date.now; + const identity = canaryRunIdentity(options.runId, options.runAttempt); + const startedAtMs = now(); + const canaryDeadline: Deadline = { + expiresAt: startedAtMs + CANARY_DEADLINE_MS, + now, + }; + const primaryDeadline: Deadline = { + expiresAt: canaryDeadline.expiresAt - CANARY_CLEANUP_RESERVE_MS, + now, + }; + const receiverOrigin = parseReceiverOrigin(options.receiverOrigin); + requireWebhookSecret(options.webhookSecret); + await loadIntegrationBoundAlertSource( + fetchFn, + options.apiKey, + options.integrationKey, + options.sourceId, + primaryDeadline, + sleep, + ); + await precleanPriorAttemptKeys({ + apiKey: options.apiKey, + deadline: primaryDeadline, + fetchFn, + runAttempt: identity.runAttempt, + runId: identity.runId, + sleep, + sourceId: options.sourceId, + }); + const key = canaryAlertKey(identity.runId, String(identity.runAttempt)); + let alertAttempted = false; + let resolutionConfirmed = false; + let created: CanaryTarget | undefined; + let primaryError: unknown; + try { + assertCleanupReserve(canaryDeadline); + const submittedOptions: WaitOptions = { + apiKey: options.apiKey, + deadline: primaryDeadline, + fetchFn, + integrationKey: options.integrationKey, + key, + receiverOrigin, + sleep, + sourceId: options.sourceId, + webhookSecret: options.webhookSecret, + }; + alertAttempted = true; + await options.onAlertAttempted?.(); + await event( + submittedOptions, + "ALERT", + key, + ); + await options.onAlertSubmitted?.(); + created = await waitForCreatedDelivery(submittedOptions, (target) => { + // The management inventory has authenticated this exact id, key, and + // source. Keep it before receiver evidence so cleanup remains able to + // resolve it if the receiver or a subsequent discovery request fails. + created ??= target; + }); + await resolveAndStabilize(submittedOptions, created, canaryDeadline); + resolutionConfirmed = true; + } catch (error) { + primaryError = error; + created ??= invalidCanaryAlertTarget(error); + } + + if (alertAttempted && !resolutionConfirmed) { + try { + await cleanupAfterAlertAttempt( + { + ...options, + deadline: canaryDeadline, + fetchFn, + key, + receiverOrigin, + sleep, + sourceId: options.sourceId, + webhookSecret: options.webhookSecret, + }, + created, + ); + } catch (cleanupError) { + if (primaryError instanceof InvalidCanaryAlertValidationError) throw primaryError; + if (cleanupError instanceof InvalidCanaryAlertValidationError) throw cleanupError; + throw new AggregateError( + primaryError ? [primaryError, cleanupError] : [cleanupError], + "iLert canary failed and cleanup could not be verified", + ); + } + } + if (primaryError) throw primaryError; +} + +export async function finalizeIlertWebhookCanary( + options: FinalizerOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const sleep = options.sleep ?? Bun.sleep; + const now = options.now ?? Date.now; + const startedAt = now(); + const identity = canaryRunIdentity(options.runId, options.runAttempt); + const sweepAttempt = canaryRunIdentity(options.runId, options.sweepAttempt ?? options.runAttempt) + .runAttempt; + const deadline: Deadline = { + expiresAt: startedAt + CANARY_FINALIZER_DEADLINE_MS, + now, + }; + const discoveryDeadline: Deadline = { + expiresAt: startedAt + CANARY_FINALIZER_DISCOVERY_DEADLINE_MS, + now, + }; + const cleanup: CleanupOptions = { + apiKey: options.apiKey, + deadline, + fetchFn, + integrationKey: options.integrationKey, + sleep, + sourceId: options.sourceId, + }; + const deterministicKeys = deterministicCanaryKeys( + identity.runId, + sweepAttempt, + options.sourceId, + ); + const keys: CanaryCleanupKey[] = [ + ...deterministicKeys, + { key: COMPATIBILITY_CANARY_KEY, sourceId: options.sourceId }, + ]; + const handoff = options.alertSubmitted ?? "unknown"; + const currentMain = deterministicKeys.find((key) => key.runAttempt === identity.runAttempt); + if (!currentMain) { + throw new Error("iLert canary cleanup could not reconstruct the producer attempt"); + } + await loadManagementBoundAlertSource( + fetchFn, + options.apiKey, + options.sourceId, + deadline, + sleep, + ); + await finalizeManagedCanaryKeys( + cleanup, + keys, + discoveryDeadline, + handoff, + currentMain, + ); +} + +export async function sweepIlertWebhookCanaryOrphans( + options: OrphanSweepOptions, +): Promise { + const fetchFn = options.fetchFn ?? fetch; + const sleep = options.sleep ?? Bun.sleep; + const now = options.now ?? Date.now; + const deadline: Deadline = { + expiresAt: now() + CANARY_ORPHAN_SWEEP_DEADLINE_MS, + now, + }; + const cleanup: ManagementCleanupOptions = { + apiKey: options.apiKey, + deadline, + fetchFn, + sleep, + sourceId: options.sourceId, + }; + const targets = new Map(); + + await loadManagementBoundAlertSource( + fetchFn, + options.apiKey, + options.sourceId, + deadline, + sleep, + ); + + for (let pass = 0; pass < CANARY_CLEANUP_ATTEMPTS; pass += 1) { + const discovered = await findOpenManagedCanaryAlerts({ + ...cleanup, + maxPages: CANARY_ORPHAN_SWEEP_MAX_PAGES, + }); + rememberTargets(targets, discovered.targets); + + const resolved = await resolveKnownAlerts(cleanup, targets); + const observed = await observeKnownAlerts(cleanup, targets); + if (discovered.errors.length > 0 || resolved.errors.length > 0 || !observed.allResolved) { + throw new AggregateError( + [...discovered.errors, ...resolved.errors, ...observed.errors], + discovered.errors.length > 0 + ? "iLert orphan canary sweep inventory was incomplete after resolving retained alerts" + : resolved.errors.length > 0 + ? "iLert orphan canary sweep could not resolve every discovered alert" + : "iLert orphan canary sweep could not verify every discovered alert as RESOLVED", + ); + } + + const remainingOpen = await findOpenManagedCanaryAlerts({ + ...cleanup, + maxPages: CANARY_ORPHAN_SWEEP_MAX_PAGES, + }); + rememberTargets(targets, remainingOpen.targets); + if (remainingOpen.errors.length > 0) { + const resolved = await resolveKnownAlerts(cleanup, targets); + const observed = await observeKnownAlerts(cleanup, targets); + throw new AggregateError( + [...remainingOpen.errors, ...resolved.errors, ...observed.errors], + "iLert orphan canary sweep inventory was incomplete after resolving retained alerts", + ); + } + await loadManagementBoundAlertSource( + fetchFn, + options.apiKey, + options.sourceId, + deadline, + sleep, + ); + if (remainingOpen.targets.length === 0) return; + if (pass + 1 < CANARY_CLEANUP_ATTEMPTS) { + await sleep(Math.min(CANARY_CLEANUP_RETRY_MS, remaining(deadline))); + } + } + + throw new Error("iLert orphan canary sweep did not stabilize every deterministic alert"); +} + +interface ManagementCleanupOptions { + apiKey: string; + deadline: Deadline; + fetchFn: Fetch; + sleep: Sleep; + sourceId: number; +} + +interface CleanupOptions extends ManagementCleanupOptions { + integrationKey: string; +} + +interface WaitOptions extends CleanupOptions { + key: string; + receiverOrigin: string; + sourceId: number; + webhookSecret: string; +} + +interface CanaryTarget { + alertId: string; + alertKey: string; + sourceId: number; + status?: "PENDING" | "ACCEPTED" | "RESOLVED"; +} + +interface CanaryObservation extends CanaryTarget { + priority: "HIGH" | "LOW"; + status: "PENDING" | "ACCEPTED" | "RESOLVED"; +} + +class InvalidCanaryAlertValidationError extends Error { + constructor(message: string, readonly target?: CanaryTarget) { + super(message); + } +} + +class InvalidCanaryAlertStatusError extends InvalidCanaryAlertValidationError { + constructor(readonly target: CanaryTarget) { + super("iLert returned a canary alert with an invalid status", target); + } +} + +class InvalidCanaryAlertPriorityError extends InvalidCanaryAlertValidationError { + constructor(readonly target: CanaryTarget) { + super("iLert returned a canary alert with an invalid priority", target); + } +} + +class InvalidCanaryAlertIdentityError extends InvalidCanaryAlertValidationError {} + +interface CanaryCleanupKey { + key: string; + sourceId: number; +} + +interface DeterministicCanaryKey extends CanaryCleanupKey { + runAttempt: number; +} + +interface ReportTimeWindow { + from: string; + until: string; +} + +const ALL_ALERT_STATES = ["PENDING", "ACCEPTED", "RESOLVED"] as const; + +async function cleanupAfterAlertAttempt( + options: WaitOptions, + created: CanaryTarget | undefined, +): Promise { + let submissionError: unknown; + try { + if (!created) { + await event( + options, + "RESOLVE", + options.key, + ); + } + } catch (error) { + submissionError = error; + } + + let target = created; + if (!target) { + target = await waitForSubmittedAlert( + options, + ["PENDING", "ACCEPTED", "RESOLVED"], + ); + } + if (target) { + await driveAlertToResolved(options, target, options.deadline); + } else { + throw new Error("iLert could not identify the attempted canary alert for cleanup", { + cause: submissionError, + }); + } +} + +async function resolveAndStabilize( + options: WaitOptions, + target: CanaryTarget, + deadline: Deadline, +): Promise { + let lastError: unknown; + const cleanupOptions = { ...options, deadline }; + for (let attempt = 0; attempt < CANARY_CLEANUP_ATTEMPTS; attempt += 1) { + assertBeforeDeadline(deadline); + try { + await resolveAlertById(cleanupOptions, target.alertId); + const observed = await observeAlertById(cleanupOptions, target); + if ( + observed.status === "RESOLVED" && + await receiverEventReceived( + cleanupOptions, + "alert-resolved", + ) + ) { + return; + } + } catch (error) { + if (error instanceof InvalidCanaryAlertValidationError) throw error; + lastError = error; + } + if (attempt + 1 < CANARY_CLEANUP_ATTEMPTS) { + await options.sleep( + Math.min(CANARY_CLEANUP_RETRY_MS, remaining(deadline)), + ); + } + } + throw new Error("iLert did not verify canary resolution during stabilization", { + cause: lastError, + }); +} + +async function waitForCreatedDelivery( + options: WaitOptions, + onValidatedTarget?: (target: CanaryTarget) => void, +): Promise { + for (let attempt = 0; attempt < CANARY_ATTEMPTS; attempt += 1) { + assertBeforeDeadline(options.deadline); + const observed = await findSubmittedAlert( + options, + ["PENDING", "ACCEPTED"], + ); + if (observed) onValidatedTarget?.(observed); + if ( + (observed?.status === "PENDING" || observed?.status === "ACCEPTED") && + observed.priority === "HIGH" && + await receiverEventReceived(options, "alert-created") + ) { + return observed; + } + if (attempt + 1 < CANARY_ATTEMPTS) { + await options.sleep(Math.min(CANARY_RETRY_MS, remaining(options.deadline))); + } + } + throw new Error("iLert did not confirm persisted Postil webhook delivery"); +} + +async function waitForSubmittedAlert( + options: WaitOptions, + states: readonly ("PENDING" | "ACCEPTED" | "RESOLVED")[], +): Promise { + for (let attempt = 0; attempt < CANARY_DISCOVERY_ATTEMPTS; attempt += 1) { + const observed = await findSubmittedAlert(options, states); + if (observed) return observed; + if (attempt + 1 < CANARY_DISCOVERY_ATTEMPTS) { + await options.sleep(Math.min(CANARY_RETRY_MS, remaining(options.deadline))); + } + } + return undefined; +} + +async function findSubmittedAlert( + options: WaitOptions, + states: readonly ("PENDING" | "ACCEPTED" | "RESOLVED")[], +): Promise { + const current = await findAlertsByKey({ + apiKey: options.apiKey, + deadline: options.deadline, + fetchFn: options.fetchFn, + ...rollingCanaryInventoryWindow(options.deadline.now()), + completeInventory: true, + key: options.key, + maxPages: CANARY_FINALIZER_TERMINAL_INVENTORY_MAX_PAGES, + sleep: options.sleep, + sourceId: options.sourceId, + states: ALL_ALERT_STATES, + }); + if (current.length > 1) { + throw new Error("iLert created multiple alerts for one canary attempt"); + } + if (!current[0]) return undefined; + const observed = canaryObservation(current[0], options.key, options.sourceId); + return states.includes(observed.status) ? observed : undefined; +} + +async function observeAlertById( + options: ManagementCleanupOptions, + target: CanaryTarget, +): Promise { + const alert = requireObject( + await management( + options.fetchFn, + options.apiKey, + `/alerts/${encodeURIComponent(target.alertId)}`, + {}, + options.deadline, + options.sleep, + ), + "iLert returned an invalid canary alert detail", + ); + if (canaryAlertId(alert) !== target.alertId) { + throw new Error("iLert returned a detail for a different canary alert"); + } + return canaryObservation(alert, target.alertKey, target.sourceId); +} + +function canaryObservation( + alert: Json, + expectedKey: string, + expectedSourceId?: number, +): CanaryObservation { + if (alert.alertKey !== expectedKey) { + throw new Error("iLert returned a canary alert with a different key"); + } + const alertId = positiveId(alert.id); + const observedSourceId = alertSourceId(alert); + if (!alertId) { + throw new InvalidCanaryAlertIdentityError( + "iLert returned a canary alert without an identity", + ); + } + if (!observedSourceId) { + throw new InvalidCanaryAlertIdentityError( + "iLert returned a canary alert without a valid source", + ); + } + if (expectedSourceId !== undefined && observedSourceId !== expectedSourceId) { + throw new InvalidCanaryAlertIdentityError( + "iLert returned a canary alert from a different source", + ); + } + const exactTarget: CanaryTarget = { + alertId, + alertKey: expectedKey, + sourceId: observedSourceId, + }; + const priority = alert.priority; + if (priority !== "HIGH" && priority !== "LOW") { + throw new InvalidCanaryAlertPriorityError(exactTarget); + } + const status = alert.status; + if (status !== "PENDING" && status !== "ACCEPTED" && status !== "RESOLVED") { + throw new InvalidCanaryAlertStatusError(exactTarget); + } + return { ...exactTarget, priority, status }; +} + +function invalidCanaryAlertTarget(error: unknown): CanaryTarget | undefined { + return invalidCanaryAlertValidationError(error)?.target; +} + +function invalidCanaryAlertValidationError( + error: unknown, + seen = new Set(), +): InvalidCanaryAlertValidationError | undefined { + if (error instanceof InvalidCanaryAlertValidationError) return error; + if (Array.isArray(error)) { + for (const nested of error) { + const validationError = invalidCanaryAlertValidationError(nested, seen); + if (validationError) return validationError; + } + return undefined; + } + if (!error || typeof error !== "object" || seen.has(error)) return undefined; + seen.add(error); + if (error instanceof AggregateError) { + for (const nested of error.errors) { + const validationError = invalidCanaryAlertValidationError(nested, seen); + if (validationError) return validationError; + } + } + if (error instanceof Error) { + return invalidCanaryAlertValidationError(error.cause, seen); + } + return undefined; +} + +function canaryAlertId(alert: Json): string { + const id = positiveId(alert.id); + if (!id) throw new Error("iLert returned a canary alert without an identity"); + return id; +} + +async function driveAlertToResolved( + options: ManagementCleanupOptions, + target: CanaryTarget, + deadline: Deadline, +): Promise { + let lastError: unknown; + let invalidValidationError: InvalidCanaryAlertValidationError | undefined; + for (let attempt = 0; attempt < CANARY_CLEANUP_ATTEMPTS; attempt += 1) { + try { + await resolveAlertById(options, target.alertId); + const observed = await observeAlertById(options, target); + if (observed.status === "RESOLVED") { + if (invalidValidationError) throw invalidValidationError; + return; + } + } catch (error) { + if (error instanceof InvalidCanaryAlertValidationError) { + invalidValidationError ??= error; + } + lastError = error; + } + if (attempt + 1 < CANARY_CLEANUP_ATTEMPTS) { + await options.sleep(Math.min(CANARY_CLEANUP_RETRY_MS, remaining(deadline))); + } + } + if (invalidValidationError) throw invalidValidationError; + throw new Error("iLert did not resolve the canary alert", { + cause: lastError, + }); +} + +async function precleanPriorAttemptKeys( + options: ManagementCleanupOptions & { + runAttempt: number; + runId: string; + sourceId: number; + }, +): Promise { + if (options.runAttempt === 1) return; + const keys = deterministicCanaryKeys( + options.runId, + options.runAttempt - 1, + options.sourceId, + ); + const targets = new Map(); + let stableEmptyScans = 0; + let lastErrors: unknown[] = []; + let invalidValidationError: InvalidCanaryAlertValidationError | undefined; + + for (let pass = 0; pass < CANARY_CLEANUP_ATTEMPTS; pass += 1) { + const discovered = await discoverOpenManagedAlerts( + options, + keys, + rollingCanaryInventoryWindow(options.deadline.now()), + CANARY_FINALIZER_TERMINAL_INVENTORY_MAX_PAGES, + true, + ); + rememberTargets(targets, discovered.alerts); + lastErrors = discovered.errors; + invalidValidationError ??= discovered.validationError; + stableEmptyScans = discovered.complete && discovered.open === 0 + ? stableEmptyScans + 1 + : 0; + await resolveKnownAlerts(options, targets); + const observed = await observeKnownAlerts(options, targets); + lastErrors = [...lastErrors, ...observed.errors]; + invalidValidationError ??= invalidCanaryAlertValidationError(observed.errors); + if (!invalidValidationError && stableEmptyScans >= FINALIZER_STABLE_EMPTY_SCANS && observed.allResolved) { + return; + } + if (pass + 1 < CANARY_CLEANUP_ATTEMPTS) { + await options.sleep(Math.min(CANARY_CLEANUP_RETRY_MS, remaining(options.deadline))); + } + } + if (invalidValidationError) throw invalidValidationError; + throw new AggregateError( + lastErrors, + "iLert did not stabilize prior-attempt canary cleanup", + ); +} + +async function finalizeManagedCanaryKeys( + options: CleanupOptions, + keys: readonly CanaryCleanupKey[], + discoveryDeadline: Deadline, + handoff: CanaryHandoff, + currentMain: DeterministicCanaryKey, +): Promise { + const targets = new Map(); + // The producer's persisted create-and-resolve proof accounts for the first + // alert, but a retry-ambiguous Event API submission can materialize later. + // Continue the full inventory sweep for that delayed duplicate. + let currentAccountedFor = handoff === "cleaned"; + let currentResolveSubmitted = false; + let fatalError: unknown; + let invalidValidationError: InvalidCanaryAlertValidationError | undefined; + let validationContextErrors: unknown[] = []; + let recentErrors: unknown[] = []; + const resolutionAttempts = new Map(); + const observationAttempts = new Map(); + + const retainErrors = (errors: readonly unknown[]) => { + if (errors.length > 0) recentErrors = [...recentErrors, ...errors].slice(-20); + }; + const retainInvalidValidation = (errors: readonly unknown[]) => { + const validationError = invalidCanaryAlertValidationError(errors); + if (!validationError) return; + invalidValidationError ??= validationError; + validationContextErrors = [...validationContextErrors, ...errors].slice(-20); + }; + const discover = async ( + deadline: Deadline, + maxPages?: number, + ): Promise<{ complete: boolean; open: number }> => { + const inventoryWindow = rollingCanaryInventoryWindow(deadline.now()); + const result = await discoverOpenManagedAlerts( + { ...options, deadline }, + keys, + inventoryWindow, + maxPages ?? CANARY_FINALIZER_TERMINAL_INVENTORY_MAX_PAGES, + true, + ); + rememberTargets(targets, result.alerts); + retainErrors(result.errors); + retainInvalidValidation(result.errors); + if (result.validationError && !invalidValidationError) { + invalidValidationError = result.validationError; + validationContextErrors = [...validationContextErrors, result.validationError].slice(-20); + } + if (handoff === "true" && !currentAccountedFor) { + try { + const current = await findAlertsByKey({ + apiKey: options.apiKey, + deadline, + fetchFn: options.fetchFn, + ...rollingCanaryInventoryWindow(deadline.now()), + key: currentMain.key, + completeInventory: true, + maxPages: maxPages ?? CANARY_FINALIZER_TERMINAL_INVENTORY_MAX_PAGES, + onTarget: (target) => targets.set(target.alertId, target), + sleep: options.sleep, + sourceId: currentMain.sourceId, + }); + if (current.length > 1) { + fatalError = new Error("iLert returned multiple current-attempt canary alerts"); + } else if (current[0]) { + const observed = canaryObservation( + current[0], + currentMain.key, + currentMain.sourceId, + ); + if (observed.priority !== "HIGH") { + fatalError = new Error("iLert persisted the current canary at non-HIGH priority"); + } + targets.set(observed.alertId, observed); + currentAccountedFor = true; + } + } catch (error) { + const target = invalidCanaryAlertTarget(error); + if (target) targets.set(target.alertId, target); + retainInvalidValidation([error]); + retainErrors([error]); + } + } + if ( + handoff !== "cleaned" && + !currentResolveSubmitted && + ![...targets.values()].some((target) => target.alertKey === currentMain.key) + ) { + try { + await event( + { ...options, deadline }, + "RESOLVE", + currentMain.key, + ); + currentResolveSubmitted = true; + } catch (error) { + retainErrors([error]); + } + } + return { complete: result.complete, open: result.open }; + }; + + while (remaining(discoveryDeadline) > 0) { + await discover(discoveryDeadline); + const discoveryOptions = { ...options, deadline: discoveryDeadline }; + const resolved = await resolveKnownAlerts(discoveryOptions, targets, resolutionAttempts); + retainErrors(resolved.errors); + const observed = await observeKnownAlerts(discoveryOptions, targets, observationAttempts); + retainErrors(observed.errors); + retainInvalidValidation(observed.errors); + if (handoff === "unknown" && currentTargetResolved(currentMain.key, targets, resolved, observed)) { + currentAccountedFor = true; + } + const delay = Math.min( + CANARY_FINALIZER_DISCOVERY_RETRY_MS, + remaining(discoveryDeadline), + remaining(options.deadline), + ); + if (delay > 0) await options.sleep(delay); + } + + let inventoryIncomplete = false; + let stableEmptyScans = 0; + const terminalPhaseDeadline: Deadline = { + expiresAt: options.deadline.expiresAt - CANARY_FINALIZER_CLEANUP_RESERVE_MS, + now: options.deadline.now, + }; + + while (remaining(terminalPhaseDeadline) > 0) { + const terminalDeadline: Deadline = { + expiresAt: Math.min( + terminalPhaseDeadline.expiresAt, + terminalPhaseDeadline.now() + CANARY_FINALIZER_TERMINAL_INVENTORY_MS, + ), + now: terminalPhaseDeadline.now, + }; + if (remaining(terminalDeadline) <= 0) break; + const terminalDiscovery = await discover( + terminalDeadline, + CANARY_FINALIZER_TERMINAL_INVENTORY_MAX_PAGES, + ); + // A later complete, empty scan proves recovery from a transient inventory + // failure. Any incomplete or non-empty scan resets that proof. + inventoryIncomplete = !terminalDiscovery.complete; + stableEmptyScans = terminalDiscovery.complete && terminalDiscovery.open === 0 + ? stableEmptyScans + 1 + : 0; + const terminalOptions = { ...options, deadline: terminalDeadline }; + const resolved = await resolveKnownAlerts(terminalOptions, targets, resolutionAttempts); + retainErrors(resolved.errors); + const observed = await observeKnownAlerts(terminalOptions, targets, observationAttempts); + retainErrors(observed.errors); + retainInvalidValidation(observed.errors); + if (handoff === "unknown" && currentTargetResolved(currentMain.key, targets, resolved, observed)) { + currentAccountedFor = true; + } + if ( + !fatalError && + !invalidValidationError && + !inventoryIncomplete && + currentAccountedFor && + observed.allResolved && + stableEmptyScans >= FINALIZER_STABLE_EMPTY_SCANS + ) { + await loadManagementBoundAlertSource( + options.fetchFn, + options.apiKey, + options.sourceId, + options.deadline, + options.sleep, + ); + return; + } + const delay = Math.min( + CANARY_CLEANUP_RETRY_MS, + remaining(terminalPhaseDeadline), + ); + if (delay > 0) await options.sleep(delay); + } + + while (remaining(options.deadline) > 0) { + const resolved = await resolveKnownAlerts(options, targets, resolutionAttempts); + retainErrors(resolved.errors); + const observed = await observeKnownAlerts(options, targets, observationAttempts); + retainErrors(observed.errors); + retainInvalidValidation(observed.errors); + if (handoff === "unknown" && currentTargetResolved(currentMain.key, targets, resolved, observed)) { + currentAccountedFor = true; + } + const delay = Math.min(CANARY_CLEANUP_RETRY_MS, remaining(options.deadline)); + if (delay > 0) await options.sleep(delay); + } + + if (invalidValidationError) { + const context = validationContextErrors.filter( + (error) => error !== invalidValidationError, + ); + if (context.length > 0) { + throw new AggregateError( + [invalidValidationError, ...context], + `iLert canary cleanup encountered ${invalidValidationError.message}`, + ); + } + throw invalidValidationError; + } + if (inventoryIncomplete) { + throw new AggregateError( + recentErrors, + "iLert canary cleanup inventory was incomplete", + ); + } + if (fatalError) { + throw new Error("iLert canary cleanup found an invalid current-attempt identity", { + cause: fatalError, + }); + } + if (!currentAccountedFor) { + throw new AggregateError( + recentErrors, + "iLert canary cleanup could not account for the accepted current-attempt submission", + ); + } + throw new AggregateError( + recentErrors, + "iLert canary cleanup did not verify every discovered alert as RESOLVED", + ); +} + +async function discoverOpenManagedAlerts( + options: ManagementCleanupOptions, + keys: readonly CanaryCleanupKey[], + window?: ReportTimeWindow, + maxPages?: number, + completeInventory = false, +): Promise<{ + alerts: CanaryTarget[]; + complete: boolean; + errors: unknown[]; + open: number; + validationError?: InvalidCanaryAlertValidationError; +}> { + const alerts: CanaryTarget[] = []; + const errors: unknown[] = []; + let open = 0; + let validationError: InvalidCanaryAlertValidationError | undefined; + const groups = [keys]; + for (const group of groups) { + if (group.length === 0) continue; + try { + const sourceId = group[0]!.sourceId; + const found = await findAlertsByKeys({ + apiKey: options.apiKey, + completeInventory, + deadline: options.deadline, + fetchFn: options.fetchFn, + keys: group.map((key) => key.key), + maxPages, + sleep: options.sleep, + sourceId, + states: completeInventory ? ALL_ALERT_STATES : ["PENDING", "ACCEPTED"], + ...(window ?? {}), + onTarget: (target) => alerts.push(target), + }); + open += found.filter((alert) => + alert.status === "PENDING" || alert.status === "ACCEPTED" + ).length; + } catch (error) { + const target = invalidCanaryAlertTarget(error); + if (target && !alerts.some((alert) => alert.alertId === target.alertId)) { + alerts.push(target); + } + validationError ??= invalidCanaryAlertValidationError(error); + errors.push(error); + } + } + return { alerts, complete: errors.length === 0, errors, open, validationError }; +} + +async function observeKnownAlerts( + options: ManagementCleanupOptions, + targets: Map, + attempts?: Map, +): Promise<{ allResolved: boolean; errors: unknown[]; resolvedIds: Set }> { + const errors: unknown[] = []; + const resolvedIds = new Set(); + let allResolved = true; + for (const target of fairlyOrderedTargets(targets, attempts)) { + if (remaining(options.deadline) <= 0) { + allResolved = false; + break; + } + attempts?.set(target.alertId, (attempts.get(target.alertId) ?? 0) + 1); + try { + const observed = await observeAlertById(options, target); + targets.set(target.alertId, observed); + if (observed.status === "RESOLVED") resolvedIds.add(target.alertId); + else allResolved = false; + } catch (error) { + errors.push(error); + allResolved = false; + } + } + return { allResolved, errors, resolvedIds }; +} + +async function resolveKnownAlerts( + options: ManagementCleanupOptions, + targets: ReadonlyMap, + attempts?: Map, +): Promise<{ errors: unknown[]; resolvedIds: Set }> { + const errors: unknown[] = []; + const resolvedIds = new Set(); + for (const target of fairlyOrderedTargets(targets, attempts)) { + if (target.status === "RESOLVED") { + resolvedIds.add(target.alertId); + continue; + } + if (remaining(options.deadline) <= 0) break; + attempts?.set(target.alertId, (attempts.get(target.alertId) ?? 0) + 1); + try { + await resolveAlertById(options, target.alertId); + resolvedIds.add(target.alertId); + } catch (error) { + errors.push(error); + } + } + return { errors, resolvedIds }; +} + +function fairlyOrderedTargets( + targets: ReadonlyMap, + attempts?: ReadonlyMap, +): CanaryTarget[] { + const retained = [...targets.values()]; + if (!attempts) return retained; + // Give later retained IDs a first attempt before retrying earlier failures. + return retained + .map((target, index) => ({ attempts: attempts.get(target.alertId) ?? 0, index, target })) + .sort((left, right) => left.attempts - right.attempts || right.index - left.index) + .map(({ target }) => target); +} + +function currentTargetResolved( + key: string, + targets: ReadonlyMap, + resolved: { resolvedIds: ReadonlySet }, + observed: { resolvedIds: ReadonlySet }, +): boolean { + return [...resolved.resolvedIds].some((alertId) => + targets.get(alertId)?.alertKey === key && observed.resolvedIds.has(alertId) + ); +} + +function rememberTargets( + targets: Map, + alerts: readonly CanaryTarget[], +): void { + for (const alert of alerts) targets.set(alert.alertId, alert); +} + +function deterministicCanaryKeys( + runId: string, + throughAttempt: number, + sourceId: number, +): DeterministicCanaryKey[] { + const keys: DeterministicCanaryKey[] = []; + for (let runAttempt = 1; runAttempt <= throughAttempt; runAttempt += 1) { + const attempt = String(runAttempt); + keys.push({ + key: canaryAlertKey(runId, attempt), + runAttempt, + sourceId, + }); + } + return keys; +} + +function rollingCanaryInventoryWindow(untilMs: number): ReportTimeWindow { + return { + from: reportTime(untilMs - CANARY_RERUN_LOOKBACK_MS), + until: reportTime(untilMs), + }; +} + +async function receiverEventReceived( + options: WaitOptions, + eventType: CanaryEventType, +): Promise { + const query = new URLSearchParams({ + alertKey: options.key, + eventType, + sourceId: String(options.sourceId), + }); + const response = await requestWithRetry( + options.fetchFn, + `${options.receiverOrigin}/api/webhooks/ilert?${query.toString()}`, + { + method: "GET", + headers: { + accept: "application/json", + authorization: basicAuthorization(options.webhookSecret), + }, + }, + options.deadline, + options.sleep, + ); + if (!response.ok) { + await releaseResponseBody(response); + throw new Error(`Postil webhook observation failed with HTTP ${response.status}`); + } + const value = object(await parseJson(response)); + if (!value || typeof value.received !== "boolean") { + throw new Error("Postil returned an invalid webhook observation"); + } + return value.received; +} + +async function preflightReceiver( + fetchFn: Fetch, + receiverOrigin: string, + webhookSecret: string, + deadline: Deadline, + sleep: Sleep, +): Promise { + const response = await requestWithRetry( + fetchFn, + `${receiverOrigin}/api/webhooks/ilert`, + { + method: "GET", + headers: { authorization: basicAuthorization(webhookSecret) }, + }, + deadline, + sleep, + ); + await releaseResponseBody(response); + if (response.status !== 204) { + throw new Error(`Postil webhook credential preflight failed with HTTP ${response.status}`); + } +} + +async function event( + options: CleanupOptions, + eventType: "ALERT" | "RESOLVE", + alertKey: string, + priority?: "HIGH" | "LOW", + summary = "Postil iLert webhook canary", +): Promise { + const response = await requestWithRetry( + options.fetchFn, + `${API_BASE}/events`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + integrationKey: options.integrationKey, + eventType, + summary: eventType === "ALERT" ? summary : `${summary} resolved`, + ...(eventType === "ALERT" + ? { + details: + "GitHub Actions is verifying the Postil iLert webhook receiver.", + priority: priority ?? "HIGH", + } + : {}), + alertKey, + }), + }, + options.deadline, + options.sleep, + false, + async () => { + await loadIntegrationBoundAlertSource( + options.fetchFn, + options.apiKey, + options.integrationKey, + options.sourceId, + options.deadline, + options.sleep, + ); + }, + ); + await releaseResponseBody(response); + if (!response.ok) { + throw new Error(`iLert event request failed with HTTP ${response.status}`); + } +} + +async function management( + fetchFn: Fetch, + apiKey: string, + path: string, + init: RequestInit, + deadline: Deadline, + sleep: Sleep, + sensitive = false, + beforeAttempt?: BeforeMutationAttempt, +): Promise { + const response = await requestWithRetry( + fetchFn, + `${API_BASE}${path}`, + { + ...init, + headers: { + accept: "application/json", + authorization: `Bearer ${apiKey}`, + ...(init.body ? { "content-type": "application/json" } : {}), + }, + }, + deadline, + sleep, + sensitive, + beforeAttempt, + ); + if (!response.ok) { + await releaseResponseBody(response); + throw new Error(`iLert management request failed with HTTP ${response.status}`); + } + return parseJson(response); +} + +async function resolveAlertById( + options: ManagementCleanupOptions, + alertId: string, +): Promise { + const response = await requestWithRetry( + options.fetchFn, + `${API_BASE}/alerts/${encodeURIComponent(alertId)}/resolve`, + { + method: "PUT", + headers: { accept: "application/json", authorization: `Bearer ${options.apiKey}` }, + }, + options.deadline, + options.sleep, + false, + async () => { + await loadManagementBoundAlertSource( + options.fetchFn, + options.apiKey, + options.sourceId, + options.deadline, + options.sleep, + ); + }, + ); + await releaseResponseBody(response); + if (response.status !== 200) { + throw new Error("iLert management alert resolution failed"); + } +} + +interface AlertKeyLookupOptions { + apiKey: string; + deadline: Deadline; + fetchFn: Fetch; + from?: string; + completeInventory?: boolean; + key: string; + maxPages?: number; + onTarget?: (target: CanaryTarget) => void; + sleep: Sleep; + sourceId?: number; + states?: readonly ("PENDING" | "ACCEPTED" | "RESOLVED")[]; + until?: string; +} + +async function findAlertsByKey(options: AlertKeyLookupOptions): Promise { + return findAlertsByKeys({ + ...options, + keys: [options.key], + }); +} + +async function findAlertsByKeys( + options: Omit & { keys: readonly string[] }, +): Promise { + const matches = new Map(); + const keys = new Set(options.keys); + const retainedTargetIds = new Set(); + let validationError: InvalidCanaryAlertValidationError | undefined; + + const retainTarget = (target: CanaryTarget) => { + if (retainedTargetIds.has(target.alertId)) return; + retainedTargetIds.add(target.alertId); + options.onTarget?.(target); + }; + const queryFor = (start?: number): URLSearchParams => { + const query = new URLSearchParams(); + if (start !== undefined) { + query.set("max-results", String(ALERT_PAGE_SIZE)); + query.set("start-index", String(start)); + } + if (options.sourceId) query.append("sources", String(options.sourceId)); + if (options.from) query.append("from", options.from); + if (options.until) query.append("until", options.until); + for (const state of options.states ?? []) query.append("states", state); + return query; + }; + const fetchCount = async (): Promise => { + assertBeforeDeadline(options.deadline); + const value = requireObject( + await management( + options.fetchFn, + options.apiKey, + `/alerts/count?${queryFor().toString()}`, + {}, + options.deadline, + options.sleep, + ), + "iLert returned an invalid alert inventory count", + ); + const count = value.count; + if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) { + throw new Error("iLert returned an invalid alert inventory count"); + } + return count; + }; + const fetchPage = async (start: number): Promise<{ alerts: Json[]; ids: string[] }> => { + assertBeforeDeadline(options.deadline); + const alerts = await management( + options.fetchFn, + options.apiKey, + `/alerts?${queryFor(start).toString()}`, + {}, + options.deadline, + options.sleep, + ); + if (!Array.isArray(alerts) || alerts.some((item) => !object(item))) { + throw new Error("iLert returned an invalid alert list during the canary"); + } + for (const item of alerts) { + const alert = object(item); + if (typeof alert?.alertKey !== "string" || !keys.has(alert.alertKey)) continue; + try { + const observed = canaryObservation(alert, alert.alertKey, options.sourceId); + retainTarget(observed); + if (options.states === undefined || options.states.includes(observed.status)) { + matches.set(observed.alertId, alert); + } + } catch (error) { + const target = invalidCanaryAlertTarget(error); + if (target) retainTarget(target); + if (error instanceof InvalidCanaryAlertValidationError) { + validationError ??= error; + continue; + } + throw error; + } + } + const ids = alerts.map((item) => { + const alertId = opaqueId(item.id); + if (!alertId) { + throw new Error("iLert returned an offset-page alert without an identity"); + } + return alertId; + }); + return { alerts, ids }; + }; + + const listCompleteInventory = async (): Promise => { + const beforeCount = await fetchCount(); + const pages: Json[] = []; + const ids = new Set(); + for (let page = 0, start = 0; ; page += 1, start += ALERT_PAGE_SIZE) { + if (options.maxPages !== undefined && page >= options.maxPages) { + throw new Error("iLert terminal canary inventory exceeded its pagination limit"); + } + const found = await fetchPage(start); + for (const [index, alert] of found.alerts.entries()) { + const id = found.ids[index]!; + if (ids.has(id)) { + throw new Error("iLert alert inventory repeated an alert identity"); + } + ids.add(id); + pages.push(alert); + } + if (found.alerts.length < ALERT_PAGE_SIZE) break; + } + const afterCount = await fetchCount(); + if (beforeCount !== afterCount || afterCount !== ids.size) { + throw new Error("iLert alert inventory changed during pagination"); + } + return pages; + }; + + try { + if (!options.completeInventory) { + for (let page = 0, start = 0; ; page += 1, start += ALERT_PAGE_SIZE) { + if (options.maxPages !== undefined && page >= options.maxPages) { + throw new Error("iLert terminal canary inventory exceeded its pagination limit"); + } + const found = await fetchPage(start); + if (found.alerts.length < ALERT_PAGE_SIZE) { + if (validationError) throw validationError; + return [...matches.values()]; + } + } + } + const first = await listCompleteInventory(); + const second = await listCompleteInventory(); + const firstIds = first.map((alert) => opaqueId(alert.id)); + const secondIds = second.map((alert) => opaqueId(alert.id)); + if ( + firstIds.length !== secondIds.length || + firstIds.some((id, index) => id !== secondIds[index]) + ) { + throw new Error("iLert alert inventory changed during continuity revalidation"); + } + if (validationError) throw validationError; + return [...matches.values()]; + } catch (error) { + if (validationError && error !== validationError) { + throw new AggregateError( + [validationError, error], + "iLert canary inventory encountered an invalid matching alert before completion", + ); + } + throw error; + } +} + +async function findOpenManagedCanaryAlerts( + options: ManagementCleanupOptions & { maxPages: number }, +): Promise<{ errors: unknown[]; targets: CanaryTarget[] }> { + const targets = new Map(); + const queryFor = (start?: number): URLSearchParams => { + const query = new URLSearchParams(); + if (start !== undefined) { + query.set("max-results", String(ALERT_PAGE_SIZE)); + query.set("start-index", String(start)); + } + query.append("sources", String(options.sourceId)); + query.append("states", "PENDING"); + query.append("states", "ACCEPTED"); + return query; + }; + const fetchCount = async (): Promise => { + assertBeforeDeadline(options.deadline); + const value = requireObject( + await management( + options.fetchFn, + options.apiKey, + `/alerts/count?${queryFor().toString()}`, + {}, + options.deadline, + options.sleep, + ), + "iLert returned an invalid orphan canary inventory count", + ); + const count = value.count; + if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) { + throw new Error("iLert returned an invalid orphan canary inventory count"); + } + return count; + }; + const listCompleteInventory = async (): Promise => { + const beforeCount = await fetchCount(); + const identities = new Set(); + const inventory: string[] = []; + for (let page = 0, start = 0; ; page += 1, start += ALERT_PAGE_SIZE) { + if (page >= options.maxPages) { + throw new Error("iLert orphan canary inventory exceeded its pagination limit"); + } + assertBeforeDeadline(options.deadline); + const pageAlerts = await management( + options.fetchFn, + options.apiKey, + `/alerts?${queryFor(start).toString()}`, + {}, + options.deadline, + options.sleep, + ); + if (!Array.isArray(pageAlerts) || pageAlerts.some((alert) => !object(alert))) { + throw new Error("iLert returned an invalid orphan canary inventory page"); + } + for (const item of pageAlerts) { + const alert = object(item)!; + const alertId = opaqueId(alert.id); + if (!alertId) { + throw new Error("iLert returned an orphan canary inventory alert without an identity"); + } + if (identities.has(alertId)) { + throw new Error("iLert orphan canary inventory repeated an alert identity"); + } + identities.add(alertId); + inventory.push(alertId); + if ( + typeof alert.alertKey !== "string" || + !isManagedCanaryKey(alert.alertKey) + ) { + continue; + } + const target = canaryObservation(alert, alert.alertKey, options.sourceId); + targets.set(target.alertId, target); + } + if (pageAlerts.length < ALERT_PAGE_SIZE) break; + } + const afterCount = await fetchCount(); + if (beforeCount !== afterCount || afterCount !== identities.size) { + throw new Error("iLert orphan canary inventory changed during pagination"); + } + return inventory; + }; + + try { + const first = await listCompleteInventory(); + const second = await listCompleteInventory(); + if ( + first.length !== second.length || + first.some((id, index) => id !== second[index]) + ) { + throw new Error("iLert orphan canary inventory changed during continuity validation"); + } + return { errors: [], targets: [...targets.values()] }; + } catch (error) { + const target = invalidCanaryAlertTarget(error); + if (target) targets.set(target.alertId, target); + return { errors: [error], targets: [...targets.values()] }; + } +} + +function alertSourceId(alert: Json): number | null { + return positiveNumber(object(alert.alertSource)?.id); +} + +interface CreateAlertActionOptions { + apiKey: string; + beforeMutationAttempt: BeforeMutationAttempt; + deadline: Deadline; + desired: Json; + fetchFn: Fetch; + sleep: Sleep; + sourceId: number; +} + +async function createAlertActionSafely( + options: CreateAlertActionOptions, +): Promise { + let created: Json; + try { + created = await managementOnce( + options.fetchFn, + options.apiKey, + "/alert-actions?include=conditions", + { method: "POST", body: JSON.stringify(options.desired) }, + options.deadline, + options.beforeMutationAttempt, + ); + } catch (error) { + if (!(error instanceof AmbiguousAlertActionCreateError)) throw error; + const candidate = await loadReservedAlertAction(options); + if (candidate && equivalentAlertAction(candidate, options.desired)) return candidate; + throw new Error( + "iLert alert-action creation was ambiguous; refusing to submit a second POST", + ); + } + const candidate = await loadReservedAlertAction(options); + if ( + !candidate || + !equivalentAlertAction(candidate, options.desired) || + actionId(candidate) !== actionId(created) + ) { + throw new Error("iLert did not confirm the created alert action in the global inventory"); + } + return candidate; +} + +async function loadReservedAlertAction( + options: CreateAlertActionOptions, +): Promise { + const candidates = await loadCandidateActions( + options.fetchFn, + options.apiKey, + options.desired, + options.deadline, + options.sleep, + ); + return reservedCandidate(candidates, options.desired, options.sourceId); +} + +async function confirmUniqueMutatedAlertAction( + options: CreateAlertActionOptions & { expectedId: string; carried?: Json }, +): Promise { + const candidates = await loadCandidateActions( + options.fetchFn, + options.apiKey, + options.desired, + options.deadline, + options.sleep, + ); + const candidate = reservedCandidate(candidates, options.desired, options.sourceId); + if (!candidate || actionId(candidate) !== options.expectedId) { + throw new Error( + "iLert did not retain exactly one reconciled alert action after mutation", + ); + } + if (!equivalentAlertAction(candidate, options.desired)) { + throw new Error("iLert did not retain the reconciled alert action"); + } + if (options.carried && !sameJson( + normalizedCarriedAlertActionFields(carriedAlertActionFields(candidate)), + normalizedCarriedAlertActionFields(options.carried), + )) { + throw new Error("iLert did not retain the existing alert-action fields"); + } +} + +function reservedCandidate( + candidates: readonly Json[], + desired: Json, + sourceId: number, +): Json | undefined { + if (candidates.some((action) => { + const actionRelationIds = relationIds(action.alertSources); + return action.connectorType !== "webhook" || + actionRelationIds === null || + !sameSet(actionRelationIds, [sourceId]); + })) { + throw new Error( + "A conflicting Postil alert action exists; refusing to change its type or source scope", + ); + } + if (candidates.length > 1) { + throw new Error( + "Multiple Postil webhook alert actions exist; refusing to choose or delete one", + ); + } + return candidates[0]; +} + +async function managementOnce( + fetchFn: Fetch, + apiKey: string, + path: string, + init: RequestInit, + deadline: Deadline, + beforeAttempt?: BeforeMutationAttempt, +): Promise { + assertBeforeDeadline(deadline); + await beforeAttempt?.(); + assertBeforeDeadline(deadline); + const timeout = Math.min(REQUEST_TIMEOUT_MS, remaining(deadline)); + try { + const response = await fetchFn(`${API_BASE}${path}`, { + ...init, + headers: { + accept: "application/json", + authorization: `Bearer ${apiKey}`, + ...(init.body ? { "content-type": "application/json" } : {}), + }, + signal: AbortSignal.timeout(timeout), + }); + if (!response.ok) { + await releaseResponseBody(response); + throw new Error(`iLert alert-action creation request is ambiguous after HTTP ${response.status}`); + } + return requireObject( + await parseJson(response), + "iLert returned an invalid alert action", + ); + } catch (error) { + throw new AmbiguousAlertActionCreateError(error); + } +} + +class AmbiguousAlertActionCreateError extends Error { + constructor(cause: unknown) { + super("iLert alert-action creation request is ambiguous", { cause }); + } +} + +async function parseJson(response: Response): Promise { + try { + return await response.json(); + } catch { + throw new Error("Remote service returned invalid JSON"); + } +} + +async function releaseResponseBody(response: Response): Promise { + await response.body?.cancel().catch(() => undefined); +} + +async function requestWithRetry( + fetchFn: Fetch, + url: string, + init: RequestInit, + deadline: Deadline, + sleep: Sleep, + sensitive = false, + beforeAttempt?: BeforeMutationAttempt, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < REQUEST_ATTEMPTS; attempt += 1) { + assertBeforeDeadline(deadline); + await beforeAttempt?.(); + // Guards may perform bounded remote validation. Re-read the deadline only + // after that work completes so validation cannot borrow time reserved for + // cleanup or dispatch a mutation after its budget has expired. + assertBeforeDeadline(deadline); + const timeout = Math.min(REQUEST_TIMEOUT_MS, remaining(deadline)); + try { + const response = await fetchFn(url, { + ...init, + signal: AbortSignal.timeout(timeout), + }); + if (!transientStatus(response.status) || attempt + 1 >= REQUEST_ATTEMPTS) { + return response; + } + const delay = retryDelay(response, deadline); + await releaseResponseBody(response); + await sleep(delay); + } catch (error) { + lastError = error; + if (attempt + 1 >= REQUEST_ATTEMPTS) break; + await sleep(Math.min(REQUEST_RETRY_MS * (attempt + 1), remaining(deadline))); + } + } + if (sensitive) throw new Error("Sensitive remote request failed after bounded retries"); + throw new Error("Remote request failed after bounded retries", { cause: lastError }); +} + +function transientStatus(status: number): boolean { + return status === 429 || status === 408 || status >= 500; +} + +function retryDelay(response: Response, deadline: Deadline): number { + const retryAfter = response.headers.get("retry-after"); + let requested = REQUEST_RETRY_MS; + if (retryAfter && /^[0-9]+$/u.test(retryAfter)) { + requested = Number(retryAfter) * 1_000; + } else if (retryAfter) { + const retryAt = Date.parse(retryAfter); + if (Number.isFinite(retryAt)) requested = Math.max(0, retryAt - deadline.now()); + } + return Math.min(requested, MAX_RETRY_AFTER_MS, remaining(deadline)); +} + +async function listActions( + fetchFn: Fetch, + apiKey: string, + deadline: Deadline, + sleep: Sleep, +): Promise { + const list = async (): Promise => { + const actions: Json[] = []; + const ids = new Set(); + for (let pageIndex = 0, start = 0; ; pageIndex += 1, start += ALERT_PAGE_SIZE - 1) { + if (pageIndex >= ALERT_ACTION_INVENTORY_MAX_PAGES) { + throw new Error("iLert alert-action inventory exceeded its pagination limit"); + } + const query = new URLSearchParams({ + "start-index": String(start), + "max-results": String(ALERT_PAGE_SIZE), + }); + const page = await management( + fetchFn, + apiKey, + `/alert-actions?${query.toString()}`, + {}, + deadline, + sleep, + ); + if (!Array.isArray(page) || page.some((item) => !object(item))) { + throw new Error("iLert returned an invalid alert-action list"); + } + for (const action of page as Json[]) { + const id = actionId(action); + // Adjacent pages overlap by one record. Deduplication makes the + // resulting fingerprint independent of that expected boundary row. + if (ids.has(id)) continue; + ids.add(id); + actions.push(action); + } + if (page.length < ALERT_PAGE_SIZE) return actions; + } + }; + const first = await list(); + const second = await list(); + const firstIds = first.map(actionId); + const secondIds = second.map(actionId); + if ( + firstIds.length !== secondIds.length || + firstIds.some((id, index) => id !== secondIds[index]) + ) { + throw new Error("iLert alert-action inventory changed during continuity validation"); + } + return second; +} + +async function loadCandidateActions( + fetchFn: Fetch, + apiKey: string, + desired: Json, + deadline: Deadline, + sleep: Sleep, +): Promise { + const webhook = object(desired.params)?.webhookUrl; + const desiredEndpoint = receiverEndpoint(webhook); + if (!desiredEndpoint) throw new Error("iLert returned an invalid desired webhook URL"); + const summaries = (await listActions(fetchFn, apiKey, deadline, sleep)) + .filter((action) => reservedActionSummary(action, desiredEndpoint)); + return Promise.all(summaries.map(async (item) => { + const id = actionId(item); + const action = requireAlertActionId( + await management( + fetchFn, + apiKey, + `/alert-actions/${encodeURIComponent(id)}?include=conditions`, + {}, + deadline, + sleep, + ), + id, + "iLert returned an invalid alert action", + ); + if ( + action.name !== ACTION_NAME && + !sameReceiverEndpoint(object(action.params)?.webhookUrl, desiredEndpoint) + ) { + throw new Error("iLert returned an inconsistent reserved alert action"); + } + if (action.name === ACTION_NAME && !receiverEndpoint(object(action.params)?.webhookUrl)) { + throw new Error("iLert returned a reserved alert action with an invalid webhook URL"); + } + return action; + })); +} + +interface ReceiverEndpoint { + hostname: string; + pathname: string; + port: string; + protocol: string; + query: string; +} + +function reservedActionSummary(action: Json, desired: ReceiverEndpoint): boolean { + if (action.name === ACTION_NAME) { + const webhook = object(action.params)?.webhookUrl; + if (webhook !== undefined && !receiverEndpoint(webhook)) { + throw new Error("iLert returned a reserved alert action with an invalid webhook URL"); + } + return true; + } + return sameReceiverEndpoint(object(action.params)?.webhookUrl, desired); +} + +function sameReceiverEndpoint(value: unknown, desired: ReceiverEndpoint): boolean { + const actual = receiverEndpoint(value); + return actual !== null && + actual.protocol === desired.protocol && + actual.hostname === desired.hostname && + actual.port === desired.port && + actual.pathname === desired.pathname && + actual.query === desired.query; +} + +function receiverEndpoint(value: unknown): ReceiverEndpoint | null { + if (typeof value !== "string") return null; + try { + const parsed = new URL(value); + const query = [...parsed.searchParams.entries()] + .sort(([leftName, leftValue], [rightName, rightValue]) => + leftName === rightName + ? leftValue < rightValue ? -1 : leftValue > rightValue ? 1 : 0 + : leftName < rightName ? -1 : 1, + ) + .map(([name, entry]) => `${encodeURIComponent(name)}=${encodeURIComponent(entry)}`) + .join("&"); + return { + hostname: parsed.hostname.toLowerCase(), + pathname: parsed.pathname, + port: parsed.port, + protocol: parsed.protocol.toLowerCase(), + query, + }; + } catch { + return null; + } +} + +async function loadIntegrationBoundAlertSource( + fetchFn: Fetch, + apiKey: string, + integrationKey: string, + sourceId: number, + deadline: Deadline, + sleep: Sleep, +): Promise { + try { + const response = await requestWithRetry( + fetchFn, + `${API_BASE}/alert-sources/${encodeURIComponent(sourceId)}`, + { headers: { accept: "application/json", authorization: `Bearer ${apiKey}` } }, + deadline, + sleep, + true, + ); + if (response.status !== 200) { + await releaseResponseBody(response); + throw new Error("binding request failed"); + } + return alertSource(await parseJson(response), sourceId, integrationKey); + } catch { + throw new Error("iLert integration binding validation failed"); + } +} + +async function loadManagementBoundAlertSource( + fetchFn: Fetch, + apiKey: string, + sourceId: number, + deadline: Deadline, + sleep: Sleep, +): Promise { + try { + const response = await requestWithRetry( + fetchFn, + `${API_BASE}/alert-sources/${encodeURIComponent(sourceId)}`, + { headers: { accept: "application/json", authorization: `Bearer ${apiKey}` } }, + deadline, + sleep, + true, + ); + if (response.status !== 200) { + await releaseResponseBody(response); + throw new Error("source request failed"); + } + return managementAlertSource(await parseJson(response), sourceId); + } catch (error) { + throw new Error("iLert management alert-source validation failed", { + cause: error, + }); + } +} + +function managementAlertSource(value: unknown, expectedId: number): Json { + const source = requireObject(value, "iLert returned an invalid alert source"); + if (positiveNumber(source.id) !== expectedId || source.integrationType !== "API") { + throw new Error("iLert alert source does not match the configured management identity"); + } + return { id: expectedId }; +} + +function alertSource( + value: unknown, + expectedId: number, + expectedIntegrationKey: string, +): Json { + const source = requireObject(value, "iLert returned an invalid alert source"); + const id = positiveNumber(source.id); + if ( + id !== expectedId || + source.integrationType !== "API" || + typeof source.integrationKey !== "string" || + !sameSecret(source.integrationKey, expectedIntegrationKey) + ) { + throw new Error("iLert alert source does not match the configured identity"); + } + return { id }; +} + +function sameSecret(left: string, right: string): boolean { + const leftBytes = Buffer.from(left, "utf8"); + const rightBytes = Buffer.from(right, "utf8"); + return leftBytes.length === rightBytes.length && timingSafeEqual(leftBytes, rightBytes); +} + +function actionId(value: unknown): string { + const id = opaqueId(object(value)?.id); + if (id) return id; + throw new Error("iLert returned an alert action without an identity"); +} + +function requireAlertActionId( + value: unknown, + expectedId: string, + invalidMessage: string, +): Json { + const action = requireObject(value, invalidMessage); + if (actionId(action) !== expectedId) { + throw new Error("iLert returned a detail for a different alert action"); + } + return action; +} + +function opaqueId(value: unknown): string | null { + if (typeof value === "number" && Number.isSafeInteger(value) && value > 0) { + return String(value); + } + if ( + typeof value === "string" && + value.length <= 128 && + /^[A-Za-z0-9._:-]+$/u.test(value) + ) { + return value; + } + return null; +} + +function positiveId(value: unknown): string | null { + const id = positiveNumber(value); + return id ? String(id) : null; +} + +function requireCanaryRunIdentity( + runId: string | undefined, + runAttempt: string | undefined, +): { runAttempt: number; runId: string } { + if (!runId || !runAttempt) { + throw new Error( + "POSTIL_ILERT_CANARY_RUN_ID and POSTIL_ILERT_CANARY_RUN_ATTEMPT are required for live reconciliation", + ); + } + return canaryRunIdentity(runId, runAttempt); +} + +function canaryRunIdentity( + runId: string, + runAttempt: string, +): { runAttempt: number; runId: string } { + const normalizedRunId = positiveId(runId); + const normalizedRunAttempt = positiveNumber(runAttempt); + if (!normalizedRunId) { + throw new Error("GitHub run ID must be a positive integer"); + } + if ( + !normalizedRunAttempt || + normalizedRunAttempt > MAX_CANARY_RUN_ATTEMPT + ) { + throw new Error( + `GitHub run attempt must be between 1 and ${MAX_CANARY_RUN_ATTEMPT}`, + ); + } + return { runAttempt: normalizedRunAttempt, runId: normalizedRunId }; +} + +function isDeterministicCanaryKey(value: string): boolean { + const match = new RegExp(`^${CANARY_KEY_PREFIX}-([1-9][0-9]*)-([1-9][0-9]*)$`, "u") + .exec(value); + if (!match) return false; + try { + canaryRunIdentity(match[1]!, match[2]!); + return true; + } catch { + return false; + } +} + +function isManagedCanaryKey(value: string): boolean { + return value === COMPATIBILITY_CANARY_KEY || isDeterministicCanaryKey(value); +} + +function reportTime(milliseconds: number): string { + return new Date(milliseconds).toISOString(); +} + +function positiveNumber(value: unknown): number | null { + const parsed = + typeof value === "string" && /^[1-9][0-9]*$/u.test(value) + ? Number(value) + : value; + return typeof parsed === "number" && Number.isSafeInteger(parsed) && parsed > 0 + ? parsed + : null; +} + +function relationIds(value: unknown): number[] | null { + if (!Array.isArray(value)) return null; + const ids: number[] = []; + for (const item of value) { + const id = positiveNumber(object(item)?.id); + if (!id) return null; + ids.push(id); + } + return ids; +} + +function hasNoHeaders(value: unknown): boolean { + return value === undefined || (Array.isArray(value) && value.length === 0); +} + +function emptyText(value: unknown): boolean { + return value === undefined || value === null || value === ""; +} + +function emptyAlertFilter(value: unknown): boolean { + return value === undefined || value === null || + (object(value) !== null && Object.keys(object(value)!).length === 0); +} + +function strings(value: unknown): string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string") + ? value + : []; +} + +function sameSet(left: T[], right: T[]): boolean { + return [...left].sort().join("\0") === [...right].sort().join("\0"); +} + +function object(value: unknown): Json | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Json) + : null; +} + +function requireObject(value: unknown, message: string): Json { + const result = object(value); + if (!result) throw new Error(message); + return result; +} + +function nonempty(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +export function validWebhookSecret(secret: string): boolean { + return /^[\x21-\x7e]{32,512}$/u.test(secret) && new Set(secret).size >= 4; +} + +function requireWebhookSecret(secret: string): void { + if (!validWebhookSecret(secret)) { + throw new Error( + "POSTIL_ILERT_WEBHOOK_SECRET must contain 32 to 512 random printable ASCII bytes", + ); + } +} + +function webhookUrl(secret: string, receiverOrigin: string): string { + requireWebhookSecret(secret); + const parsed = new URL(parseReceiverOrigin(receiverOrigin)); + return `https://${encodeURIComponent("postil-ilert")}:${encodeURIComponent(secret)}@${parsed.host}/api/webhooks/ilert`; +} + +function basicAuthorization(secret: string): string { + return `Basic ${Buffer.from(`postil-ilert:${secret}`, "utf8").toString("base64")}`; +} + +function remaining(deadline: Deadline): number { + return deadline.expiresAt - deadline.now(); +} + +function assertBeforeDeadline(deadline: Deadline): void { + if (remaining(deadline) <= 0) throw new Error("iLert canary deadline expired"); +} + +function assertCleanupReserve(deadline: Deadline): void { + if (remaining(deadline) < CANARY_CLEANUP_RESERVE_MS) { + throw new Error("iLert canary cannot start without its cleanup reserve"); + } +} + +function environment( + values: Record, + name: string, +): string { + const value = values[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +interface CliOptions { + args?: string[]; + env?: Record; + fetchFn?: Fetch; + sleep?: Sleep; + now?: Clock; + log?: (message: string) => void; +} + +export async function runCli(options: CliOptions = {}): Promise { + const args = options.args ?? process.argv.slice(2); + if ( + args.some( + (arg) => + arg !== "--dry-run" && arg !== "--canary" && arg !== "--finalize-canary" && + arg !== "--sweep-canary-orphans" && arg !== "--validate-webhook-secret", + ) + ) { + throw new Error( + "usage: reconcile-ilert-alert-stream.ts [--validate-webhook-secret|--dry-run|--canary|--finalize-canary|--sweep-canary-orphans]", + ); + } + const dryRun = args.includes("--dry-run"); + const canary = args.includes("--canary"); + const finalizeCanary = args.includes("--finalize-canary"); + const sweepCanaryOrphans = args.includes("--sweep-canary-orphans"); + const validateWebhookSecret = args.includes("--validate-webhook-secret"); + if (validateWebhookSecret && (dryRun || canary || finalizeCanary || sweepCanaryOrphans)) { + throw new Error("--validate-webhook-secret cannot be combined with reconciliation commands"); + } + if (dryRun && (canary || finalizeCanary || sweepCanaryOrphans)) { + throw new Error("--dry-run cannot be combined with a canary command"); + } + if ( + [canary, finalizeCanary, sweepCanaryOrphans].filter(Boolean).length > 1 + ) { + throw new Error("canary recovery commands cannot be combined"); + } + if (!validateWebhookSecret && !dryRun && !canary && !finalizeCanary && !sweepCanaryOrphans) { + throw new Error( + "usage: live reconciliation requires --canary or --sweep-canary-orphans; use --dry-run to preview", + ); + } + const values = options.env ?? process.env; + const log = options.log ?? console.log; + + if (validateWebhookSecret) { + requireWebhookSecret(environment(values, "POSTIL_ILERT_WEBHOOK_SECRET")); + log("POSTIL_ILERT_WEBHOOK_SECRET is valid"); + return; + } + + if (finalizeCanary) { + const alertSubmitted = canaryAlertSubmission(values); + await finalizeIlertWebhookCanary({ + alertSubmitted, + apiKey: environment(values, "ILERT_API_KEY"), + fetchFn: options.fetchFn, + integrationKey: environment(values, "ILERT_INTEGRATION_KEY"), + now: options.now, + runAttempt: environment(values, "POSTIL_ILERT_CANARY_RUN_ATTEMPT"), + sweepAttempt: values.POSTIL_ILERT_CANARY_SWEEP_ATTEMPT, + runId: environment(values, "POSTIL_ILERT_CANARY_RUN_ID"), + sleep: options.sleep, + sourceId: requiredSourceId(values), + }); + log("iLert canary cleanup is stabilized"); + return; + } + + if (sweepCanaryOrphans) { + await sweepIlertWebhookCanaryOrphans({ + apiKey: environment(values, "ILERT_API_KEY"), + fetchFn: options.fetchFn, + now: options.now, + sleep: options.sleep, + sourceId: requiredSourceId(values), + }); + log("iLert orphan canary sweep is stabilized"); + return; + } + + const sourceId = requiredSourceId(values); + const shared = { + apiKey: environment(values, "ILERT_API_KEY"), + fetchFn: options.fetchFn, + integrationKey: environment(values, "ILERT_INTEGRATION_KEY"), + now: options.now, + receiverOrigin: environment(values, "POSTIL_ILERT_RECEIVER_ORIGIN"), + sleep: options.sleep, + sourceId, + webhookSecret: environment(values, "POSTIL_ILERT_WEBHOOK_SECRET"), + }; + const runId = canary + ? environment(values, "POSTIL_ILERT_CANARY_RUN_ID") + : undefined; + const runAttempt = canary + ? environment(values, "POSTIL_ILERT_CANARY_RUN_ATTEMPT") + : undefined; + if (canary) { + await recordCanaryAlertSubmission(values, "unknown", runAttempt); + } + const result = await reconcileIlertAlertAction({ + ...shared, + dryRun, + runAttempt, + runId, + }); + log(`iLert webhook-action reconciliation${dryRun ? " plan" : ""}: ${result.operation}`); + if (canary) { + let alertAccepted = false; + try { + await verifyIlertWebhookCanary({ + ...shared, + onAlertAttempted: () => recordCanaryAlertSubmission(values, "unknown", runAttempt), + onAlertSubmitted: async () => { + alertAccepted = true; + await recordCanaryAlertSubmission(values, "true", runAttempt); + }, + runAttempt: runAttempt!, + runId: runId!, + }); + } catch (error) { + await recordCanaryAlertSubmission( + values, + alertAccepted ? "true" : "unknown", + runAttempt, + ); + throw error; + } + await recordCanaryAlertSubmission(values, "cleaned", runAttempt); + log("Postil persisted the iLert canary create and resolve webhooks"); + } +} + +function requiredSourceId(values: Record): number { + const sourceId = Number(environment(values, "POSTIL_ILERT_ALERT_SOURCE_ID")); + if (!Number.isSafeInteger(sourceId) || sourceId <= 0) { + throw new Error("POSTIL_ILERT_ALERT_SOURCE_ID must be a positive integer"); + } + return sourceId; +} + +function canaryAlertSubmission( + values: Record, +): CanaryHandoff | undefined { + const value = values.POSTIL_ILERT_CANARY_ALERT_SUBMITTED; + if (value === undefined || value === "") return undefined; + if (value === "true") return "true"; + if (value === "unknown" || value === "cleaned") return value; + throw new Error("POSTIL_ILERT_CANARY_ALERT_SUBMITTED must be true, unknown, or cleaned"); +} + +async function recordCanaryAlertSubmission( + values: Record, + submitted: CanaryHandoff, + producerAttempt?: string, +): Promise { + const output = values.GITHUB_OUTPUT; + if (!output) return; + await appendFile( + output, + `alert_submitted=${submitted}\n${producerAttempt ? `producer_attempt=${producerAttempt}\n` : ""}`, + "utf8", + ); +} + +if (import.meta.main) await runCli(); diff --git a/scripts/run-release-migrations.ts b/scripts/run-release-migrations.ts index de01e4d9..93ec9d44 100644 --- a/scripts/run-release-migrations.ts +++ b/scripts/run-release-migrations.ts @@ -7,6 +7,7 @@ import { Pool } from "pg"; import { COMPATIBLE_MANAGED_RELEASE_PROTOCOL, applyReviewedFeedbackControlMigration, + applyReviewedIlertCanaryAlertKeyMigration, applyReviewedMonitoringDeliveryMigration, type ManagedReleaseMigrationIdentity, prepareCompatibleManagedRelease, @@ -80,6 +81,16 @@ export async function runReleaseMigrations( spawnCommand, signal, ); + // The reviewed migrations above add columns only. Their lookup index is + // built concurrently here, before preparation, so a failed or blocked build + // stops the release while the running fleet is unchanged. + await runReleaseDatabaseCommand( + ["bun", "run", "operational:indexes", "--managed-release"], + "release managed operational indexes", + databaseEnvironment, + spawnCommand, + signal, + ); await prepareCompatibleRelease(databaseEnvironment); } @@ -171,40 +182,49 @@ export async function prepareReviewedAdditiveMigration( const releaseSha = databaseEnvironment.POSTIL_RELEASE_SHA ?? ""; const protocol = requireCompatibleReleaseProtocol(databaseEnvironment); const migrations = checkedInReleaseMigrations(); - const predecessor = migrations.at(-2); - const latest = migrations.at(-1); - if (!predecessor || !latest) throw new Error("reviewed release migration identities are missing"); + // The reviewed migrations occupy the journal tail in application order. + const reviewed = [ + { apply: applyReviewedMonitoringDeliveryMigration, file: "0062_monitor_delivery_receipt.sql" }, + { apply: applyReviewedFeedbackControlMigration, file: "0063_review_feedback_control.sql" }, + { apply: applyReviewedIlertCanaryAlertKeyMigration, file: "0064_ilert_canary_alert_key.sql" }, + ].map((step, index, steps) => { + const through = migrations.slice(0, migrations.length - (steps.length - 1 - index)); + return { ...step, through, identity: through.at(-1) }; + }); + if (migrations.length < reviewed.length || reviewed.some((step) => !step.identity)) { + throw new Error("reviewed release migration identities are missing"); + } const journal = await pool.query<{ created_at: string }>( "SELECT created_at::text FROM drizzle.__drizzle_migrations WHERE created_at = ANY($1::bigint[])", - [[predecessor.folderMillis, latest.folderMillis]], + [reviewed.map((step) => step.identity!.folderMillis)], ); - const applied = new Set(journal.rows.map((row) => row.created_at)); - if (applied.has(String(latest.folderMillis)) && !applied.has(String(predecessor.folderMillis))) { + const journaled = new Set(journal.rows.map((row) => row.created_at)); + const applied = reviewed.map((step) => journaled.has(String(step.identity!.folderMillis))); + const latestApplied = applied.lastIndexOf(true); + if (applied[1] && !applied[0]) { throw new Error("reviewed feedback control migration is missing its monitoring predecessor"); } - let monitoringPending = false; - if (!applied.has(String(latest.folderMillis))) { - monitoringPending = await applyReviewedMonitoringDeliveryMigration( + if (applied[2] && !applied[1]) { + throw new Error("reviewed alert key migration is missing its feedback control predecessor"); + } + // The latest journaled step is verified against exactly its own history, + // and every later step applies in order. A retry therefore never presents + // an already journaled migration to an earlier step as unknown. + let pending = false; + for (const step of reviewed.slice(Math.max(latestApplied, 0))) { + const stepPending = await step.apply( pool, sourceReleaseSha, releaseSha, protocol, - migrations.slice(0, -1), - await readFile(new URL("../drizzle/0062_monitor_delivery_receipt.sql", import.meta.url), "utf8"), + step.through, + await readFile(new URL(`../drizzle/${step.file}`, import.meta.url), "utf8"), { dryRun }, ); - if (dryRun && monitoringPending) return true; + if (dryRun && stepPending) return true; + pending ||= stepPending; } - const feedbackPending = await applyReviewedFeedbackControlMigration( - pool, - sourceReleaseSha, - releaseSha, - protocol, - migrations, - await readFile(new URL("../drizzle/0063_review_feedback_control.sql", import.meta.url), "utf8"), - { dryRun }, - ); - return monitoringPending || feedbackPending; + return pending; } finally { if (!existingPool) await pool.end(); } diff --git a/src/app/api/webhooks/ilert/route.ts b/src/app/api/webhooks/ilert/route.ts index 2de34faf..1f308a76 100644 --- a/src/app/api/webhooks/ilert/route.ts +++ b/src/app/api/webhooks/ilert/route.ts @@ -4,8 +4,10 @@ import { readBoundedWebhookBody } from "@/lib/crypto/webhook"; import { getDb } from "@/lib/db"; import { configuredIlertWebhookSecret, + hasIlertCanaryAlertEvent, ILERT_WEBHOOK_MAX_BODY_BYTES, isApplicationJson, + isIlertCanaryAlertKey, parseIlertWebhookBody, recordIlertAlertEvent, verifyIlertWebhookAuthorization, @@ -15,27 +17,64 @@ import { reportOperationalFailure } from "@/lib/server-observability"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; -export async function POST(request: Request): Promise { - const secret = configuredIlertWebhookSecret(); - if (!secret) { - return NextResponse.json({ error: "not found" }, { status: 404 }); +const canarySourceId = /^[1-9][0-9]{0,18}$/u; +const POSTGRES_MAX_BIGINT = 9_223_372_036_854_775_807n; + +export async function GET(request: Request): Promise { + const unauthorized = authorizeIlertWebhook(request); + if (unauthorized) return unauthorized; + + const query = new URL(request.url).searchParams; + if (query.size === 0) { + return new NextResponse(null, { + status: 204, + headers: { "cache-control": "no-store" }, + }); } + const alertKey = query.get("alertKey"); + const eventType = query.get("eventType"); + const sourceId = query.get("sourceId"); if ( - !verifyIlertWebhookAuthorization( - request.headers.get("authorization"), - secret, - ) + query.size !== 3 || + !alertKey || + !isIlertCanaryAlertKey(alertKey) || + (eventType !== "alert-created" && eventType !== "alert-resolved") || + !sourceId || + !canarySourceId.test(sourceId) || + BigInt(sourceId) > POSTGRES_MAX_BIGINT ) { return NextResponse.json( - { error: "unauthorized" }, + { error: "invalid canary observation" }, + { status: 400, headers: { "cache-control": "no-store" } }, + ); + } + + try { + const received = await hasIlertCanaryAlertEvent( + getDb(), + alertKey, + eventType, + BigInt(sourceId), + ); + return NextResponse.json( + { received }, + { headers: { "cache-control": "no-store" } }, + ); + } catch (error) { + reportOperationalFailure("web", "ilert_webhook_processing_failed", error); + return NextResponse.json( + { error: "webhook observation unavailable" }, { - status: 401, - headers: { - "www-authenticate": 'Basic realm="postil-ilert", charset="UTF-8"', - }, + status: 503, + headers: { "cache-control": "no-store", "retry-after": "5" }, }, ); } +} + +export async function POST(request: Request): Promise { + const unauthorized = authorizeIlertWebhook(request); + if (unauthorized) return unauthorized; if (!isApplicationJson(request.headers.get("content-type"))) { return NextResponse.json( { error: "application/json required" }, @@ -76,3 +115,27 @@ export async function POST(request: Request): Promise { ); } } + +function authorizeIlertWebhook(request: Request): NextResponse | null { + const secret = configuredIlertWebhookSecret(); + if (!secret) { + return NextResponse.json({ error: "not found" }, { status: 404 }); + } + if ( + verifyIlertWebhookAuthorization( + request.headers.get("authorization"), + secret, + ) + ) { + return null; + } + return NextResponse.json( + { error: "unauthorized" }, + { + status: 401, + headers: { + "www-authenticate": 'Basic realm="postil-ilert", charset="UTF-8"', + }, + }, + ); +} diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index a78257b6..8462e3b1 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -1682,6 +1682,7 @@ export const ilertAlertEvents = pgTable( .generatedAlwaysAsIdentity(), eventId: uuid("event_id").notNull(), alertId: text("alert_id").notNull(), + alertKey: text("alert_key"), eventType: text("event_type").notNull(), status: text("status").notNull(), priority: text("priority").notNull(), @@ -1699,10 +1700,17 @@ export const ilertAlertEvents = pgTable( (t) => [ uniqueIndex("ilert_alert_events_event_id_idx").on(t.eventId), index("ilert_alert_events_alert_sequence_idx").on(t.alertId, t.sequence), + index("ilert_alert_events_canary_observation_idx") + .on(t.alertKey, t.eventType, t.alertSourceId) + .where(sql`${t.alertKey} IS NOT NULL`), check( "ilert_alert_events_alert_id_check", sql`${t.alertId} ~ '^[1-9][0-9]{0,63}$'`, ), + check( + "ilert_alert_events_alert_key_check", + sql`${t.alertKey} IS NULL OR length(${t.alertKey}) BETWEEN 1 AND 512`, + ), check( "ilert_alert_events_event_type_check", sql`length(${t.eventType}) <= 64 AND ${t.eventType} ~ '^alert-[a-z]+(-[a-z]+)*$'`, diff --git a/src/lib/ilert-alerts.ts b/src/lib/ilert-alerts.ts index 71630133..da40cea6 100644 --- a/src/lib/ilert-alerts.ts +++ b/src/lib/ilert-alerts.ts @@ -28,6 +28,8 @@ const ILERT_ALERT_PENDING_NOTIFICATION_LIMIT = 256; const POSTGRES_MAX_BIGINT = 9_223_372_036_854_775_807n; const ILERT_EVENT_WRITE_LOCK = 580_178_254_943_325_001n; const SUPABASE_POOLER_HOST = /(?:^|[.])pooler[.]supabase[.]com$/iu; +const ILERT_CANARY_ALERT_KEY = + /^postil-ilert-webhook-canary-([1-9][0-9]{0,15})-([1-9]|[1-4][0-9]|5[01])$/u; const ilertEventType = z .string() @@ -118,6 +120,7 @@ export interface StoredIlertAlertEvent { sequence: bigint; eventId: string; alertId: string; + alertKey: string | null; eventType: IlertAlertEventType; status: "PENDING" | "ACCEPTED" | "RESOLVED"; priority: "HIGH" | "LOW"; @@ -209,6 +212,11 @@ export function parseIlertWebhookBody( return parsed.success ? parsed.data : null; } +export function isIlertCanaryAlertKey(value: string): boolean { + const match = ILERT_CANARY_ALERT_KEY.exec(value); + return match !== null && Number.isSafeInteger(Number(match[1])); +} + /** Insert and notify atomically. Duplicate iLert event ids are silent. */ export async function recordIlertAlertEvent( db: Database, @@ -226,6 +234,9 @@ export async function recordIlertAlertEvent( .values({ eventId: event.eventId, alertId: event.id, + // iLert may send an empty key for an alert without one; the column + // holds NULL for every event that carries no usable key. + alertKey: event.alertKey || null, eventType: event.eventType, status: event.status, priority: event.priority, @@ -277,6 +288,26 @@ export async function getIlertAlertEvent( return (rows[0] as StoredIlertAlertEvent | undefined) ?? null; } +export async function hasIlertCanaryAlertEvent( + db: Database, + alertKey: string, + eventType: "alert-created" | "alert-resolved", + alertSourceId: bigint, +): Promise { + const rows = await db + .select({ sequence: schema.ilertAlertEvents.sequence }) + .from(schema.ilertAlertEvents) + .where( + and( + eq(schema.ilertAlertEvents.alertKey, alertKey), + eq(schema.ilertAlertEvents.eventType, eventType), + eq(schema.ilertAlertEvents.alertSourceId, alertSourceId), + ), + ) + .limit(1); + return rows.length === 1; +} + /** Require a renewable CLI login and the separate operator allowlist. */ export async function authorizeIlertAlertStream( db: Database, diff --git a/src/lib/release-job-rollout.ts b/src/lib/release-job-rollout.ts index 987c490e..5aa6ccdd 100644 --- a/src/lib/release-job-rollout.ts +++ b/src/lib/release-job-rollout.ts @@ -471,6 +471,33 @@ const REVIEWED_FEEDBACK_CONTROL_MIGRATION = { hash: "ae760e265f2a6f6fbaf1bad71c6e3552bb25f819a285f8e9f02db2f7a110d8d8", } as const; +const REVIEWED_ILERT_CANARY_ALERT_KEY_MIGRATION = { + folderMillis: 1791572108139, + hash: "5c86858f27d8b7311bbe863468b421648dbe8a76853680f93a56425acdadb9c4", +} as const; + +/** Apply the alert key column only after the reviewed feedback control schema. */ +export async function applyReviewedIlertCanaryAlertKeyMigration( + pool: Pool, + sourceReleaseSha: string, + releaseSha: string, + protocol: string, + migrations: readonly ManagedReleaseMigrationIdentity[], + source: string, + options: { dryRun?: boolean } = {}, +): Promise { + checkedMigrationIdentities(migrations); + const predecessor = migrations.filter((migration) => + migration.folderMillis < REVIEWED_ILERT_CANARY_ALERT_KEY_MIGRATION.folderMillis + ).sort((left, right) => right.folderMillis - left.folderMillis)[0]; + if (predecessor?.folderMillis !== REVIEWED_FEEDBACK_CONTROL_MIGRATION.folderMillis || + predecessor.hash !== REVIEWED_FEEDBACK_CONTROL_MIGRATION.hash) { + throw new Error("alert key migration requires the reviewed feedback control predecessor"); + } + return applyExactReviewedMigration(pool, sourceReleaseSha, releaseSha, protocol, + migrations, source, REVIEWED_ILERT_CANARY_ALERT_KEY_MIGRATION, options); +} + /** Apply the feedback control only after the reviewed monitor delivery schema. */ export async function applyReviewedFeedbackControlMigration( pool: Pool, diff --git a/tests/compatible-release-rollout.test.ts b/tests/compatible-release-rollout.test.ts index fea88cd2..404b54e5 100644 --- a/tests/compatible-release-rollout.test.ts +++ b/tests/compatible-release-rollout.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { Client, Pool } from "pg"; +import { ensureOperationalIndexes } from "../scripts/ensure-operational-indexes"; import { checkedInReleaseMigrations, runReleaseMigrations } from "../scripts/run-release-migrations"; import { COMPATIBLE_MANAGED_RELEASE_BOOTSTRAP_SHAS, @@ -12,6 +13,7 @@ import { PRIVATE_REVIEW_AUTHOR_CAPABILITY, RELEASE_V1_JOBS_CAPABILITY, applyReviewedFeedbackControlMigration, + applyReviewedIlertCanaryAlertKeyMigration, applyReviewedMonitoringDeliveryMigration, compatibleManagedReleaseProtocolCapability, hostedInferenceCapability, @@ -24,9 +26,29 @@ import { const TEST_URL = process.env.POSTIL_TEST_DATABASE_URL; const describeDatabase = TEST_URL ? describe : describe.skip; +/** The alert key check as PostgreSQL records it, or no rows before migration 0064. */ +async function alertKeyCheck(pool: Pool): Promise> { + return (await pool.query<{ validated: boolean; definition: string }>( + `SELECT convalidated AS validated, pg_get_constraintdef(oid) AS definition + FROM pg_constraint + WHERE conrelid = 'public.ilert_alert_events'::regclass + AND conname = 'ilert_alert_events_alert_key_check'`, + )).rows; +} + +const UNVALIDATED_ALERT_KEY_CHECK = [{ + validated: false, + definition: expect.stringMatching(/^CHECK \(.*alert_key.*\) NOT VALID$/), +}]; + +/** The checked-in history that ends at reviewed feedback control migration 0063. */ +function migrationsThroughFeedbackControl() { + return checkedInReleaseMigrations().slice(0, -1); +} + describe("compatible managed release identity", () => { test("rejects malformed full migration lists before opening any database transaction", async () => { - const migrations = checkedInReleaseMigrations(); + const migrations = migrationsThroughFeedbackControl(); const latest = migrations.at(-1)!; const source = await readFile(join(import.meta.dir, "..", "drizzle", "0063_review_feedback_control.sql"), "utf8"); let connected = false; @@ -65,14 +87,15 @@ describe("compatible managed release identity", () => { }); }); -describeDatabase("managed release upgrade from monitoring migration 0061", () => { +describeDatabase("managed release upgrade from feedback migration 0061", () => { const databaseName = `postil_release_upgrade_${process.pid}_${Date.now()}`; const cleanDatabaseName = `${databaseName}_clean`; const sourceRelease = COMPATIBLE_MANAGED_RELEASE_BOOTSTRAP_SHAS[0]!; const targetRelease = "a".repeat(40); const migrations = checkedInReleaseMigrations(); - const monitoring = migrations.at(-2)!; - const feedback = migrations.at(-1)!; + const monitoring = migrations.at(-3)!; + const feedback = migrations.at(-2)!; + const alertKey = migrations.at(-1)!; let admin: Client; let pool: Pool; let cleanPool: Pool; @@ -88,7 +111,7 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => const migration = new Client({ connectionString: url.toString() }); await migration.connect(); for (const file of (await readdir(join(import.meta.dir, "..", "drizzle"))) - .filter((name) => /^\d{4}_.*\.sql$/.test(name) && !/^006[23]_/.test(name)) + .filter((name) => /^\d{4}_.*\.sql$/.test(name) && !/^006[234]_/.test(name)) .sort()) { const source = await readFile(join(import.meta.dir, "..", "drizzle", file), "utf8"); for (const statement of source.split("--> statement-breakpoint")) { @@ -99,7 +122,7 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => await migration.query(`CREATE TABLE drizzle.__drizzle_migrations ( id serial PRIMARY KEY, hash text NOT NULL, created_at bigint )`); - for (const identity of migrations.slice(0, -2)) { + for (const identity of migrations.slice(0, -3)) { await migration.query( "INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ($1, $2)", [identity.hash, identity.folderMillis], @@ -144,11 +167,24 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => await admin?.end(); }, 30_000); - test("rolls back a failed 0062, then applies 0062 and 0063 once in order", async () => { + test("rolls back each failed step, then applies 0062, 0063, and 0064 once in order", async () => { const commands: string[][] = []; + // The managed index command runs in process against the upgraded database. const release = () => runReleaseMigrations(environment, - (command) => { commands.push([...command]); return { exited: Promise.resolve(0) }; }, + (command) => { + commands.push([...command]); + return { + exited: command.includes("operational:indexes") + ? ensureOperationalIndexes(pool, { managedRelease: true }).then(() => 0, () => 1) + : Promise.resolve(0), + }; + }, undefined, async () => true); + const canaryIndex = async () => (await pool.query<{ definition: string | null; valid: boolean | null }>( + `SELECT pg_get_indexdef(indexrelid) AS definition, indisvalid AS valid + FROM pg_index + WHERE indexrelid = to_regclass('public.ilert_alert_events_canary_observation_idx')`, + )).rows; const journal = async () => (await pool.query<{ created_at: string; hash: string }>( "SELECT created_at::text, hash FROM drizzle.__drizzle_migrations WHERE created_at >= $1 ORDER BY created_at", [monitoring.folderMillis], @@ -156,12 +192,12 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => const feedbackSource = await readFile(join(import.meta.dir, "..", "drizzle", "0063_review_feedback_control.sql"), "utf8"); await expect(applyReviewedFeedbackControlMigration(pool, sourceRelease, targetRelease, - COMPATIBLE_MANAGED_RELEASE_PROTOCOL, migrations, feedbackSource)) + COMPATIBLE_MANAGED_RELEASE_PROTOCOL, migrations.slice(0, -1), feedbackSource)) .rejects.toThrow("pending migrations"); expect(await journal()).toEqual([]); const monitoringSource = await readFile(join(import.meta.dir, "..", "drizzle", "0062_monitor_delivery_receipt.sql"), "utf8"); await expect(applyReviewedMonitoringDeliveryMigration(pool, sourceRelease, targetRelease, - COMPATIBLE_MANAGED_RELEASE_PROTOCOL, migrations.slice(0, -1), monitoringSource + "\n")) + COMPATIBLE_MANAGED_RELEASE_PROTOCOL, migrations.slice(0, -2), monitoringSource + "\n")) .rejects.toThrow("reviewed identity"); expect(await journal()).toEqual([]); @@ -188,11 +224,39 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => expect(commands).toEqual([]); await pool.query("ALTER TABLE drizzle.__drizzle_migrations DISABLE TRIGGER reject_feedback_journal_insert"); + const alertKeyColumn = async () => (await pool.query( + "SELECT column_name FROM information_schema.columns WHERE table_name='ilert_alert_events' AND column_name='alert_key'", + )).rows; + await pool.query(`CREATE FUNCTION reject_alert_key_journal_insert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.created_at = ${alertKey.folderMillis} THEN RAISE EXCEPTION 'injected 0064 journal failure'; END IF; RETURN NEW; END $$; + CREATE TRIGGER reject_alert_key_journal_insert BEFORE INSERT ON drizzle.__drizzle_migrations + FOR EACH ROW EXECUTE FUNCTION reject_alert_key_journal_insert()`); + await expect(release()).rejects.toThrow("injected 0064 journal failure"); + expect(await journal()).toEqual([ + { created_at: String(monitoring.folderMillis), hash: monitoring.hash }, + { created_at: String(feedback.folderMillis), hash: feedback.hash }, + ]); + expect(await alertKeyColumn()).toEqual([]); + expect(await alertKeyCheck(pool)).toEqual([]); + expect(await canaryIndex()).toEqual([]); + expect(commands).toEqual([]); + await pool.query("ALTER TABLE drizzle.__drizzle_migrations DISABLE TRIGGER reject_alert_key_journal_insert"); + await release(); expect(await journal()).toEqual([ { created_at: String(monitoring.folderMillis), hash: monitoring.hash }, { created_at: String(feedback.folderMillis), hash: feedback.hash }, + { created_at: String(alertKey.folderMillis), hash: alertKey.hash }, ]); + expect(await alertKeyColumn()).toEqual([{ column_name: "alert_key" }]); + expect(await alertKeyCheck(pool)).toEqual(UNVALIDATED_ALERT_KEY_CHECK); + const builtIndex = await canaryIndex(); + expect(builtIndex).toEqual([{ + definition: expect.stringContaining( + "USING btree (alert_key, event_type, alert_source_id) WHERE (alert_key IS NOT NULL)", + ), + valid: true, + }]); expect((await pool.query("SELECT id, mode FROM review_feedback_control")).rows) .toEqual([{ id: 1, mode: "disabled" }]); expect((await pool.query("SELECT column_name FROM information_schema.columns WHERE table_name='private_monitor_incidents' AND column_name='last_delivery_receipt'")).rows) @@ -205,10 +269,14 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => expect(inserted.rows).toEqual([{ status: "queued" }]); }); await release(); - expect(await journal()).toHaveLength(2); + expect(await journal()).toHaveLength(3); + expect(await alertKeyCheck(pool)).toEqual(UNVALIDATED_ALERT_KEY_CHECK); + expect(await canaryIndex()).toEqual(builtIndex); expect(commands).toEqual([ ["bun", "run", "hosted:verify-provider"], + ["bun", "run", "operational:indexes", "--managed-release"], ["bun", "run", "hosted:verify-provider"], + ["bun", "run", "operational:indexes", "--managed-release"], ]); }, 30_000); @@ -223,10 +291,13 @@ describeDatabase("managed release upgrade from monitoring migration 0061", () => const expected = [ { created_at: String(monitoring.folderMillis), hash: monitoring.hash }, { created_at: String(feedback.folderMillis), hash: feedback.hash }, + { created_at: String(alertKey.folderMillis), hash: alertKey.hash }, ]; expect(await journal()).toEqual(expected); + expect(await alertKeyCheck(cleanPool)).toEqual(UNVALIDATED_ALERT_KEY_CHECK); await release(); expect(await journal()).toEqual(expected); + expect(await alertKeyCheck(cleanPool)).toEqual(UNVALIDATED_ALERT_KEY_CHECK); expect((await cleanPool.query("SELECT id, mode FROM review_feedback_control")).rows) .toEqual([{ id: 1, mode: "disabled" }]); }, 30_000); @@ -251,7 +322,7 @@ for (const sourceRelease of COMPATIBLE_MANAGED_RELEASE_BOOTSTRAP_SHAS) { const migration = new Client({ connectionString: url.toString() }); await migration.connect(); for (const file of (await readdir(join(import.meta.dir, "..", "drizzle"))) - .filter((name) => /^\d{4}_.*\.sql$/.test(name) && !name.startsWith("0063_")) + .filter((name) => /^\d{4}_.*\.sql$/.test(name) && !name.startsWith("0064_")) .sort()) { const source = await readFile( join(import.meta.dir, "..", "drizzle", file), @@ -301,16 +372,17 @@ for (const sourceRelease of COMPATIBLE_MANAGED_RELEASE_BOOTSTRAP_SHAS) { }, 30_000); test("applies only the reviewed additive migration atomically without retiring the source", async () => { - const source = await readFile(join(import.meta.dir, "..", "drizzle", "0063_review_feedback_control.sql"), "utf8"); + const source = await readFile(join(import.meta.dir, "..", "drizzle", "0064_ilert_canary_alert_key.sql"), "utf8"); const latest = migrations.at(-1)!; const previous = migrations.at(-2)!; const capabilities = async () => (await pool.query("SELECT name FROM deployment_capabilities ORDER BY name")).rows; const before = await capabilities(); const apply = (dryRun = false, identities = migrations, sql = source) => - applyReviewedFeedbackControlMigration(pool, sourceRelease, targetRelease, + applyReviewedIlertCanaryAlertKeyMigration(pool, sourceRelease, targetRelease, COMPATIBLE_MANAGED_RELEASE_PROTOCOL, identities, sql, { dryRun }); const unchanged = async () => { - expect((await pool.query("SELECT to_regclass('public.review_feedback_control') AS relation")).rows[0].relation).toBeNull(); + expect((await pool.query("SELECT column_name FROM information_schema.columns WHERE table_name='ilert_alert_events' AND column_name='alert_key'")).rows).toEqual([]); + expect(await alertKeyCheck(pool)).toEqual([]); expect((await pool.query("SELECT count(*)::int AS count FROM drizzle.__drizzle_migrations WHERE created_at=$1", [latest.folderMillis])).rows[0].count).toBe(0); expect(await capabilities()).toEqual(before); }; @@ -371,9 +443,30 @@ for (const sourceRelease of COMPATIBLE_MANAGED_RELEASE_BOOTSTRAP_SHAS) { POSTIL_RELEASE_PROTOCOL: COMPATIBLE_MANAGED_RELEASE_PROTOCOL }, (command) => { commands.push([...command]); return { exited: Promise.resolve(0) }; }, undefined, async () => true); - expect(commands).toEqual([["bun", "run", "hosted:verify-provider"]]); + expect(commands).toEqual([ + ["bun", "run", "hosted:verify-provider"], + ["bun", "run", "operational:indexes", "--managed-release"], + ]); + expect(await alertKeyCheck(pool)).toEqual(UNVALIDATED_ALERT_KEY_CHECK); expect(await apply()).toBe(false); expect(await apply(true)).toBe(false); + expect(await alertKeyCheck(pool)).toEqual(UNVALIDATED_ALERT_KEY_CHECK); + // The deployed release keeps writing events without the new column, + // while a new writer cannot store an out-of-range key. + const event = (alertKey?: string) => pool.query( + `INSERT INTO ilert_alert_events + (event_id, alert_id, event_type, status, priority, summary, details, + alert_source_id, alert_source_name, report_time, occurred_at, + payload_sha256${alertKey === undefined ? "" : ", alert_key"}) + VALUES (gen_random_uuid(), '20000001', 'alert-created', 'PENDING', 'HIGH', 'Fixture alert', + '', 2269078, 'Postil production', now(), now(), repeat('a', 64)${alertKey === undefined ? "" : ", $1"}) + RETURNING alert_key`, + alertKey === undefined ? [] : [alertKey], + ); + expect((await event()).rows).toEqual([{ alert_key: null }]); + await expect(event("")).rejects.toMatchObject({ + code: "23514", constraint: "ilert_alert_events_alert_key_check", + }); expect(await capabilities()).toEqual(before); expect((await pool.query("SELECT id, mode FROM review_feedback_control")).rows).toEqual([{ id: 1, mode: "disabled" }]); expect((await pool.query("SELECT hash FROM drizzle.__drizzle_migrations WHERE created_at=$1", [latest.folderMillis])).rows).toEqual([{ hash: latest.hash }]); diff --git a/tests/env-validation.test.ts b/tests/env-validation.test.ts index 1fa6fe78..e5adf959 100644 --- a/tests/env-validation.test.ts +++ b/tests/env-validation.test.ts @@ -281,6 +281,8 @@ describe("web startup environment validation", () => { process.env.WORKER_HEARTBEAT_INTERVAL_MS = "30000"; process.env.POSTIL_MONITOR_ALERT_STATE_PATH = "/var/lib/postil-monitor/alert-state.json"; + const integrationKey = crypto.randomUUID(); + process.env.ILERT_INTEGRATION_KEY = integrationKey; process.env.OPENROUTER_MANAGEMENT_API_KEY = "management-test-key"; process.env.POSTIL_OPENROUTER_DEVELOPMENT_KEY_NAME = "development-test-key"; process.env.POSTIL_OPENROUTER_PRODUCTION_KEY_NAME = "production-test-key"; @@ -299,6 +301,13 @@ describe("web startup environment validation", () => { delete process.env.WORKER_HEARTBEAT_INTERVAL_MS; expect(() => validateEnv("monitor")).toThrow(/WORKER_HEARTBEAT_INTERVAL_MS/); process.env.WORKER_HEARTBEAT_INTERVAL_MS = "30000"; + // A self-hosted monitor without an alerting account still starts; the + // missing key surfaces on each delivery attempt instead. + delete process.env.ILERT_INTEGRATION_KEY; + expect(() => validateEnv("monitor")).not.toThrow(); + process.env.ILERT_INTEGRATION_KEY = " "; + expect(() => validateEnv("monitor")).not.toThrow(); + process.env.ILERT_INTEGRATION_KEY = integrationKey; delete process.env.OPENROUTER_MANAGEMENT_API_KEY; delete process.env.POSTIL_OPENROUTER_DEVELOPMENT_KEY_NAME; delete process.env.POSTIL_OPENROUTER_PRODUCTION_KEY_NAME; diff --git a/tests/github-approval-binding-migration.test.ts b/tests/github-approval-binding-migration.test.ts index fc4bac99..6a076c05 100644 --- a/tests/github-approval-binding-migration.test.ts +++ b/tests/github-approval-binding-migration.test.ts @@ -33,9 +33,9 @@ describeDb("GitHub approval binding migration", () => { const migrationsDirectory = join(import.meta.dir, "..", "drizzle"); const migrations = (await readdir(migrationsDirectory)) - .filter((file) => file.endsWith(".sql") && file < "0037_") + .filter((file) => file.endsWith(".sql")) .sort(); - for (const migration of migrations) { + for (const migration of migrations.filter((file) => file < "0037_")) { await applyMigration(client, join(migrationsDirectory, migration)); } @@ -86,6 +86,11 @@ describeDb("GitHub approval binding migration", () => { )).rows[0]!.id; await applyMigration(client, join(migrationsDirectory, "0037_github_approval_bindings.sql")); + for (const migration of migrations.filter( + (file) => file > "0037_github_approval_bindings.sql", + )) { + await applyMigration(client, join(migrationsDirectory, migration)); + } const operationalPool = new Pool({ connectionString: isolatedDatabaseUrl, max: 1 }); try { await ensureOperationalIndexes(operationalPool); diff --git a/tests/health.test.ts b/tests/health.test.ts index 31d3b497..f398a806 100644 --- a/tests/health.test.ts +++ b/tests/health.test.ts @@ -1,11 +1,21 @@ import { beforeEach, describe, expect, mock, test } from "bun:test"; import { spawnSync } from "node:child_process"; -import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { parse } from "yaml"; +import { CANARY_FINALIZER_BUDGETS } from "../scripts/reconcile-ilert-alert-stream"; + let queryCount = 0; let queryImpl: (text: string) => Promise; @@ -192,6 +202,736 @@ describe("/api/health/monitor", () => { }); describe("production monitor workflow", () => { + test("owns a bounded, accurately disclosed iLert webhook reconciliation canary", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: Record< + string, + { + if?: string; + name?: string; + needs?: string | string[]; + outputs?: Record; + "timeout-minutes"?: number; + steps?: Array<{ + "continue-on-error"?: boolean; + env?: Record; + id?: string; + if?: string; + name?: string; + run?: string; + "timeout-minutes"?: number; + uses?: string; + with?: Record; + }>; + } + >; + }; + expect(source).toContain("reconcile_alert_stream:"); + expect(source).toContain( + "description: Reconcile the iLert webhook action and send then resolve a HIGH-priority production test alert; it can invoke the escalation path; cannot be combined with test_alert", + ); + const architecture = await readFile( + new URL("../ARCHITECTURE.md", import.meta.url), + "utf8", + ); + expect(architecture).toContain( + "HIGH-priority production test alert that can invoke the escalation path", + ); + expect(architecture).toContain( + "A `cleaned` handoff means the\nprimary canary also proves the exact persisted `alert-resolved` receiver event.", + ); + expect(architecture).toContain( + "A cleaned handoff still performs this\nmanagement sweep to resolve a delayed Event API duplicate.", + ); + expect(source).toContain( + "but this job still sweeps iLert management inventory for a delayed Event", + ); + expect(source).toContain( + " test_alert:\n description: Send a HIGH-priority test alert only after production checks pass, then attempt to resolve it; it can invoke the escalation path; cannot be combined with reconcile_alert_stream", + ); + expect(source).toContain("POSTIL_ILERT_CANARY_RUN_ATTEMPT"); + const alertStream = workflow.jobs["alert-stream"]; + expect(alertStream?.needs).toEqual(["deployment-trust", "smoke", "release-recovery"]); + const setupAndSecretLoadBudgetMinutes = 5; + const runnerHeadroomMinutes = 8; + expect(alertStream?.name).toBe("Reconcile and verify iLert webhook delivery"); + expect(alertStream?.if).toBe( + "${{ always() && needs.deployment-trust.outputs.trusted == 'true' && inputs.reconcile_alert_stream == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }}", + ); + expect(alertStream?.outputs).toMatchObject({ + "alert-submitted": "${{ steps.canary.outputs.alert_submitted }}", + "producer-attempt": "${{ steps.canary.outputs.producer_attempt }}", + }); + expect(alertStream?.steps?.map((step) => step.name)).toContain( + "Preview iLert alert-stream reconciliation", + ); + expect(alertStream?.steps?.map((step) => step.name)).toContain( + "Reconcile and verify the attempt-specific iLert webhook canary", + ); + const preview = alertStream?.steps?.find( + (step) => step.name === "Preview iLert alert-stream reconciliation", + ); + const canary = alertStream?.steps?.find( + (step) => step.name === "Reconcile and verify the attempt-specific iLert webhook canary", + ); + const alertStreamSetupSteps = (alertStream?.steps ?? []).slice(0, 5); + expect(alertStreamSetupSteps.map((step) => step["timeout-minutes"])).toEqual([1, 1, 1, 1, 1]); + expect(alertStreamSetupSteps.reduce( + (total, step) => total + (step["timeout-minutes"] ?? 0), + 0, + )).toBe(setupAndSecretLoadBudgetMinutes); + expect(boundedStepMinutes(preview)).toBe(7); + expect(boundedStepMinutes(canary)).toBe(13); + expect(canary?.env).toMatchObject({ + POSTIL_ILERT_CANARY_RUN_ATTEMPT: "${{ github.run_attempt }}", + POSTIL_ILERT_CANARY_RUN_ID: "${{ github.run_id }}", + POSTIL_ILERT_RECEIVER_ORIGIN: "https://postil.dev", + }); + const inJobFinalizer = alertStream?.steps?.find( + (step) => step.name === "Resolve and stabilize the current iLert canary before job exit", + ); + const cleanupIndex = alertStream?.steps?.indexOf(inJobFinalizer!) ?? -1; + const canaryIndex = alertStream?.steps?.indexOf(canary!) ?? -1; + expect(cleanupIndex).toBeGreaterThan(canaryIndex); + expect(boundedStepMinutes(inJobFinalizer)).toBe(4); + expect(inJobFinalizer?.if).toBe( + "${{ always() && steps.canary.outcome != 'skipped' }}", + ); + expect(alertStream?.["timeout-minutes"]).toBeGreaterThanOrEqual( + setupAndSecretLoadBudgetMinutes + + (alertStream?.steps ?? []).reduce( + (total, step) => total + boundedStepMinutes(step), + 0, + ) + + runnerHeadroomMinutes, + ); + expect(alertStream?.["timeout-minutes"]! - ( + setupAndSecretLoadBudgetMinutes + + (alertStream?.steps ?? []).slice(0, cleanupIndex).reduce( + (total, step) => total + boundedStepMinutes(step), + 0, + ) + )).toBeGreaterThanOrEqual(boundedStepMinutes(inJobFinalizer) + runnerHeadroomMinutes); + expect(inJobFinalizer?.run).not.toContain(" --canary"); + expect(inJobFinalizer?.env).toMatchObject({ + POSTIL_ILERT_CANARY_ALERT_SUBMITTED: "${{ steps.canary.outputs.alert_submitted }}", + POSTIL_ILERT_CANARY_RUN_ATTEMPT: "${{ github.run_attempt }}", + POSTIL_ILERT_CANARY_SWEEP_ATTEMPT: "${{ github.run_attempt }}", + POSTIL_ILERT_CANARY_RUN_ID: "${{ github.run_id }}", + }); + const finalizer = workflow.jobs["alert-stream-finalize"]; + expect(finalizer?.needs).toEqual(["deployment-trust", "alert-stream"]); + expect(finalizer?.["timeout-minutes"]).toBe(11); + expect(finalizer?.if).toBe( + "${{ always() && needs.deployment-trust.outputs.trusted == 'true' && inputs.reconcile_alert_stream == true && needs.alert-stream.result != 'skipped' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }}", + ); + expect(finalizer?.name).toBe("Finalize iLert webhook canary cleanup"); + const finalizerCleanup = finalizer?.steps?.find( + (step) => step.name === "Resolve and stabilize the reconstructible iLert canary", + ); + expect(finalizerCleanup?.run).toContain("timeout 4m bun run scripts/reconcile-ilert-alert-stream.ts --finalize-canary"); + const finalizerCleanupIndex = finalizer?.steps?.indexOf(finalizerCleanup!) ?? -1; + const finalizerSetupSteps = (finalizer?.steps ?? []).slice(0, finalizerCleanupIndex); + const finalizerSetupBudgetMinutes = finalizerSetupSteps.reduce( + (total, step) => total + (step["timeout-minutes"] ?? 0), + 0, + ); + const finalizerHeadroomMinutes = 2; + expect(finalizerCleanupIndex).toBe((finalizer?.steps?.length ?? 0) - 1); + expect(finalizerSetupSteps).toHaveLength(4); + expect(finalizerSetupSteps.map((step) => step["timeout-minutes"])).toEqual([1, 1, 1, 1]); + expect(finalizerSetupBudgetMinutes).toBe(4); + expect(boundedStepMinutes(finalizerCleanup)).toBe(4); + expect( + finalizerSetupBudgetMinutes + + boundedStepMinutes(finalizerCleanup) + + finalizerHeadroomMinutes, + ).toBeLessThan(finalizer?.["timeout-minutes"] ?? 0); + expect(finalizer?.steps?.find( + (step) => step.name === "Resolve and stabilize the reconstructible iLert canary", + )?.env).toMatchObject({ + POSTIL_ILERT_CANARY_ALERT_SUBMITTED: + "${{ needs.alert-stream.outputs.alert-submitted }}", + POSTIL_ILERT_CANARY_RUN_ATTEMPT: + "${{ needs.alert-stream.outputs.producer-attempt || github.run_attempt }}", + POSTIL_ILERT_CANARY_SWEEP_ATTEMPT: "${{ github.run_attempt }}", + POSTIL_ILERT_CANARY_RUN_ID: "${{ github.run_id }}", + }); + const finalizerAttempt = finalizer?.steps?.find( + (step) => step.name === "Resolve and stabilize the reconstructible iLert canary", + )?.env?.POSTIL_ILERT_CANARY_RUN_ATTEMPT; + expect(resolveFinalizerAttempt(finalizerAttempt, "1", "2")).toBe("1"); + expect(resolveFinalizerAttempt(finalizerAttempt, "", "2")).toBe("2"); + for (const job of [alertStream, finalizer]) { + expect(job?.steps?.find((step) => step.uses?.startsWith("actions/checkout@"))?.with) + .toMatchObject({ ref: "${{ github.sha }}" }); + } + const expectScopedSecretLoads = ( + job: typeof alertStream, + expectedSecrets: string[], + ) => { + const scopedSecretLoads = job?.steps?.filter((step) => step.with?.["secret-name"]); + expect(scopedSecretLoads?.map((step) => step.with?.["secret-name"])).toEqual(expectedSecrets); + expect(scopedSecretLoads?.every((step) => + step.uses === "Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db" && + typeof step.with?.["secret-path"] === "string", + )).toBe(true); + }; + expectScopedSecretLoads(alertStream, ["ILERT_API_KEY", "ILERT_INTEGRATION_KEY", "POSTIL_ILERT_WEBHOOK_SECRET"]); + expectScopedSecretLoads(finalizer, ["ILERT_API_KEY", "ILERT_INTEGRATION_KEY"]); + for (const job of [alertStream, finalizer]) { + const managementLoad = job?.steps?.find((step) => + step.with?.["secret-name"] === "ILERT_API_KEY" + ); + expect(managementLoad?.with?.["identity-id"]) + .toBe("${{ secrets.INFISICAL_ILERT_MACHINE_IDENTITY_ID }}"); + } + expect(finalizer?.steps?.filter((step) => step.with?.["secret-name"]) + .every((step) => step.if === undefined)).toBe(true); + expect(workflow.jobs.notify?.if).toBe( + "${{ always() && needs.deployment-trust.outputs.trusted == 'true' && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || (inputs.test_alert == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success')) && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }}", + ); + const notifySecret = workflow.jobs.notify?.steps?.find( + (step) => step.name === "Load required iLert integration secret from Infisical", + ); + expect(notifySecret).toMatchObject({ + uses: "Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db", + with: { + "secret-name": "ILERT_INTEGRATION_KEY", + "secret-path": "/postil", + }, + }); + expect(notifySecret).not.toHaveProperty("continue-on-error"); + expect(workflow.jobs.notify?.steps?.find((step) => step.name === "Send ilert event")) + .toMatchObject({ + uses: "./.github/actions/ilert-event", + with: { "require-delivery": true }, + }); + expect(workflow.jobs.resolve?.if).toBe( + "${{ needs.deployment-trust.outputs.trusted == 'true' && needs.smoke.result == 'success' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') && (github.event_name != 'workflow_dispatch' || inputs.reconcile_alert_stream != true) && inputs.test_alert != true }}", + ); + }); + + test("finalizes only a successful manual test alert after notification settles", async () => { + const workflow = parse(await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + )) as { + jobs: Record; + }>; + }>; + }; + const notify = workflow.jobs.notify!; + const finalizer = workflow.jobs["resolve-test-alert"]!; + expect(finalizer.needs).toEqual(["deployment-trust", "smoke", "release-recovery", "notify"]); + expect(finalizer.if).toBe( + "${{ always() && needs.deployment-trust.outputs.trusted == 'true' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.test_alert == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success' }}", + ); + expect(finalizer.if).not.toContain("needs.notify.result == 'success'"); + expect(finalizer["timeout-minutes"]).toBe(5); + expect(finalizer.steps.map((step) => step["timeout-minutes"])).toEqual([1, 1, 2]); + const secretLoad = (job: typeof notify) => job.steps.find( + (step) => step.with?.["secret-name"] === "ILERT_INTEGRATION_KEY", + ); + expect(secretLoad(finalizer)).toEqual(secretLoad(notify)); + expect(finalizer.steps.filter((step) => step.with?.["secret-name"])).toHaveLength(1); + const notification = notify.steps.find((step) => step.name === "Send ilert event")!; + const resolution = finalizer.steps.find((step) => step.name === "Resolve ilert test alert")!; + expect(notification.with?.["alert-key"]).toContain("postil-production-monitor-test"); + expect(resolution).toMatchObject({ + uses: "./.github/actions/ilert-event", + with: { + "event-type": "RESOLVE", + "alert-key": "postil-production-monitor-test", + "require-delivery": true, + }, + }); + expect(resolution.with?.["alert-key"]).not.toBe("postil-production-monitor"); + expect(workflow.jobs.resolve!.if).toContain("inputs.test_alert != true"); + }); + + test("routes manual test alerts only when their own resolver can run", async () => { + const workflow = parse(await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + )) as { + jobs: Record }>; + }>; + }; + const alertKey = workflow.jobs.notify!.steps.find((step) => + step.name === "Send ilert event" + )!.with!["alert-key"]; + expect(alertKey).toBeDefined(); + for (const [smoke, release, notification, resolution, key] of [ + ["cancelled", "success", false, false, null], + ["failure", "success", true, false, "postil-production-monitor"], + ["success", "failure", true, false, "postil-release-recovery"], + ["success", "cancelled", true, false, "postil-release-recovery"], + ["cancelled", "failure", true, false, "postil-release-recovery"], + ["success", "success", true, true, "postil-production-monitor-test"], + ] as const) { + const context = { + needs: { + deploymentTrust: { outputs: { trusted: "true" } }, + smoke: { result: smoke }, + releaseRecovery: { result: release }, + }, + inputs: { test_alert: true }, + github: { event_name: "workflow_dispatch", ref: "refs/heads/main" }, + }; + const sends = evaluateMonitorExpression(workflow.jobs.notify!.if, context); + const resolves = evaluateMonitorExpression( + workflow.jobs["resolve-test-alert"]!.if, + context, + ); + expect(sends).toBe(notification); + expect(resolves).toBe(resolution); + expect(sends ? evaluateMonitorExpression(alertKey!, context) : null).toBe(key); + } + }); + + test("resolves an accepted test alert after its response is lost without closing production", async () => { + const workflow = parse(await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + )) as { + jobs: Record }> }>; + }; + const action = parse(await readFile( + new URL("../.github/actions/ilert-event/action.yml", import.meta.url), + "utf8", + )) as { runs: { steps: Array<{ run: string }> } }; + const notification = workflow.jobs.notify!.steps.find((step) => step.name === "Send ilert event")!; + const resolution = workflow.jobs["resolve-test-alert"]!.steps.find( + (step) => step.name === "Resolve ilert test alert", + )!; + const workingDirectory = mkdtempSync(join(tmpdir(), "postil-test-alert-")); + const eventsFile = join(workingDirectory, "events.ndjson"); + const integrationIdentity = randomUUID(); + const testKey = resolution.with!["alert-key"] as string; + const notificationKey = notification.with!["alert-key"] as string; + expect(notificationKey).toContain(testKey); + const curl = join(workingDirectory, "curl"); + writeFileSync(curl, `#!/usr/bin/env bash +set -euo pipefail +for argument in "$@"; do + if [[ "$argument" == @* ]]; then payload="\${argument#@}"; fi +done +jq -c . "$payload" >> "${eventsFile}" +if jq -e '.eventType == "ALERT"' "$payload" >/dev/null; then + printf '500' +else + printf '202' +fi +`); + chmodSync(curl, 0o700); + const runEvent = (eventType: string, summary: string, alertKey: string, required: boolean) => + spawnSync("bash", ["-c", action.runs.steps[0]!.run], { + cwd: workingDirectory, + env: { + ...process.env, + PATH: `${workingDirectory}:${process.env.PATH ?? ""}`, + GITHUB_STEP_SUMMARY: join(workingDirectory, "summary"), + RUNNER_TEMP: workingDirectory, + ILERT_INTEGRATION_KEY: integrationIdentity, + EVENT_TYPE: eventType, + SUMMARY: summary, + ALERT_KEY: alertKey, + DETAILS: "", + PRIORITY: "HIGH", + REQUIRE_DELIVERY: String(required), + }, + encoding: "utf8", + }); + try { + const alert = runEvent("ALERT", "Postil production monitor test alert", testKey, true); + expect(alert.status).toBe(1); + const resolved = runEvent( + String(resolution.with!["event-type"]), + String(resolution.with!["summary"]), + testKey, + resolution.with!["require-delivery"] === true, + ); + expect(resolved.status).toBe(0); + const events = (await readFile(eventsFile, "utf8")).trim().split("\n").map( + (line) => JSON.parse(line) as { integrationKey: string; eventType: string; alertKey: string }, + ); + expect(events.map(({ eventType, alertKey }) => ({ eventType, alertKey }))).toEqual([ + { eventType: "ALERT", alertKey: testKey }, + { eventType: "RESOLVE", alertKey: testKey }, + ]); + expect(events.every((event) => event.integrationKey === integrationIdentity)).toBe(true); + const openAlerts = new Set(["postil-production-monitor"]); + for (const event of events) { + if (event.eventType === "ALERT") openAlerts.add(event.alertKey); + if (event.eventType === "RESOLVE") openAlerts.delete(event.alertKey); + } + expect([...openAlerts]).toEqual(["postil-production-monitor"]); + } finally { + rmSync(workingDirectory, { recursive: true, force: true }); + } + }); + + test("fits finalizer cleanup inside GitHub's cancellation grace", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: Record }>; + }; + const cleanupCommands = ["alert-stream", "alert-stream-finalize"].map((jobName) => + workflow.jobs[jobName]?.steps.find((step) => + step.run?.includes("--finalize-canary") + )?.run + ); + expect(CANARY_FINALIZER_BUDGETS.discoveryMs + + CANARY_FINALIZER_BUDGETS.terminalValidationMs + + CANARY_FINALIZER_BUDGETS.cleanupMs).toBe(CANARY_FINALIZER_BUDGETS.operationMs); + expect(CANARY_FINALIZER_BUDGETS.shellMs - CANARY_FINALIZER_BUDGETS.operationMs) + .toBeGreaterThanOrEqual(30_000); + expect(CANARY_FINALIZER_BUDGETS.cancellationGraceMs - CANARY_FINALIZER_BUDGETS.shellMs) + .toBeGreaterThanOrEqual(60_000); + expect(cleanupCommands).toEqual([ + "timeout 4m bun run scripts/reconcile-ilert-alert-stream.ts --finalize-canary", + "timeout 4m bun run scripts/reconcile-ilert-alert-stream.ts --finalize-canary", + ]); + expect(boundedStepMinutes({ run: cleanupCommands[0] }) * 60_000) + .toBe(CANARY_FINALIZER_BUDGETS.shellMs); + }); + + test("prevents non-main manual dispatches from loading production credentials or local actions", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: Record; + uses?: string; + with?: Record; + }>; + }>; + }; + const protectedJobs = Object.entries(workflow.jobs).filter(([, job]) => + JSON.stringify(job).includes("secrets.") || + job.steps?.some((step) => + step.uses?.startsWith("Infisical/secrets-action@") || + step.uses === "./.github/actions/ilert-event" + ) + ); + expect(protectedJobs.map(([name]) => name).sort()).toEqual([ + "alert-stream", + "alert-stream-finalize", + "canary-orphan-sweep", + "notify", + "release-recovery", + "resolve", + "resolve-release-recovery", + "resolve-test-alert", + "smoke", + ]); + for (const [name, job] of protectedJobs) { + expect(job.if).toContain("github.ref == 'refs/heads/main'"); + if (name !== "resolve-test-alert") { + expect(job.if).toContain("github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main'"); + } + } + + const deploySource = await readFile( + new URL("../.github/workflows/deploy.yml", import.meta.url), + "utf8", + ); + const deploy = parse(deploySource) as { + on: Record; + jobs: Record; + }; + expect(Object.keys(deploy.on)).toEqual(["workflow_dispatch"]); + expect(Object.keys(deploy.jobs)).toEqual(["deploy"]); + expect(deploy.jobs.deploy!.if).toContain("github.event_name == 'workflow_dispatch'"); + }); + + test("accepts deploy workflow runs only from the main deployment branch", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + on: { + workflow_run: { + branches: string[]; + types: string[]; + workflows: string[]; + }; + }; + }; + expect(workflow.on.workflow_run).toEqual({ + workflows: ["deploy"], + types: ["completed"], + branches: ["main"], + }); + }); + + test("gives every required iLert delivery a bounded end-to-end budget", async () => { + const [workflowSource, actionSource] = await Promise.all([ + readFile(new URL("../.github/workflows/production-monitor.yml", import.meta.url), "utf8"), + readFile(new URL("../.github/actions/ilert-event/action.yml", import.meta.url), "utf8"), + ]); + const workflow = parse(workflowSource) as { + jobs: Record; + }>; + }>; + }; + const retryMaxTime = /--retry-max-time\s+(\d+)/u.exec(actionSource); + const requestMaxTime = /--max-time\s+(\d+)/u.exec(actionSource); + const connectTimeout = /--connect-timeout\s+(\d+)/u.exec(actionSource); + const retryDelay = /--retry-delay\s+(\d+)/u.exec(actionSource); + const retryCount = /--retry\s+(\d+)/u.exec(actionSource); + expect(retryMaxTime?.[1]).toBeDefined(); + expect(requestMaxTime?.[1]).toBeDefined(); + expect(connectTimeout?.[1]).toBeDefined(); + expect(retryDelay?.[1]).toBeDefined(); + expect(retryCount?.[1]).toBeDefined(); + const requestMaxSeconds = Number(requestMaxTime?.[1]); + const retryMaxSeconds = Number(retryMaxTime?.[1]); + const retryDelaySeconds = Number(retryDelay?.[1]); + const retryAttempts = Number(retryCount?.[1]); + expect(Number(connectTimeout?.[1])).toBeLessThanOrEqual(requestMaxSeconds); + expect(retryMaxSeconds).toBeLessThanOrEqual(45); + expect(retryMaxSeconds).toBeGreaterThan(retryDelaySeconds * retryAttempts); + + const requiredDeliveries = Object.entries(workflow.jobs).flatMap(([name, job]) => + job.steps + .filter((step) => + step.uses === "./.github/actions/ilert-event" && + step.with?.["require-delivery"] === true + ) + .map((delivery) => ({ delivery, job, name })) + ); + expect(requiredDeliveries.map(({ name }) => name).sort()).toEqual([ + "notify", + "resolve", + "resolve-release-recovery", + "resolve-test-alert", + ]); + + // retry-max-time includes failed requests and retry delays, but curl may + // start one final max-time request immediately before that timer expires. + const curlWorstCaseSeconds = retryMaxSeconds + requestMaxSeconds; + const shellOverheadSeconds = 15; + const minimumDeliveryHeadroomSeconds = 30; + for (const { delivery, job } of requiredDeliveries) { + expect(delivery?.with?.["require-delivery"]).toBe(true); + expect(delivery["timeout-minutes"]! * 60 - curlWorstCaseSeconds - shellOverheadSeconds) + .toBeGreaterThanOrEqual(minimumDeliveryHeadroomSeconds); + expect(job.steps.every((step) => (step["timeout-minutes"] ?? 0) > 0)).toBe(true); + const totalStepBudgetMinutes = job.steps.reduce( + (total, step) => total + step["timeout-minutes"]!, + 0, + ); + expect(job["timeout-minutes"] - totalStepBudgetMinutes).toBeGreaterThanOrEqual(1); + } + }); + + test("schedules a bounded management-only sweep for interrupted canary cleanup", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: { + "canary-orphan-sweep": { + if: string; + "timeout-minutes": number; + steps: Array<{ + env?: Record; + name?: string; + "timeout-minutes"?: number; + uses?: string; + with?: Record; + run?: string; + }>; + }; + }; + }; + const sweep = workflow.jobs["canary-orphan-sweep"]; + expect(sweep.if).toBe( + "${{ needs.deployment-trust.outputs.trusted == 'true' && vars.POSTIL_ILERT_CANARY_SWEEP_ENABLED == 'true' && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.reconcile_alert_stream != true && inputs.test_alert != true)) && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }}", + ); + expect(sweep["timeout-minutes"]).toBe(21); + const managementLoad = sweep.steps.find( + (step) => step.name === "Load iLert management secret from Infisical", + ); + expect(managementLoad).toMatchObject({ + uses: "Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db", + with: { + "identity-id": "${{ secrets.INFISICAL_ILERT_MACHINE_IDENTITY_ID }}", + "secret-name": "ILERT_API_KEY", + "secret-path": "/", + }, + }); + expect(sweep.steps.some((step) => step.with?.["secret-name"] === "ILERT_INTEGRATION_KEY")) + .toBe(false); + const recovery = sweep.steps.find( + (step) => step.name === "Resolve and verify orphaned iLert canaries", + ); + expect(recovery?.run).toBe( + "timeout 14m bun run scripts/reconcile-ilert-alert-stream.ts --sweep-canary-orphans", + ); + expect(recovery?.env).toEqual({ + POSTIL_ILERT_ALERT_SOURCE_ID: "${{ vars.POSTIL_ILERT_ALERT_SOURCE_ID }}", + }); + }); + + test("resolves a production failure after manual success and scheduled success", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: { + resolve: { + if: string; + steps: Array<{ + "continue-on-error"?: boolean; + if?: string; + id?: string; + name?: string; + uses?: string; + with?: Record; + }>; + }; + }; + }; + const resolve = workflow.jobs.resolve; + expect(resolve.if).toBe( + "${{ needs.deployment-trust.outputs.trusted == 'true' && needs.smoke.result == 'success' && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') && (github.event_name != 'workflow_dispatch' || inputs.reconcile_alert_stream != true) && inputs.test_alert != true }}", + ); + expect(resolve.steps.some((step) => step.name === "Check whether the previous run failed")).toBe(false); + expect(resolve.steps.find((step) => step.name === "Load iLert integration secret from Infisical")).toMatchObject({ + uses: "Infisical/secrets-action@6cd3f7c0e4cc0d2395ee4ef414eb6eeb5d3e73db", + with: { + "secret-name": "ILERT_INTEGRATION_KEY", + "secret-path": "/postil", + }, + }); + expect(resolve.steps.find((step) => step.name === "Load iLert integration secret from Infisical")) + .not.toHaveProperty("continue-on-error"); + expect(resolve.steps.find((step) => step.name === "Resolve ilert alert")).toMatchObject({ + uses: "./.github/actions/ilert-event", + with: { "require-delivery": true }, + }); + }); + + test("keeps release-recovery resolution delivery-required", async () => { + const source = await readFile( + new URL("../.github/workflows/production-monitor.yml", import.meta.url), + "utf8", + ); + const workflow = parse(source) as { + jobs: { + "resolve-release-recovery": { + steps: Array<{ + name?: string; + uses?: string; + with?: Record; + }>; + }; + }; + }; + expect(workflow.jobs["resolve-release-recovery"].steps.find( + (step) => step.name === "Resolve ilert release recovery alert", + )).toMatchObject({ + uses: "./.github/actions/ilert-event", + with: { "require-delivery": true }, + }); + }); + + test("keeps the iLert management credential outside application environment files", async () => { + const [compose, environmentExample] = await Promise.all([ + readFile(new URL("../docker-compose.yml", import.meta.url), "utf8"), + readFile(new URL("../.env.example", import.meta.url), "utf8"), + ]); + expect(environmentExample).not.toContain("ILERT_API_KEY="); + expect(compose).not.toContain("ILERT_API_KEY"); + }); + + test("routine iLert resolve safely no-ops when its optional key is absent", async () => { + const source = await readFile( + new URL("../.github/actions/ilert-event/action.yml", import.meta.url), + "utf8", + ); + const action = parse(source) as { + runs: { steps: Array<{ name?: string; run?: string }> }; + }; + const run = action.runs.steps.find((step) => step.name === "Send event")?.run; + expect(run).toBeDefined(); + const result = spawnSync("bash", ["-c", run!], { + encoding: "utf8", + env: { + ALERT_KEY: "test-alert", + DETAILS: "", + EVENT_TYPE: "RESOLVE", + NODE_ENV: "test", + PATH: process.env.PATH, + PRIORITY: "HIGH", + REQUIRE_DELIVERY: "false", + SUMMARY: "test resolve", + } as NodeJS.ProcessEnv, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("nothing to resolve"); + }); + + test("production monitor ALERT delivery fails when the iLert integration key is unavailable", async () => { + const source = await readFile( + new URL("../.github/actions/ilert-event/action.yml", import.meta.url), + "utf8", + ); + const action = parse(source) as { + runs: { steps: Array<{ name?: string; run?: string }> }; + }; + const run = action.runs.steps.find((step) => step.name === "Send event")?.run; + expect(run).toBeDefined(); + const result = spawnSync("bash", ["-c", run!], { + encoding: "utf8", + env: { + ALERT_KEY: "test-alert", + DETAILS: "", + EVENT_TYPE: "ALERT", + NODE_ENV: "test", + PATH: process.env.PATH, + PRIORITY: "HIGH", + REQUIRE_DELIVERY: "true", + SUMMARY: "test alert", + } as NodeJS.ProcessEnv, + }); + expect(result.status).toBe(1); + expect(result.stdout).toContain("ILERT_INTEGRATION_KEY is unset"); + }); + test("requires notification delivery and preserves optional events", async () => { const workflow = parse( await readFile( @@ -219,7 +959,7 @@ describe("production monitor workflow", () => { (step) => step.name === "Send ilert event", ); expect(workflow.jobs.notify.if).toBe( - "${{ always() && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || inputs.test_alert == true) }}", + "${{ always() && needs.deployment-trust.outputs.trusted == 'true' && (needs.smoke.result == 'failure' || needs.release-recovery.result == 'failure' || needs.release-recovery.result == 'cancelled' || (inputs.test_alert == true && needs.smoke.result == 'success' && needs.release-recovery.result == 'success')) && (github.event_name != 'workflow_dispatch' || github.ref == 'refs/heads/main') }}", ); expect(notification?.with?.["require-delivery"]).toBe(true); for (const [steps, name] of [ @@ -228,7 +968,7 @@ describe("production monitor workflow", () => { ] as const) { const resolveStep = steps.find((step) => step.name === name); expect(resolveStep).toBeDefined(); - expect(resolveStep?.with?.["require-delivery"]).toBeUndefined(); + expect(resolveStep?.with?.["require-delivery"]).toBe(true); } const action = parse( @@ -493,3 +1233,37 @@ exit "${"${stale_found}"}" } }); }); + +function evaluateMonitorExpression( + expression: string, + context: { needs: unknown; inputs: unknown; github: unknown }, +): unknown { + const body = /^\$\{\{\s*(.*?)\s*\}\}$/u.exec(expression)?.[1]; + if (!body || !/^[a-z0-9_.\s'()&|=!/\-]+$/iu.test(body)) { + throw new Error("Unsupported monitor expression"); + } + const javascript = body + .replaceAll("needs.deployment-trust", "needs.deploymentTrust") + .replaceAll("needs.release-recovery", "needs.releaseRecovery"); + const evaluate = new Function( + "needs", "inputs", "github", "always", `return (${javascript});`, + ) as (needs: unknown, inputs: unknown, github: unknown, always: () => boolean) => unknown; + return evaluate(context.needs, context.inputs, context.github, () => true); +} + +function resolveFinalizerAttempt( + expression: string | undefined, + producerAttempt: string, + workflowAttempt: string, +): string { + if (expression !== "${{ needs.alert-stream.outputs.producer-attempt || github.run_attempt }}") { + throw new Error("finalizer attempt handoff is not recoverable"); + } + return producerAttempt || workflowAttempt; +} + +function boundedStepMinutes(step: { run?: string } | undefined): number { + const match = /^timeout (\d+)m\b/u.exec(step?.run ?? ""); + if (!match) return 0; + return Number(match[1]); +} diff --git a/tests/ilert-alert-stream.test.ts b/tests/ilert-alert-stream.test.ts index 51d06653..8de686fb 100644 --- a/tests/ilert-alert-stream.test.ts +++ b/tests/ilert-alert-stream.test.ts @@ -5,11 +5,12 @@ import { expect, test, } from "bun:test"; -import { createHash } from "node:crypto"; +import { createHash, randomBytes, randomInt } from "node:crypto"; import { drizzle } from "drizzle-orm/node-postgres"; import { Client, type Notification, type Pool } from "pg"; +import { ensureOperationalIndexes } from "../scripts/ensure-operational-indexes"; import { closeDb, schema, type Database } from "@/lib/db"; import { authorizeIlertAlertStream, @@ -23,13 +24,14 @@ import { } from "./ephemeral-database"; const describeDb = process.env.POSTIL_TEST_DATABASE_URL ? describe : describe.skip; -const TEST_WEBHOOK_SECRET = "test-ilert-webhook-password-32-bytes"; +const TEST_WEBHOOK_SECRET = randomBytes(32).toString("base64url"); +const TEST_CANARY_KEY = `postil-ilert-webhook-canary-${randomInt(1_000_000_000, 10_000_000_000)}-2`; const OPERATOR_GITHUB_ID = 991_001; const CUSTOMER_ADMIN_GITHUB_ID = 991_002; -const OPERATOR_TOKEN = `pcli_${"a".repeat(43)}`; -const CUSTOMER_ADMIN_TOKEN = `pcli_${"b".repeat(43)}`; -const LEGACY_OPERATOR_TOKEN = `pcli_${"c".repeat(43)}`; -const ROTATING_OPERATOR_TOKEN = `pcli_${"d".repeat(43)}`; +const OPERATOR_TOKEN = `pcli_${randomBytes(32).toString("base64url")}`; +const CUSTOMER_ADMIN_TOKEN = `pcli_${randomBytes(32).toString("base64url")}`; +const LEGACY_OPERATOR_TOKEN = `pcli_${randomBytes(32).toString("base64url")}`; +const ROTATING_OPERATOR_TOKEN = `pcli_${randomBytes(32).toString("base64url")}`; describeDb("iLert durable operator alert stream", () => { let ephemeral: EphemeralDatabase; @@ -107,6 +109,42 @@ describeDb("iLert durable operator alert stream", () => { restoreEnvironment("POSTIL_ILERT_WEBHOOK_SECRET", originalWebhookSecret); }, 30_000); + test("installs the partial canary observation index without requiring legacy keys", async () => { + const absent = await pool.query<{ index_name: string | null }>( + `SELECT to_regclass( + 'public.ilert_alert_events_canary_observation_idx' + )::text AS index_name`, + ); + expect(absent.rows).toEqual([{ index_name: null }]); + + const installed = await ensureOperationalIndexes(pool); + expect(installed).toContain("ilert_alert_events_canary_observation_idx"); + + const index = await pool.query<{ definition: string; predicate: string | null }>( + `SELECT pg_get_indexdef(indexrelid) AS definition, + pg_get_expr(indpred, indrelid) AS predicate + FROM pg_index + WHERE indexrelid = to_regclass( + 'public.ilert_alert_events_canary_observation_idx' + )`, + ); + + expect(index.rows).toEqual([{ + definition: expect.stringContaining( + "USING btree (alert_key, event_type, alert_source_id)", + ), + predicate: "(alert_key IS NOT NULL)", + }]); + const column = await pool.query<{ is_nullable: string }>( + `SELECT is_nullable + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'ilert_alert_events' + AND column_name = 'alert_key'`, + ); + expect(column.rows).toEqual([{ is_nullable: "YES" }]); + }); + test("deduplicates event ids and notifies only after the row commits", async () => { const listener = new Client({ connectionString: ephemeral.url }); await listener.connect(); @@ -156,9 +194,50 @@ describeDb("iLert durable operator alert stream", () => { sequence, eventId: "7b21f505-bd0f-49a2-bf8f-f238919b23fc", alertId: "12797430", + alertKey: null, }); }); + test("accepts an empty alert key and stores it as absent", async () => { + const eventId = "0f0e9c5e-6d0b-4c57-9a55-3f8f2f3f6a10"; + const response = await postWebhook(eventFixture({ alertKey: "", eventId })); + expect(response.status).toBe(202); + const stored = await pool.query<{ alert_key: string | null }>( + "SELECT alert_key FROM ilert_alert_events WHERE event_id = $1", + [eventId], + ); + expect(stored.rows).toEqual([{ alert_key: null }]); + }); + + test("observes only exact persisted canary lifecycle events", async () => { + const created = await postWebhook(eventFixture({ + alertKey: TEST_CANARY_KEY, + eventId: "f57d2564-908d-40d1-89cc-bc60a0e816ec", + })); + expect(created.status).toBe(202); + + const { GET } = await import("@/app/api/webhooks/ilert/route"); + const observe = (eventType: "alert-created" | "alert-resolved") => GET( + new Request( + `https://postil.dev/api/webhooks/ilert?alertKey=${TEST_CANARY_KEY}&eventType=${eventType}&sourceId=2269078`, + { headers: { authorization: basicAuthorization() } }, + ), + ); + expect(await (await observe("alert-created")).json()).toEqual({ received: true }); + expect(await (await observe("alert-resolved")).json()).toEqual({ received: false }); + + const resolved = await postWebhook(eventFixture({ + alertKey: TEST_CANARY_KEY, + eventId: "18da024f-9d34-4a3c-8fd7-a5919b99c074", + eventType: "alert-resolved", + status: "RESOLVED", + })); + expect(resolved.status).toBe(202); + const resolution = await observe("alert-resolved"); + expect(resolution.headers.get("cache-control")).toBe("no-store"); + expect(await resolution.json()).toEqual({ received: true }); + }); + test("requires both a renewable CLI token and the operator GitHub allowlist", async () => { expect( await authorizeIlertAlertStream(db, `Bearer ${OPERATOR_TOKEN}`), @@ -265,6 +344,7 @@ describeDb("iLert durable operator alert stream", () => { ); const names = columns.rows.map((row) => row.column_name); expect(names).toContain("payload_sha256"); + expect(names).toContain("alert_key"); expect(names).not.toContain("payload"); expect(names).not.toContain("raw_payload"); expect(names).not.toContain("authorization"); diff --git a/tests/ilert-alerts.test.ts b/tests/ilert-alerts.test.ts index e2767ad2..9c6faac5 100644 --- a/tests/ilert-alerts.test.ts +++ b/tests/ilert-alerts.test.ts @@ -1,14 +1,19 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, mock, test } from "bun:test"; +import { randomBytes, randomInt } from "node:crypto"; import { readFile } from "node:fs/promises"; import { join } from "node:path"; +import { type SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core/dialect"; import { configuredIlertWebhookSecret, formatIlertAlertSseEvent, + hasIlertCanaryAlertEvent, ILERT_WEBHOOK_MAX_BODY_BYTES, ILERT_WEBHOOK_USERNAME, ilertAlertStreamDatabaseUrl, isApplicationJson, + isIlertCanaryAlertKey, parseIlertLastEventId, parseIlertWebhookBody, verifyIlertWebhookAuthorization, @@ -16,9 +21,33 @@ import { } from "@/lib/ilert-alerts"; const ORIGINAL_WEBHOOK_SECRET = process.env.POSTIL_ILERT_WEBHOOK_SECRET; -const TEST_WEBHOOK_SECRET = "test-ilert-webhook-password-32-bytes"; +const TEST_WEBHOOK_SECRET = randomBytes(32).toString("base64url"); +const CANARY_ATTEMPT = randomInt(1_000_000_000, 10_000_000_000); +const CANARY_PREFIX = "postil-ilert-webhook-canary-"; +const TEST_CANARY_KEY = `${CANARY_PREFIX}${CANARY_ATTEMPT}-2`; +let observationResult: () => Promise> = async () => []; +let databaseAccessCount = 0; + +mock.module("@/lib/db", () => ({ + getDb: () => { + databaseAccessCount += 1; + return { + select: () => ({ + from: () => ({ + where: () => ({ limit: observationResult }), + }), + }), + }; + }, +})); + +mock.module("@/lib/server-observability", () => ({ + reportOperationalFailure: () => undefined, +})); afterEach(() => { + observationResult = async () => []; + databaseAccessCount = 0; if (ORIGINAL_WEBHOOK_SECRET === undefined) { delete process.env.POSTIL_ILERT_WEBHOOK_SECRET; } else { @@ -31,9 +60,8 @@ describe("iLert webhook input", () => { expect(configuredIlertWebhookSecret(undefined)).toBeNull(); expect(configuredIlertWebhookSecret("x".repeat(31))).toBeNull(); expect(configuredIlertWebhookSecret("x".repeat(32))).toBeNull(); - expect(configuredIlertWebhookSecret("0123456789abcdef".repeat(2))).toBe( - "0123456789abcdef".repeat(2), - ); + const validSecret = randomBytes(32).toString("base64url"); + expect(configuredIlertWebhookSecret(validSecret)).toBe(validSecret); expect(configuredIlertWebhookSecret(`x${"y".repeat(31)}\0`)).toBeNull(); expect(configuredIlertWebhookSecret("x".repeat(513))).toBeNull(); }); @@ -71,6 +99,14 @@ describe("iLert webhook input", () => { expect(isApplicationJson(null)).toBe(false); }); + test("accepts only exact attempt-specific canary keys", () => { + expect(isIlertCanaryAlertKey(TEST_CANARY_KEY)).toBe(true); + expect(isIlertCanaryAlertKey(`${CANARY_PREFIX}${CANARY_ATTEMPT}-51`)).toBe(true); + expect(isIlertCanaryAlertKey("routine-alert-key")).toBe(false); + expect(isIlertCanaryAlertKey(`${CANARY_PREFIX}${CANARY_ATTEMPT}-52`)).toBe(false); + expect(isIlertCanaryAlertKey(`${CANARY_PREFIX}${"9".repeat(16)}-1`)).toBe(false); + }); + test("accepts bounded official alert extensions and rejects non-alert events", () => { expect(parseIlertWebhookBody(eventBody())).toMatchObject({ id: "12797430", @@ -89,6 +125,15 @@ describe("iLert webhook input", () => { incident.eventType = "incident-created"; expect(parseIlertWebhookBody(jsonBody(incident))).toBeNull(); + // An empty alert key stays a deliverable event; storage records it as absent. + const withEmptyKey = eventFixture() as Record; + withEmptyKey.alertKey = ""; + expect(parseIlertWebhookBody(jsonBody(withEmptyKey))).toMatchObject({ + id: "12797430", + alertKey: "", + }); + withEmptyKey.alertKey = "x".repeat(513); + expect(parseIlertWebhookBody(jsonBody(withEmptyKey))).toBeNull(); }); test("accepts the configured creation, escalation, raised, and lifecycle events", () => { @@ -175,9 +220,122 @@ describe("iLert webhook input", () => { ), ).toHaveProperty("status", 413); }); + + test("the route preflights receiver credentials without reading alert state", async () => { + const { GET } = await import("@/app/api/webhooks/ilert/route"); + process.env.POSTIL_ILERT_WEBHOOK_SECRET = TEST_WEBHOOK_SECRET; + + expect(await GET(new Request("https://postil.dev/api/webhooks/ilert"))).toHaveProperty( + "status", + 401, + ); + expect(await GET(new Request("https://postil.dev/api/webhooks/ilert", { + headers: { authorization: validAuthorization() }, + }))).toHaveProperty("status", 204); + expect(await GET(new Request( + "https://postil.dev/api/webhooks/ilert?alertId=1&eventType=alert-comment-added&sourceId=2", + { headers: { authorization: validAuthorization() } }, + ))).toHaveProperty("status", 400); + expect(await GET(new Request( + "https://postil.dev/api/webhooks/ilert?alertKey=routine-alert&eventType=alert-created&sourceId=2", + { headers: { authorization: validAuthorization() } }, + ))).toHaveProperty("status", 400); + }); + + test("the route returns an authenticated exact receiver-event observation", async () => { + const { GET } = await import("@/app/api/webhooks/ilert/route"); + process.env.POSTIL_ILERT_WEBHOOK_SECRET = TEST_WEBHOOK_SECRET; + observationResult = async () => [{ sequence: 1n }]; + + const response = await GET(new Request( + `https://postil.dev/api/webhooks/ilert?alertKey=${TEST_CANARY_KEY}&eventType=alert-created&sourceId=2269078`, + { headers: { authorization: validAuthorization() } }, + )); + + expect(response.status).toBe(200); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(await response.json()).toEqual({ received: true }); + }); + + test("rejects source ids above PostgreSQL bigint before database access", async () => { + const { GET } = await import("@/app/api/webhooks/ilert/route"); + process.env.POSTIL_ILERT_WEBHOOK_SECRET = TEST_WEBHOOK_SECRET; + + const response = await GET(new Request( + `https://postil.dev/api/webhooks/ilert?alertKey=${TEST_CANARY_KEY}&eventType=alert-created&sourceId=9223372036854775808`, + { headers: { authorization: validAuthorization() } }, + )); + + expect(response.status).toBe(400); + expect(await response.json()).toEqual({ error: "invalid canary observation" }); + expect(databaseAccessCount).toBe(0); + }); + + test("the route returns 503 when the receiver-event read fails", async () => { + const { GET } = await import("@/app/api/webhooks/ilert/route"); + process.env.POSTIL_ILERT_WEBHOOK_SECRET = TEST_WEBHOOK_SECRET; + observationResult = async () => { + throw new Error("database unavailable"); + }; + + const response = await GET(new Request( + `https://postil.dev/api/webhooks/ilert?alertKey=${TEST_CANARY_KEY}&eventType=alert-resolved&sourceId=2269078`, + { headers: { authorization: validAuthorization() } }, + )); + + expect(response.status).toBe(503); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(response.headers.get("retry-after")).toBe("5"); + expect(await response.json()).toEqual({ error: "webhook observation unavailable" }); + }); }); describe("iLert operator stream protocol", () => { + test("checks exact canary key, event type, and source persistence", async () => { + const events = [ + { alertKey: TEST_CANARY_KEY, eventType: "alert-resolved", alertSourceId: 2269078n, sequence: 1n }, + { alertKey: `${CANARY_PREFIX}${CANARY_ATTEMPT + 1}-2`, eventType: "alert-resolved", alertSourceId: 2269078n, sequence: 2n }, + { alertKey: TEST_CANARY_KEY, eventType: "alert-created", alertSourceId: 2269079n, sequence: 3n }, + { alertKey: TEST_CANARY_KEY, eventType: "alert-resolved", alertSourceId: 2269079n, sequence: 4n }, + ]; + const dialect = new PgDialect(); + const expectedPredicates = [ + [TEST_CANARY_KEY, "alert-resolved", 2269078n], + [TEST_CANARY_KEY, "alert-created", 2269078n], + ]; + let predicateCalls = 0; + const db = { + select() { + return { + from: () => ({ + where: (predicate: SQL) => ({ + limit: async () => { + const query = dialect.sqlToQuery(predicate); + expect(query.sql).toContain('"ilert_alert_events"."alert_key" = $1'); + expect(query.sql).not.toContain('"ilert_alert_events"."alert_id"'); + expect(query.sql).toContain('"ilert_alert_events"."event_type" = $2'); + expect(query.sql).toContain('"ilert_alert_events"."alert_source_id" = $3'); + const [alertKey, eventType, alertSourceId] = query.params; + expect([alertKey, eventType, alertSourceId]).toEqual( + expectedPredicates[predicateCalls++]!, + ); + return events + .filter((event) => + event.alertKey === alertKey && + event.eventType === eventType && + event.alertSourceId === alertSourceId + ) + .map(({ sequence }) => ({ sequence })); + }, + }), + }), + }; + }, + } as never; + expect(await hasIlertCanaryAlertEvent(db, TEST_CANARY_KEY, "alert-resolved", 2269078n)).toBe(true); + expect(await hasIlertCanaryAlertEvent(db, TEST_CANARY_KEY, "alert-created", 2269078n)).toBe(false); + }); + test("preserves runtime secret metadata without credential staging", async () => { const deploy = await readFile( join(import.meta.dir, "..", ".github", "workflows", "deploy.yml"), @@ -196,17 +354,27 @@ describe("iLert operator stream protocol", () => { }); test("uses a session endpoint for Supabase LISTEN connections", () => { - expect(ilertAlertStreamDatabaseUrl( - "postgresql://user:password@aws-0-eu.pooler.supabase.com:6543/postgres?pgbouncer=true", - )).toBe( - "postgresql://user:password@aws-0-eu.pooler.supabase.com:5432/postgres", + const username = randomBytes(8).toString("hex"); + const password = randomBytes(16).toString("hex"); + const pooler = new URL("postgresql://aws-0-eu.pooler.supabase.com:6543/postgres?pgbouncer=true"); + pooler.username = username; + pooler.password = password; + const session = new URL(pooler); + session.port = "5432"; + session.searchParams.delete("pgbouncer"); + expect(ilertAlertStreamDatabaseUrl(pooler.toString())).toBe(session.toString()); + + const direct = new URL("postgresql://database.internal:5432/postil"); + direct.username = username; + direct.password = password; + expect(ilertAlertStreamDatabaseUrl(direct.toString())).toBe(direct.toString()); + + const unsupported = new URL("postgresql://unknown.example:6543/postil"); + unsupported.username = username; + unsupported.password = password; + expect(() => ilertAlertStreamDatabaseUrl(unsupported.toString())).toThrow( + "requires a session-capable database endpoint", ); - expect(ilertAlertStreamDatabaseUrl( - "postgresql://user:password@database.internal:5432/postil", - )).toBe("postgresql://user:password@database.internal:5432/postil"); - expect(() => ilertAlertStreamDatabaseUrl( - "postgresql://user:password@unknown.example:6543/postil", - )).toThrow("requires a session-capable database endpoint"); }); test("parses only bounded PostgreSQL sequence cursors", () => { @@ -312,6 +480,7 @@ function storedEvent( sequence: 1n, eventId: "7b21f505-bd0f-49a2-bf8f-f238919b23fc", alertId: "12797430", + alertKey: null, eventType: "alert-created", status: "PENDING", priority: "HIGH", diff --git a/tests/ilert-canary-alert-key-migration.test.ts b/tests/ilert-canary-alert-key-migration.test.ts new file mode 100644 index 00000000..0ccb1d28 --- /dev/null +++ b/tests/ilert-canary-alert-key-migration.test.ts @@ -0,0 +1,443 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Client, Pool } from "pg"; + +import { + ensureOperationalIndexes, + MANAGED_RELEASE_OPERATIONAL_INDEXES, +} from "../scripts/ensure-operational-indexes"; +import alertKeySnapshot from "../drizzle/meta/0064_snapshot.json"; + +const TEST_URL = process.env.POSTIL_TEST_DATABASE_URL; +const describeDb = TEST_URL ? describe : describe.skip; + +describe("iLert canary alert key migration metadata", () => { + test("records the applied schema as the latest generated snapshot", async () => { + const previousSnapshot = JSON.parse( + await readFile( + new URL("../drizzle/meta/0063_snapshot.json", import.meta.url), + "utf8", + ), + ) as { id: string }; + const snapshot = alertKeySnapshot as { + id: string; + prevId: string; + tables: Record; + indexes: Record; + where?: string; + }>; + checkConstraints: Record; + }>; + }; + const journal = JSON.parse( + await readFile( + new URL("../drizzle/meta/_journal.json", import.meta.url), + "utf8", + ), + ) as { + entries: Array<{ + idx: number; + version: string; + when: number; + tag: string; + breakpoints: boolean; + }>; + }; + const events = snapshot.tables["public.ilert_alert_events"]!; + + expect(snapshot.prevId).toBe(previousSnapshot.id); + expect(snapshot.id).not.toBe(previousSnapshot.id); + expect(events.columns.alert_key).toEqual({ + name: "alert_key", + type: "text", + primaryKey: false, + notNull: false, + }); + expect(events.indexes.ilert_alert_events_canary_observation_idx).toMatchObject({ + columns: [ + { expression: "alert_key" }, + { expression: "event_type" }, + { expression: "alert_source_id" }, + ], + where: '"ilert_alert_events"."alert_key" IS NOT NULL', + }); + expect( + events.checkConstraints.ilert_alert_events_alert_key_check?.value, + ).toContain('length("ilert_alert_events"."alert_key") BETWEEN 1 AND 512'); + expect(journal.entries.at(-1)).toEqual({ + idx: 63, + version: "7", + when: 1791572108139, + tag: "0064_ilert_canary_alert_key", + breakpoints: true, + }); + }); +}); + +describeDb("iLert canary alert key migration", () => { + const databaseName = `postil_ilert_canary_key_${process.pid}_${Date.now()}`; + let adminClient: Client | undefined; + let client: Client | undefined; + let pool: Pool | undefined; + let migrationSource = ""; + let preMigrationError: unknown; + let preMigrationIndex: string | null = null; + let preMigrationReleaseStep = false; + let firstReconciliation: string[] = []; + let secondReconciliation: string[] = []; + + beforeAll(async () => { + adminClient = new Client({ connectionString: TEST_URL }); + await adminClient.connect(); + await adminClient.query(`CREATE DATABASE "${databaseName}"`); + + const databaseUrl = new URL(TEST_URL!); + databaseUrl.pathname = `/${databaseName}`; + const isolatedDatabaseUrl = databaseUrl.toString(); + client = new Client({ connectionString: isolatedDatabaseUrl }); + await client.connect(); + pool = new Pool({ connectionString: isolatedDatabaseUrl, max: 1 }); + + const migrationsDirectory = join(import.meta.dir, "..", "drizzle"); + const migrations = (await readdir(migrationsDirectory)) + .filter((file) => file.endsWith(".sql") && file < "0064_") + .sort(); + for (const migration of migrations) { + await applyMigration(client, join(migrationsDirectory, migration)); + } + + try { + await ensureOperationalIndexes(pool); + } catch (error) { + preMigrationError = error; + } + preMigrationIndex = (await client.query<{ index_name: string | null }>( + `SELECT to_regclass( + 'public.ilert_alert_events_canary_observation_idx' + )::text AS index_name`, + )).rows[0]?.index_name ?? null; + preMigrationReleaseStep = (await client.query<{ completed: boolean }>( + `SELECT EXISTS ( + SELECT 1 FROM release_steps WHERE name = 'operational-indexes-v6' + ) AS completed`, + )).rows[0]?.completed ?? false; + + // A row written before the migration stands in for existing production data. + await insertAlertEvent(client, PRE_MIGRATION_EVENT_ID); + + migrationSource = await readFile( + join(migrationsDirectory, "0064_ilert_canary_alert_key.sql"), + "utf8", + ); + await client.query("BEGIN"); + try { + for (const statement of migrationSource.split("--> statement-breakpoint")) { + if (statement.trim()) await client.query(statement); + } + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } + + await client.query( + `CREATE INDEX "ilert_alert_events_canary_observation_idx" + ON "ilert_alert_events" ("alert_key") + WHERE "alert_key" IS NULL`, + ); + firstReconciliation = await ensureOperationalIndexes(pool); + secondReconciliation = await ensureOperationalIndexes(pool); + }, 60_000); + + afterAll(async () => { + await client?.end(); + await pool?.end(); + if (adminClient) { + await adminClient.query(`DROP DATABASE IF EXISTS "${databaseName}"`); + await adminClient.end(); + } + }); + + test("keeps the migration transactional and refuses premature reconciliation", async () => { + expect(migrationSource).not.toContain("CREATE INDEX"); + expect(String(preMigrationError)).toContain( + "requires migrated columns public.ilert_alert_events.alert_key", + ); + expect(preMigrationIndex).toBeNull(); + expect(preMigrationReleaseStep).toBe(false); + const column = await client!.query<{ exists: boolean }>( + `SELECT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'ilert_alert_events' + AND column_name = 'alert_key' + ) AS exists`, + ); + expect(column.rows[0]?.exists).toBe(true); + }); + + test("adds the alert key check without validating existing rows", async () => { + expect(migrationSource).toContain( + 'ADD CONSTRAINT "ilert_alert_events_alert_key_check" CHECK', + ); + expect(migrationSource.trimEnd().endsWith(" NOT VALID;")).toBe(true); + expect(migrationSource).not.toContain("VALIDATE CONSTRAINT"); + + const constraint = await client!.query<{ + definition: string; + type: string; + validated: boolean; + }>( + `SELECT pg_get_constraintdef(oid) AS definition, + contype::text AS type, + convalidated AS validated + FROM pg_constraint + WHERE conrelid = 'public.ilert_alert_events'::regclass + AND conname = 'ilert_alert_events_alert_key_check'`, + ); + expect(constraint.rows).toEqual([ + { + definition: expect.stringMatching(/^CHECK \(.*alert_key.*\) NOT VALID$/), + type: "c", + validated: false, + }, + ]); + const existing = await client!.query<{ alert_key: string | null }>( + "SELECT alert_key FROM ilert_alert_events WHERE event_id = $1", + [PRE_MIGRATION_EVENT_ID], + ); + expect(existing.rows).toEqual([{ alert_key: null }]); + }); + + test("enforces the alert key check on every new and updated row", async () => { + const boundaryKey = "k".repeat(512); + // The deployed release writes rows without naming the new column. + await insertAlertEvent(client!, "00000000-0000-4000-8000-000000000002"); + await insertAlertEvent(client!, "00000000-0000-4000-8000-000000000003", null); + await insertAlertEvent(client!, "00000000-0000-4000-8000-000000000004", "k"); + await insertAlertEvent(client!, "00000000-0000-4000-8000-000000000005", boundaryKey); + + for (const [eventId, invalid] of [ + ["00000000-0000-4000-8000-000000000006", ""], + ["00000000-0000-4000-8000-000000000007", `${boundaryKey}k`], + ] as const) { + await expect(insertAlertEvent(client!, eventId, invalid)).rejects.toMatchObject({ + code: "23514", + constraint: "ilert_alert_events_alert_key_check", + }); + await expect(client!.query( + "UPDATE ilert_alert_events SET alert_key = $2 WHERE event_id = $1", + [PRE_MIGRATION_EVENT_ID, invalid], + )).rejects.toMatchObject({ + code: "23514", + constraint: "ilert_alert_events_alert_key_check", + }); + } + + const stored = await client!.query<{ event_id: string; key_length: number | null }>( + `SELECT event_id::text, length(alert_key) AS key_length + FROM ilert_alert_events + ORDER BY event_id`, + ); + expect(stored.rows).toEqual([ + { event_id: PRE_MIGRATION_EVENT_ID, key_length: null }, + { event_id: "00000000-0000-4000-8000-000000000002", key_length: null }, + { event_id: "00000000-0000-4000-8000-000000000003", key_length: null }, + { event_id: "00000000-0000-4000-8000-000000000004", key_length: 1 }, + { event_id: "00000000-0000-4000-8000-000000000005", key_length: 512 }, + ]); + }); + + test("repairs and idempotently verifies the exact partial composite index", async () => { + expect(firstReconciliation).toContain( + "ilert_alert_events_canary_observation_idx", + ); + expect(secondReconciliation).toEqual(firstReconciliation); + + const result = await client!.query<{ + access_method: string; + columns: string[]; + indisready: boolean; + indisvalid: boolean; + predicate: string; + table_name: string; + }>( + `SELECT access_method.amname AS access_method, + array_agg(attribute.attname::text ORDER BY key.ordinality)::text[] AS columns, + index_state.indisready, + index_state.indisvalid, + pg_get_expr(index_state.indpred, index_state.indrelid) AS predicate, + index_state.indrelid::regclass::text AS table_name + FROM pg_index AS index_state + JOIN pg_class AS index_class + ON index_class.oid = index_state.indexrelid + JOIN pg_am AS access_method + ON access_method.oid = index_class.relam + CROSS JOIN LATERAL unnest(index_state.indkey) + WITH ORDINALITY AS key(attnum, ordinality) + JOIN pg_attribute AS attribute + ON attribute.attrelid = index_state.indrelid + AND attribute.attnum = key.attnum + WHERE index_state.indexrelid = + to_regclass('public.ilert_alert_events_canary_observation_idx') + GROUP BY access_method.amname, + index_state.indisready, + index_state.indisvalid, + index_state.indpred, + index_state.indrelid`, + ); + expect(result.rows).toEqual([ + { + access_method: "btree", + columns: ["alert_key", "event_type", "alert_source_id"], + indisready: true, + indisvalid: true, + predicate: "(alert_key IS NOT NULL)", + table_name: "ilert_alert_events", + }, + ]); + }); + + test("leaves a valid index with an unexpected definition in place and fails closed", async () => { + // A unique index under the repairable name still enforces an invariant. + const canaryIndex = "ilert_alert_events_canary_observation_idx"; + await client!.query(`DROP INDEX "${canaryIndex}"`); + await client!.query( + `CREATE UNIQUE INDEX "${canaryIndex}" ON "ilert_alert_events" ("event_id")`, + ); + const uniqueCanary = await indexIdentity(client!, canaryIndex); + expect(uniqueCanary).toMatchObject({ unique: true, valid: true }); + await expect(ensureOperationalIndexes(pool!)).rejects.toThrow( + `operational index ${canaryIndex} has an unexpected definition`, + ); + expect(await indexIdentity(client!, canaryIndex)).toEqual(uniqueCanary); + + await client!.query(`DROP INDEX "${canaryIndex}"`); + expect(await ensureOperationalIndexes(pool!)).toEqual(firstReconciliation); + + // Indexes that predate the canary lookup never opt in to repair, whether + // or not the drifted index is unique. + for (const [name, drifted] of [ + [ + "finding_approvals_github_delivery_idx", + 'CREATE UNIQUE INDEX "finding_approvals_github_delivery_idx" ON "finding_approvals" ("id")', + ], + [ + "reviews_running_started_at_idx", + 'CREATE INDEX "reviews_running_started_at_idx" ON "reviews" ("id")', + ], + ] as const) { + const expected = (await client!.query<{ definition: string }>( + "SELECT pg_get_indexdef(to_regclass($1)) AS definition", + [`public.${name}`], + )).rows[0]!.definition; + await client!.query(`DROP INDEX "${name}"`); + await client!.query(drifted); + const before = await indexIdentity(client!, name); + expect(before?.valid).toBe(true); + await expect(ensureOperationalIndexes(pool!)).rejects.toThrow( + `operational index ${name} has an unexpected definition`, + ); + expect(await indexIdentity(client!, name)).toEqual(before); + await client!.query(`DROP INDEX "${name}"`); + await client!.query(expected); + } + expect(await ensureOperationalIndexes(pool!)).toEqual(firstReconciliation); + }, 60_000); + + test("builds only the canary index on the managed release path", async () => { + const canaryIndex = "ilert_alert_events_canary_observation_idx"; + const unrelated = "reviews_running_started_at_idx"; + const expected = (await client!.query<{ definition: string }>( + "SELECT pg_get_indexdef(to_regclass($1)) AS definition", + [`public.${unrelated}`], + )).rows[0]!.definition; + const releaseStep = async () => (await client!.query<{ completed_at: Date; details: unknown }>( + "SELECT completed_at, details FROM release_steps WHERE name = 'operational-indexes-v6'", + )).rows; + const stepBefore = await releaseStep(); + expect(stepBefore).toHaveLength(1); + + // The managed path neither inspects nor repairs an unrelated index. + await client!.query(`DROP INDEX "${unrelated}"`); + await client!.query(`CREATE INDEX "${unrelated}" ON "reviews" ("id")`); + const driftedUnrelated = await indexIdentity(client!, unrelated); + await client!.query(`DROP INDEX "${canaryIndex}"`); + + expect(MANAGED_RELEASE_OPERATIONAL_INDEXES).toEqual([canaryIndex]); + expect(await ensureOperationalIndexes(pool!, { managedRelease: true })).toEqual([ + canaryIndex, + ]); + expect(await indexIdentity(client!, canaryIndex)).toMatchObject({ + unique: false, + valid: true, + }); + expect(await indexIdentity(client!, unrelated)).toEqual(driftedUnrelated); + expect(await releaseStep()).toEqual(stepBefore); + expect((await client!.query<{ statement_timeout: string }>( + "SELECT current_setting('statement_timeout') AS statement_timeout", + )).rows[0]?.statement_timeout).toBe("0"); + const pooled = await pool!.query<{ statement_timeout: string }>( + "SELECT current_setting('statement_timeout') AS statement_timeout", + ); + expect(pooled.rows[0]?.statement_timeout).toBe("0"); + // A second managed run verifies the built index without rebuilding it. + const built = await indexIdentity(client!, canaryIndex); + await ensureOperationalIndexes(pool!, { managedRelease: true }); + expect(await indexIdentity(client!, canaryIndex)).toEqual(built); + + await client!.query(`DROP INDEX "${unrelated}"`); + await client!.query(expected); + expect(await ensureOperationalIndexes(pool!)).toEqual(firstReconciliation); + }, 60_000); +}); + +const PRE_MIGRATION_EVENT_ID = "00000000-0000-4000-8000-000000000001"; + +/** Insert one event, naming the alert key column only when a key is supplied. */ +async function insertAlertEvent( + client: Client, + eventId: string, + alertKey?: string | null, +): Promise { + const keyed = alertKey !== undefined; + await client.query( + `INSERT INTO ilert_alert_events + (event_id, alert_id, event_type, status, priority, summary, details, + alert_source_id, alert_source_name, report_time, occurred_at, + payload_sha256${keyed ? ", alert_key" : ""}) + VALUES ($1, '20000001', 'alert-created', 'PENDING', 'HIGH', 'Fixture alert', + '', 2269078, 'Postil production', now(), now(), repeat('a', 64)${keyed ? ", $2" : ""})`, + keyed ? [eventId, alertKey] : [eventId], + ); +} + +async function indexIdentity( + client: Client, + name: string, +): Promise<{ oid: string; unique: boolean; valid: boolean } | null> { + const result = await client.query<{ oid: string; unique: boolean; valid: boolean }>( + `SELECT indexrelid::text AS oid, indisunique AS unique, indisvalid AS valid + FROM pg_index + WHERE indexrelid = to_regclass($1)`, + [`public.${name}`], + ); + return result.rows[0] ?? null; +} + +async function applyMigration(client: Client, path: string): Promise { + const sql = await readFile(path, "utf8"); + for (const statement of sql.split("--> statement-breakpoint")) { + if (statement.trim()) await client.query(statement); + } +} diff --git a/tests/migration-lint.test.ts b/tests/migration-lint.test.ts index 0114cad5..8ba6b92c 100644 --- a/tests/migration-lint.test.ts +++ b/tests/migration-lint.test.ts @@ -409,6 +409,15 @@ describe("migration lint", () => { ), "utf8", ); + const ilertCanaryAlertKeyMigration = await readFile( + join( + import.meta.dir, + "..", + "drizzle", + "0064_ilert_canary_alert_key.sql", + ), + "utf8", + ); expect(migration).toContain('CREATE TABLE "release_steps"'); expect(migration).not.toContain("CREATE INDEX"); @@ -438,6 +447,12 @@ describe("migration lint", () => { expect(publicationLifecycleRepairMigration).toContain( "hashtextextended('postil:publication-lifecycle-release-v2', 0)", ); + expect(ilertCanaryAlertKeyMigration).not.toContain("CREATE INDEX"); + // Validating the check would scan the live table under its exclusive lock. + expect(ilertCanaryAlertKeyMigration).toMatch( + /ADD CONSTRAINT "ilert_alert_events_alert_key_check" CHECK \(.*\) NOT VALID;\s*$/, + ); + expect(ilertCanaryAlertKeyMigration).not.toContain("VALIDATE CONSTRAINT"); expect(releaseScript).toContain( 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "reviews_publication_lifecycle_pending_idx"', ); @@ -462,6 +477,12 @@ describe("migration lint", () => { expect(releaseScript).toContain( 'CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "finding_approvals_github_delivery_idx"', ); + expect(releaseScript).toContain( + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS "ilert_alert_events_canary_observation_idx"', + ); + expect(releaseScript).toContain( + 'columns: ["alert_key", "event_type", "alert_source_id"]', + ); expect(releaseScript).toContain("!state.indisvalid || !state.indisready"); expect(releaseScript).toContain("DROP INDEX CONCURRENTLY IF EXISTS"); expect(releaseScript).toContain("pg_try_advisory_lock($1, $2)"); @@ -479,6 +500,13 @@ describe("migration lint", () => { expect(releasePreparationScript).toContain( '["bun", "run", "notifications:quiesce"]', ); + expect( + releasePreparationScript.indexOf('["bun", "run", "db:migrate"]'), + ).toBeLessThan( + releasePreparationScript.indexOf( + '["bun", "run", "operational:indexes"]', + ), + ); }); test("CLI refresh sessions add only family-scoped credentials", async () => { diff --git a/tests/reconcile-ilert-alert-stream.test.ts b/tests/reconcile-ilert-alert-stream.test.ts new file mode 100644 index 00000000..55e4cb51 --- /dev/null +++ b/tests/reconcile-ilert-alert-stream.test.ts @@ -0,0 +1,3415 @@ +import { describe, expect, test } from "bun:test"; +import { randomBytes } from "node:crypto"; + +import { + ALERT_TRIGGER_TYPES, + CANARY_FINALIZER_BUDGETS, + CANARY_RERUN_LOOKBACK_MS, + canaryAlertKey, + desiredAlertAction, + equivalentAlertAction, + finalizeIlertWebhookCanary, + MAX_CANARY_RUN_ATTEMPT, + type Fetch, + parseReceiverOrigin, + reconcileIlertAlertAction, + runCli, + sweepIlertWebhookCanaryOrphans, + validWebhookSecret, + verifyIlertWebhookCanary, +} from "../scripts/reconcile-ilert-alert-stream"; + +// Credential fixtures are generated per run so the source carries no +// credential-shaped literal. +const fixtureCredential = (bytes = 24): string => randomBytes(bytes).toString("hex"); +const API_KEY = fixtureCredential(); +const INTEGRATION_KEY = fixtureCredential(); +const REPLACEMENT_INTEGRATION_KEY = fixtureCredential(); +const SOURCE_ID = 42; +const RECEIVER_ORIGIN = "https://postil.example"; +const WEBHOOK_SECRET = fixtureCredential(); +const RUN_ID = "12345"; +const RUN_ATTEMPT = "1"; +const SOURCE = { + id: SOURCE_ID, + integrationKey: INTEGRATION_KEY, + name: "Postil test source", + integrationType: "API", + escalationPolicy: { + id: 7, + name: "Test escalation", + escalationRules: [], + }, +}; + +const reconcileOptions = { + apiKey: API_KEY, + integrationKey: INTEGRATION_KEY, + receiverOrigin: RECEIVER_ORIGIN, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + sourceId: SOURCE_ID, + sleep: async () => undefined, + webhookSecret: WEBHOOK_SECRET, +}; + +const canaryOptions = { + ...reconcileOptions, + runAttempt: RUN_ATTEMPT, + runId: RUN_ID, + sleep: async () => undefined, +}; + +describe("iLert webhook-action reconciliation", () => { + test("shares the receiver-secret contract with deployment validation", async () => { + expect(validWebhookSecret(WEBHOOK_SECRET)).toBe(true); + expect(validWebhookSecret("x".repeat(32))).toBe(false); + expect(validWebhookSecret("short")).toBe(false); + await runCli({ + args: ["--validate-webhook-secret"], + env: { POSTIL_ILERT_WEBHOOK_SECRET: WEBHOOK_SECRET }, + log: () => undefined, + }); + await expect(runCli({ + args: ["--validate-webhook-secret"], + env: { POSTIL_ILERT_WEBHOOK_SECRET: "x".repeat(32) }, + log: () => undefined, + })).rejects.toThrow("POSTIL_ILERT_WEBHOOK_SECRET must contain"); + }); + + test("builds an unfiltered default-body action for a documented API source", () => { + const prefix = fixtureCredential(); + const secret = `${prefix}:/?#%value`; + const desired = desiredAlertAction(SOURCE, secret, RECEIVER_ORIGIN); + expect(desired).toMatchObject({ + alertSources: [{ id: SOURCE_ID }], + connectorType: "webhook", + conditions: "", + name: "Postil operator alert stream", + triggerMode: "AUTOMATIC", + triggerTypes: [...ALERT_TRIGGER_TYPES], + params: { + bodyTemplate: "", + webhookUrl: + `https://postil-ilert:${prefix}%3A%2F%3F%23%25value@postil.example/api/webhooks/ilert`, + }, + }); + }); + + test("includes hidden conditions and custom body templates in equivalence", () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const actual = structuredClone(desired) as Record; + actual.id = "42"; + actual.triggerTypes = [...ALERT_TRIGGER_TYPES].reverse(); + expect(equivalentAlertAction(actual, desired)).toBe(true); + + actual.conditions = "alert.priority == 'HIGH'"; + expect(equivalentAlertAction(actual, desired)).toBe(false); + actual.conditions = ""; + (actual.params as Record).bodyTemplate = "{}"; + expect(equivalentAlertAction(actual, desired)).toBe(false); + (actual.params as Record).bodyTemplate = ""; + (actual.params as Record).headers = [ + { key: "Authorization", value: "unsupported" }, + ]; + expect(equivalentAlertAction(actual, desired)).toBe(false); + }); + + test("rejects receiver URLs with unsafe schemes, credentials, or non-origin data", () => { + const receiverPassword = fixtureCredential(); + const receiverWithCredentials = `https://user:${receiverPassword}@postil.example`; + expect(new URL(receiverWithCredentials).password).toBe(receiverPassword); + expect(parseReceiverOrigin("https://postil.example/")).toBe(RECEIVER_ORIGIN); + for (const value of [ + "http://postil.example", + receiverWithCredentials, + "https://postil.example/path", + "https://postil.example?query=1", + "https://postil.example/#fragment", + ]) { + expect(() => parseReceiverOrigin(value)).toThrow("HTTPS origin"); + } + }); + + test("requires the configured source to be an Event API source before action lookup", async () => { + const requests: Request[] = []; + await expect( + reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: queuedFetch(requests, [ + Response.json({ ...SOURCE, integrationType: "GITHUB" }), + ]), + }), + ).rejects.toThrow("integration binding validation failed"); + expect(requests).toHaveLength(1); + expect(new URL(requests[0]!.url).pathname).toBe(`/api/alert-sources/${SOURCE_ID}`); + expect(requests[0]!.method).toBe("GET"); + }); + + test("rejects a mismatched management binding before action mutation or Event API use", async () => { + const requests: Request[] = []; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request); + return Response.json({ ...SOURCE, id: SOURCE_ID + 1 }); + }; + await expect(reconcileIlertAlertAction({ ...reconcileOptions, fetchFn })) + .rejects.toThrow("integration binding validation failed"); + expect(requests.filter((request) => + request.url.endsWith("/events") || + (request.url.includes("/alert-actions") && request.method !== "GET") + )).toHaveLength(0); + expect(requests).toHaveLength(1); + }); + + test("rejects an integration-key mismatch before action mutation", async () => { + const requests: Request[] = []; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request); + return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + }, + })).rejects.toThrow("integration binding validation failed"); + expect(new URL(requests[0]!.url).pathname).toBe(`/api/alert-sources/${SOURCE_ID}`); + expect(requests).toHaveLength(1); + }); + + test("redacts a sensitive binding failure and emits no Event API mutation", async () => { + const integrationKey = fixtureCredential(); + const apiKey = fixtureCredential(); + const requests: Request[] = []; + const logs: string[] = []; + let error: unknown; + try { + await runCli({ + args: ["--dry-run"], + env: cliEnvironment({ ILERT_API_KEY: apiKey, ILERT_INTEGRATION_KEY: integrationKey }), + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request); + throw new Error(`request failed for ${request.url} with Bearer ${apiKey}`); + }, + log: (message) => logs.push(message), + }); + } catch (caught) { + error = caught; + } + expect(String(error)).toBe("Error: iLert integration binding validation failed"); + expect(String(error)).not.toContain(integrationKey); + expect(String(error)).not.toContain(apiKey); + expect(logs.join("\n")).not.toContain(integrationKey); + expect(logs.join("\n")).not.toContain(apiKey); + expect(requests.every((request) => !request.url.endsWith("/events"))).toBe(true); + }); + + test("scans the global inventory and avoids detail calls for unrelated actions", async () => { + const requests: Request[] = []; + const unrelated = Array.from({ length: 100 }, (_, index) => ({ + id: String(index + 1), + name: `Unrelated action ${index + 1}`, + params: { webhookUrl: `https://example.test/${index + 1}` }, + })); + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + dryRun: true, + fetchFn: queuedFetch(requests, [ + Response.json(SOURCE), + Response.json(unrelated), + Response.json([]), + ]), + }); + expect(result).toEqual({ actionId: null, operation: "create" }); + expect(requests).toHaveLength(5); + expect(requests[1]!.url).not.toContain("source="); + expect(requests[2]!.url).toContain("start-index=99"); + expect(requests[4]!.url).toContain("start-index=99"); + }); + + test("fails before POST when a disappearing action shifts the reserved action across pages", async () => { + const requests: Request[] = []; + const firstPage = Array.from({ length: 100 }, (_, index) => ({ + id: String(index + 1), + name: `Unrelated action ${index + 1}`, + })); + const shifted = [ + ...firstPage.slice(1), + { id: "101", name: "Postil operator alert stream" }, + ]; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alert-actions") { + const start = Number(url.searchParams.get("start-index")); + const scan = requests.filter((seen) => + new URL(seen.url).pathname === "/api/alert-actions" && + new URL(seen.url).searchParams.get("start-index") === "0" + ).length; + if (scan === 1) return Response.json(start === 0 ? firstPage : []); + return Response.json(start === 0 ? shifted : []); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("alert-action inventory changed during continuity validation"); + expect(requests.some((request) => + request.method === "POST" && new URL(request.url).pathname === "/api/alert-actions" + )).toBe(false); + }); + + test("requests conditions and reconciles condition and body-template drift", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { + ...desired, + conditions: "alert.priority == 'HIGH'", + id: "72", + params: { ...(desired.params as object), bodyTemplate: "{}" }, + }; + const requests: Request[] = []; + let updated = false; + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: String(desired.name) }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(updated ? { ...desired, id: "72" } : drifted); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + updated = true; + return Response.json({ ...desired, id: "72" }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + }); + expect(result).toEqual({ actionId: "72", operation: "update" }); + const preflight = requests.find((request) => request.url === `${RECEIVER_ORIGIN}/api/webhooks/ilert`); + expect(preflight?.headers.get("authorization")).toStartWith("Basic "); + const update = requests.find((request) => request.method === "PUT"); + expect(update?.url).toContain("include=conditions"); + const body = await update!.json() as Record; + expect(body.conditions).toBe(""); + expect(body.params).toMatchObject({ bodyTemplate: "" }); + }); + + test("preserves writable action fields from the boundary detail without echoing server fields", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const existing = { + ...desired, + id: "72", + conditions: "alert.priority == 'HIGH'", + alertSources: [{ id: SOURCE_ID, integrationKey: INTEGRATION_KEY, name: "Expanded source" }], + teams: [{ id: 9, name: "Operations", managedBy: { id: 1 } }], + escalationEndedDelaySec: 0, + notResolvedDelaySec: 0, + connectorId: "connector-9", + connector: { id: "connector-9", name: "Expanded connector" }, + managedBy: { id: 1 }, + bidirectional: false, + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-02T00:00:00Z", + }; + const requests: Request[] = []; + let current: Record = existing; + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions") return Response.json([{ id: 72, name: desired.name }]); + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(current); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + current = await request.json() as Record; + return Response.json(current); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + }); + expect(result).toEqual({ actionId: "72", operation: "update" }); + const update = requests.find((request) => request.method === "PUT"); + expect(update).toBeDefined(); + const body = await update!.json() as Record; + expect(body).toMatchObject({ + id: "72", + conditions: "", + teams: [{ id: 9 }], + escalationEndedDelaySec: 0, + notResolvedDelaySec: 0, + connectorId: "connector-9", + alertSources: [{ id: SOURCE_ID }], + }); + expect(body.teams).toEqual([{ id: 9 }]); + expect(body.alertSources).toEqual([{ id: SOURCE_ID }]); + for (const key of ["connector", "managedBy", "bidirectional", "createdAt", "updatedAt", "integrationKey"]) { + expect(body).not.toHaveProperty(key); + } + }); + + test("accepts absent and null empty fields after a successful update with numeric detail ID", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const existing = { + ...desired, + id: 72, + conditions: "alert.priority == 'HIGH'", + teams: [], + connectorId: "", + notResolvedDelaySec: 0, + }; + let updated = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions") return Response.json([{ id: "72", name: desired.name }]); + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(updated + ? { + ...desired, + id: 72, + teams: null, + connectorId: null, + notResolvedDelaySec: null, + escalationEndedDelaySec: null, + } + : existing); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + updated = true; + const body = await request.json() as Record; + expect(body.id).toBe("72"); + expect(body.teams).toEqual([]); + expect(body.connectorId).toBe(""); + expect(body.notResolvedDelaySec).toBe(0); + return Response.json({ ...desired, id: 72 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).resolves.toEqual({ actionId: "72", operation: "update" }); + expect(updated).toBe(true); + }); + + for (const [field, value, error] of [ + ["notResolvedDelaySec", 86_400, "requires its trigger type"], + ["escalationEndedDelaySec", 1, "requires its trigger type"], + ["notResolvedDelaySec", 172_801, "invalid alert-action"], + ["escalationEndedDelaySec", 7_201, "invalid alert-action"], + ["notResolvedDelaySec", -1, "invalid alert-action"], + ["notResolvedDelaySec", "45", "invalid alert-action"], + ] as const) { + test(`rejects incompatible or out-of-schema ${field} ${value} before PUT`, async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const existing = { ...desired, id: 72, conditions: "alert.priority == 'HIGH'", [field]: value }; + const requests: Request[] = []; + let failure: unknown; + try { + await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions") return Response.json([{ id: "72", name: desired.name }]); + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") return Response.json(existing); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + }); + } catch (caught) { + failure = caught; + } + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain(field); + expect((failure as Error).message).toContain(error); + expect((failure as Error).message).not.toContain(String(value)); + expect(requests.some((request) => request.method === "PUT")).toBe(false); + }); + } + + for (const field of ["teams", "connectorId"] as const) { + test(`rejects loss of nonempty ${field} after PUT`, async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const existing = { + ...desired, + id: "72", + conditions: "alert.priority == 'HIGH'", + teams: [{ id: 9 }], + connectorId: "connector-9", + }; + let current: Record = existing; + let putCount = 0; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions") return Response.json([{ id: 72, name: desired.name }]); + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") return Response.json(current); + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + putCount += 1; + const body = await request.json() as Record; + expect(body.id).toBe("72"); + expect(body.teams).toEqual([{ id: 9 }]); + expect(body.connectorId).toBe("connector-9"); + current = { ...body, [field]: field === "teams" ? [] : null }; + return Response.json(current); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("did not retain the existing alert-action fields"); + expect(putCount).toBe(1); + }); + } + + test("clears an active deprecated alertFilter and verifies its removal", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const legacy = { + ...desired, + alertFilter: { operator: "AND", predicates: [{ field: "ALERT_SUMMARY" }] }, + id: "72", + }; + const requests: Request[] = []; + let updated = false; + await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: String(desired.name) }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(updated ? { ...desired, id: "72" } : legacy); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + updated = true; + return Response.json({ ...desired, id: "72" }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + }); + const update = requests.find((request) => request.method === "PUT"); + expect(update).toBeDefined(); + expect(await update!.json()).toMatchObject({ alertFilter: null, conditions: "" }); + }); + + test("runs the receiver preflight without Event API mutation for an equivalent action", async () => { + const desired = { ...desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN), id: "72" }; + const actionName = "Postil operator alert stream"; + const requests: Request[] = []; + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: queuedFetch(requests, [ + Response.json(SOURCE), + Response.json([{ id: "72", name: actionName }]), + Response.json(desired), + new Response(null, { status: 204 }), + Response.json(SOURCE), + Response.json(desired), + ]), + }); + expect(result).toEqual({ actionId: "72", operation: "unchanged" }); + expect(requests.filter((request) => request.url.endsWith("/events"))).toHaveLength(0); + expect(requests.some((request) => request.url === `${RECEIVER_ORIGIN}/api/webhooks/ilert`)).toBe(true); + expect(requests.some((request) => request.method === "PUT")).toBe(false); + }); + + test("finds a reserved action globally when its source scope drifted", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + dryRun: true, + fetchFn: queuedFetch(requests, [ + Response.json(SOURCE), + Response.json([{ id: "72", name: desired.name }]), + Response.json({ ...desired, alertSources: [{ id: SOURCE_ID + 1 },], id: "72" }), + ]), + })).rejects.toThrow("conflicting Postil alert action"); + expect(requests[1]!.url).not.toContain("source="); + }); + + test("fails closed for a renamed action at the receiver endpoint with an older credential", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const syntheticBasicAuthRedactionMarker = fixtureCredential(); + const renamed = { + ...desired, + alertSources: [{ id: SOURCE_ID + 1 }], + id: "72", + name: "Legacy operator webhook", + params: { + ...(desired.params as Record), + webhookUrl: `https://postil-ilert:${syntheticBasicAuthRedactionMarker}@postil.example/api/webhooks/ilert`, + }, + }; + let error: unknown; + try { + await reconcileIlertAlertAction({ + ...reconcileOptions, + dryRun: true, + fetchFn: queuedFetch([], [ + Response.json(SOURCE), + Response.json([{ + id: "72", + name: renamed.name, + params: renamed.params, + }]), + Response.json(renamed), + ]), + }); + } catch (caught) { + error = caught; + } + expect(String(error)).toContain("conflicting Postil alert action"); + expect(String(error)).not.toContain(syntheticBasicAuthRedactionMarker); + }); + + test("adopts one equivalent action after an ambiguous committed POST without retrying", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + let created = false; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (request.url === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (request.method === "POST" && request.url.includes("/alert-actions")) { + created = true; + throw new Error("connection reset after commit"); + } + if (request.url.includes("/alert-actions")) { + if (!created) return Response.json([]); + if (request.url.includes("include=conditions")) return Response.json({ ...desired, id: "73" }); + return Response.json([{ id: "73", name: desired.name }]); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn, + sleep: async () => undefined, + }); + expect(result).toEqual({ actionId: "73", operation: "create" }); + expect(requests.filter((request) => + request.method === "POST" && request.url.includes("/alert-actions") + )).toHaveLength(1); + }); + + test("fails closed when ambiguous action creation finds an equivalent and a conflicting reserved candidate", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const oldCredential = fixtureCredential(); + let created = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (request.url === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (request.method === "POST" && request.url.includes("/alert-actions")) { + created = true; + throw new Error("connection reset after commit"); + } + if (request.url.includes("/alert-actions")) { + if (!created) return Response.json([]); + if (request.url.includes("include=conditions")) { + const id = new URL(request.url).pathname.split("/").at(-1); + return Response.json(id === "73" + ? { ...desired, id: "73" } + : { + ...desired, + alertSources: [{ id: SOURCE_ID + 1 }], + id: "74", + name: "Legacy operator webhook", + params: { + ...(desired.params as Record), + webhookUrl: `https://postil-ilert:${oldCredential}@postil.example/api/webhooks/ilert`, + }, + }); + } + return Response.json([ + { id: "73", name: desired.name }, + { + id: "74", + name: "Legacy operator webhook", + params: { + webhookUrl: `https://postil-ilert:${oldCredential}@postil.example/api/webhooks/ilert`, + }, + }, + ]); + } + throw new Error("unexpected request"); + }, + })).rejects.toThrow("conflicting Postil alert action"); + }); + + test("does not persist a credential when the deployed receiver rejects preflight", async () => { + const requests: Request[] = []; + await expect( + reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: queuedFetch(requests, [ + Response.json(SOURCE), + Response.json([]), + Response.json({ error: "unauthorized" }, { status: 401 }), + ]), + }), + ).rejects.toThrow("credential preflight failed with HTTP 401"); + expect(requests.filter((request) => request.url.endsWith("/events"))).toHaveLength(0); + expect(requests.some((request) => request.url.includes("/alert-actions") && request.method !== "GET")).toBe(false); + }); + + test("propagates a receiver preflight failure instead of recovering a hypothetical created action", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + let actionInventories = 0; + let receiverPreflights = 0; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + receiverPreflights += 1; + return receiverPreflights === 1 + ? new Response(null, { status: 204 }) + : Response.json({ error: "unavailable" }, { status: 503 }); + } + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + actionInventories += 1; + // This equivalent action is visible only if failed pre-POST + // validation incorrectly enters ambiguous-create recovery. + return Response.json(actionInventories <= 4 ? [] : [{ id: "73", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/73") return Response.json({ ...desired, id: "73" }); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("credential preflight failed with HTTP 503"); + expect(receiverPreflights).toBeGreaterThanOrEqual(2); + expect(actionInventories).toBe(4); + expect(requests.some((request) => request.method === "POST")).toBe(false); + }); + + test("creates a missing action only after receiver preflight", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + let created = false; + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json(created ? [{ id: "71", name: desired.name }] : []); + } + if (url.pathname === "/api/alert-actions" && request.method === "POST") { + created = true; + return Response.json({ ...desired, id: "71" }); + } + if (url.pathname === "/api/alert-actions/71") return Response.json({ ...desired, id: "71" }); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + }); + expect(result).toEqual({ actionId: "71", operation: "create" }); + const create = requests.find((request) => + request.method === "POST" && request.url.includes("/alert-actions"), + ); + expect(create).toBeDefined(); + const createIndex = requests.indexOf(create!); + const preflightIndexes = requests.flatMap((request, index) => + request.url === `${RECEIVER_ORIGIN}/api/webhooks/ilert` ? [index] : [], + ); + expect(preflightIndexes).toHaveLength(2); + expect(preflightIndexes[1]).toBeLessThan(createIndex); + expect(requests.slice(preflightIndexes[1]! + 1, createIndex).some((request) => + request.method === "GET" && new URL(request.url).pathname === "/api/alert-actions" + )).toBe(true); + const body = await create!.json() as { + alertSources: Array>; + }; + expect(body.alertSources[0]).not.toHaveProperty("integrationKey"); + }); + + test("refuses POST when a reserved action appears during receiver preflight", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + let boundary = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + boundary = true; + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions") { + return Response.json(boundary ? [{ id: "73", name: desired.name }] : []); + } + if (url.pathname === "/api/alert-actions/73") { + return Response.json({ ...desired, id: "73" }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("appeared before creation"); + expect(requests.some((request) => + request.method === "POST" || request.method === "PUT" + )).toBe(false); + }); + + test("refuses POST when the action inventory changes during its immediate attempt preflight", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + let receiverPreflights = 0; + let appearedDuringAttemptPreflight = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + receiverPreflights += 1; + if (receiverPreflights === 2) appearedDuringAttemptPreflight = true; + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json(appearedDuringAttemptPreflight + ? [{ id: "73", name: desired.name }] + : []); + } + if (url.pathname === "/api/alert-actions/73" && request.method === "GET") { + return Response.json({ ...desired, id: "73" }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("appeared before creation"); + expect(receiverPreflights).toBe(2); + expect(requests.filter((request) => request.method === "POST")).toHaveLength(0); + }); + + test("revalidates the source binding after receiver preflight before POST", async () => { + const requests: Request[] = []; + let boundary = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + return Response.json(boundary + ? { ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY } + : SOURCE); + } + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + boundary = true; + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions") return Response.json([]); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("integration binding validation failed"); + expect(requests.filter((request) => + new URL(request.url).pathname.startsWith("/api/alert-sources/") + )).toHaveLength(2); + expect(requests.some((request) => request.method === "POST")).toBe(false); + }); + + test("refuses PUT when a reserved action changes during receiver preflight", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const initial = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const concurrent = { ...initial, conditions: "alert.status == 'PENDING'" }; + const requests: Request[] = []; + let boundary = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + boundary = true; + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions") { + return Response.json([{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72") { + return Response.json(boundary ? concurrent : initial); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("changed before mutation"); + expect(requests.some((request) => request.method === "PUT")).toBe(false); + }); + + test("fails closed when a second reserved action appears during PUT", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const requests: Request[] = []; + let mutated = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (request.method === "PUT" && url.pathname === "/api/alert-actions/72") { + mutated = true; + return Response.json({ ...desired, id: "72" }); + } + if (url.pathname === "/api/alert-actions") { + return Response.json(mutated + ? [{ id: "72", name: desired.name }, { id: "73", name: desired.name }] + : [{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72") { + return Response.json(mutated ? { ...desired, id: "72" } : drifted); + } + if (url.pathname === "/api/alert-actions/73") return Response.json({ ...desired, id: "73" }); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("Multiple Postil webhook alert actions exist"); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(1); + }); + + test("fails closed when a second reserved action appears during POST", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const requests: Request[] = []; + let mutated = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (request.method === "POST" && url.pathname === "/api/alert-actions") { + mutated = true; + return Response.json({ ...desired, id: "72" }); + } + if (url.pathname === "/api/alert-actions") { + return Response.json(mutated + ? [{ id: "72", name: desired.name }, { id: "73", name: desired.name }] + : []); + } + if (url.pathname === "/api/alert-actions/72" || url.pathname === "/api/alert-actions/73") { + return Response.json({ ...desired, id: url.pathname.endsWith("72") ? "72" : "73" }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("Multiple Postil webhook alert actions exist"); + expect(requests.filter((request) => request.method === "POST")).toHaveLength(1); + }); + + test("revalidates the source binding immediately before action mutation", async () => { + const requests: Request[] = []; + let sourceReads = 0; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + sourceReads += 1; + return Response.json(sourceReads >= 3 + ? { ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY } + : SOURCE); + } + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions") return Response.json([]); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("integration binding validation failed"); + expect(requests.some((request) => request.method === "POST" || request.method === "PUT")).toBe(false); + }); + + test("does not retry PUT after a transient response rotates the source binding", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const requests: Request[] = []; + let rotated = false; + let updateAttempts = 0; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + return Response.json(rotated ? { ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY } : SOURCE); + } + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") return Response.json(drifted); + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + updateAttempts += 1; + rotated = true; + return new Response(null, { status: 503 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("integration binding validation failed"); + expect(updateAttempts).toBe(1); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(1); + }); + + test("does not retry PUT after receiver preflight fails during backoff", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const requests: Request[] = []; + let receiverPreflights = 0; + let updateAttempts = 0; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + receiverPreflights += 1; + return new Response(null, { status: receiverPreflights >= 3 ? 401 : 204 }); + } + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(drifted); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + updateAttempts += 1; + return new Response(null, { status: 503 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("credential preflight failed with HTTP 401"); + expect(receiverPreflights).toBe(3); + expect(updateAttempts).toBe(1); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(1); + }); + + test("does not dispatch PUT when its validation crosses the reconciliation deadline", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const requests: Request[] = []; + let currentTime = 0; + let receiverPreflights = 0; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + now: () => currentTime, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + receiverPreflights += 1; + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + if (receiverPreflights === 2) currentTime = 180_001; + return Response.json(drifted); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("deadline expired"); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(0); + }); + + test("does not retry PUT after a transient response changes the reserved action", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const concurrent = { ...drifted, conditions: "alert.status == 'PENDING'" }; + const requests: Request[] = []; + let changed = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(changed ? concurrent : drifted); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + changed = true; + return new Response(null, { status: 503 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("changed before mutation"); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(1); + }); + + test("does not retry PUT when the reserved action changes during retry preflight", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const drifted = { ...desired, conditions: "alert.priority == 'HIGH'", id: "72" }; + const concurrent = { ...drifted, conditions: "alert.status == 'PENDING'" }; + const requests: Request[] = []; + let receiverPreflights = 0; + let changedDuringRetryPreflight = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + receiverPreflights += 1; + if (receiverPreflights === 3) changedDuringRetryPreflight = true; + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + return Response.json([{ id: "72", name: desired.name }]); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "GET") { + return Response.json(changedDuringRetryPreflight ? concurrent : drifted); + } + if (url.pathname === "/api/alert-actions/72" && request.method === "PUT") { + return new Response(null, { status: 503 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("changed before mutation"); + expect(receiverPreflights).toBe(3); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(1); + }); + + test("does not issue a second POST after an ambiguous transient creation response", async () => { + const requests: Request[] = []; + let rotated = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + return Response.json(rotated ? { ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY } : SOURCE); + } + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (url.pathname === "/api/alert-actions" && request.method === "GET") return Response.json([]); + if (url.pathname === "/api/alert-actions" && request.method === "POST") { + rotated = true; + return new Response(null, { status: 503 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("iLert alert-action creation was ambiguous"); + expect(requests.filter((request) => request.method === "POST")).toHaveLength(1); + }); + + test("releases a failed creation body before ambiguity recovery", async () => { + const lifecycle: string[] = []; + let creationBodyPending = false; + let creationBodyCancellations = 0; + let creationAttempted = false; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + if (creationBodyPending) { + throw new Error("recovery started before creation body cancellation"); + } + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.href === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) { + return new Response(null, { status: 204 }); + } + if (url.pathname === "/api/alert-actions" && request.method === "GET") { + if (creationAttempted) lifecycle.push("recovery-inventory"); + return Response.json([]); + } + if (url.pathname === "/api/alert-actions" && request.method === "POST") { + creationAttempted = true; + creationBodyPending = true; + lifecycle.push("creation-response"); + return new Response(new ReadableStream({ + cancel() { + creationBodyCancellations += 1; + creationBodyPending = false; + lifecycle.push("creation-cancelled"); + }, + }), { status: 403 }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("iLert alert-action creation was ambiguous"); + expect(creationBodyPending).toBe(false); + expect(creationBodyCancellations).toBe(1); + expect(lifecycle.slice(0, 3)).toEqual([ + "creation-response", + "creation-cancelled", + "recovery-inventory", + ]); + }); + + test("fails closed after creation when the global inventory finds a concurrent reserved candidate", async () => { + const desired = desiredAlertAction(SOURCE, WEBHOOK_SECRET, RECEIVER_ORIGIN); + const oldCredential = fixtureCredential(); + for (const conflictingSummary of [ + { id: "74", name: desired.name }, + { + id: "74", + name: "Legacy operator webhook", + params: { + webhookUrl: `https://postil-ilert:${oldCredential}@postil.example/api/webhooks/ilert`, + }, + }, + ]) { + let inventoryScans = 0; + const requests: Request[] = []; + await expect(reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (request.url === `${RECEIVER_ORIGIN}/api/webhooks/ilert`) return new Response(null, { status: 204 }); + if (request.method === "POST" && request.url.includes("/alert-actions")) { + return Response.json({ id: "73" }); + } + if (request.url.includes("/alert-actions?")) { + inventoryScans += 1; + return Response.json(inventoryScans === 1 + ? [] + : [{ id: "73", name: desired.name }, conflictingSummary]); + } + if (request.url.includes("/alert-actions/")) { + const id = new URL(request.url).pathname.split("/").at(-1); + return Response.json(id === "73" + ? { ...desired, id: "73" } + : { ...desired, ...conflictingSummary, id: "74" }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("inventory changed during continuity validation"); + expect(requests.filter((request) => + request.method === "POST" && request.url.includes("/alert-actions") + )).toHaveLength(0); + } + }); + + test("bounds 429 handling and honors a capped Retry-After delay", async () => { + const requests: Request[] = []; + const sleeps: number[] = []; + const lifecycle: string[] = []; + let sourceAttempts = 0; + const result = await reconcileIlertAlertAction({ + ...reconcileOptions, + dryRun: true, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + sourceAttempts += 1; + lifecycle.push(`attempt-${sourceAttempts}`); + if (sourceAttempts === 1) { + return new Response(new ReadableStream({ + cancel() { + lifecycle.push("cancelled"); + }, + }), { status: 429, headers: { "retry-after": "60" } }); + } + return Response.json(SOURCE); + } + if (url.pathname === "/api/alert-actions") return Response.json([]); + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + sleep: async (milliseconds) => { + sleeps.push(milliseconds); + lifecycle.push("slept"); + }, + }); + expect(result.operation).toBe("create"); + expect(sleeps).toEqual([10_000]); + expect(lifecycle).toEqual(["attempt-1", "cancelled", "slept", "attempt-2"]); + expect(requests).toHaveLength(4); + }); + + test("reports only an HTTP status when a provider request is rejected", async () => { + const providerBody = "provider response containing credential material"; + try { + await reconcileIlertAlertAction({ + ...reconcileOptions, + fetchFn: async (input) => new URL(String(input)).pathname.startsWith("/api/alert-sources/") + ? Response.json(SOURCE) + : new Response(providerBody, { status: 403 }), + }); + throw new Error("expected reconciliation to fail"); + } catch (error) { + expect(String(error)).toContain("HTTP 403"); + expect(String(error)).not.toContain(providerBody); + } + }); +}); + +describe("iLert webhook canary", () => { + test("uses unique deterministic keys for every workflow attempt", () => { + expect(canaryAlertKey(RUN_ID, "1")).toBe( + "postil-ilert-webhook-canary-12345-1", + ); + expect(canaryAlertKey(RUN_ID, "2")).toBe( + "postil-ilert-webhook-canary-12345-2", + ); + expect(canaryAlertKey(RUN_ID, "51")).toBe( + "postil-ilert-webhook-canary-12345-51", + ); + }); + + test("accepts attempt 51 as a finalizer sweep ceiling", async () => { + const service = canaryService({ + initialAlerts: [ + { + alertKey: canaryAlertKey(RUN_ID, "50"), + id: 98, + status: "PENDING", + }, + ], + }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn: service.fetchFn, + runAttempt: "1", + sweepAttempt: "51", + }); + expect(service.alertListRequests).toBeGreaterThan(0); + expect(service.requests.some((request) => request.url.endsWith("/alerts/98/resolve"))).toBe(true); + }); + + test("recovers an old-run orphan without touching routine, lookalike, or other-source alerts", async () => { + const oldRunId = "67890"; + const service = canaryService({ + initialAlerts: [ + { + alertKey: canaryAlertKey(oldRunId, "2"), + id: 98, + status: "PENDING", + }, + { + alertKey: "postil-production-monitor", + id: 99, + status: "PENDING", + }, + { + alertKey: "postil-ilert-webhook-canary-not-a-run-id", + id: 100, + status: "PENDING", + }, + { + alertKey: canaryAlertKey(oldRunId, "3"), + id: 101, + sourceId: SOURCE_ID + 1, + status: "PENDING", + }, + { + alertKey: "postil-production-monitor-test", + id: 102, + status: "PENDING", + }, + { + alertKey: "postil-production-monitor-test-extra", + id: 103, + status: "PENDING", + }, + ], + }); + await sweepIlertWebhookCanaryOrphans({ + apiKey: API_KEY, + fetchFn: service.fetchFn, + sleep: async () => undefined, + sourceId: SOURCE_ID, + }); + const resolvedIds = service.requests + .filter((request) => request.method === "PUT") + .map((request) => /\/alerts\/([1-9][0-9]*)\/resolve$/u.exec(new URL(request.url).pathname)?.[1]); + expect(resolvedIds).toEqual(["98", "102"]); + expect(service.events).toEqual([]); + expect(service.statuses).toEqual([ + "RESOLVED", + "PENDING", + "PENDING", + "PENDING", + "RESOLVED", + "PENDING", + ]); + }); + + test("resolves a retained exact ID then fails closed after a continuity mismatch", async () => { + const service = canaryService({ + initialAlerts: [{ + alertKey: canaryAlertKey("67890", "2"), + id: 98, + status: "PENDING", + }], + }); + let alertCountRequests = 0; + let alertListRequests = 0; + await expect(sweepIlertWebhookCanaryOrphans({ + apiKey: API_KEY, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname === "/api/alerts/count") { + alertCountRequests += 1; + return Response.json({ count: alertCountRequests < 3 ? 1 : 0 }); + } + if (new URL(request.url).pathname === "/api/alerts") { + alertListRequests += 1; + if (alertListRequests === 2) return Response.json([]); + } + return service.fetchFn(input, init); + }, + sleep: async () => undefined, + sourceId: SOURCE_ID, + })).rejects.toThrow("inventory was incomplete after resolving retained alerts"); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + )).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("fails closed when unrelated orphan inventory identities churn between scans", async () => { + const requests: Request[] = []; + let alertListRequests = 0; + await expect(sweepIlertWebhookCanaryOrphans({ + apiKey: API_KEY, + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alerts/count") return Response.json({ count: 1 }); + if (url.pathname === "/api/alerts") { + alertListRequests += 1; + return Response.json([{ + alertKey: `routine-alert-${alertListRequests}`, + alertSource: { id: SOURCE_ID }, + id: alertListRequests, + priority: "HIGH", + status: "PENDING", + }]); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + sleep: async () => undefined, + sourceId: SOURCE_ID, + })).rejects.toThrow("inventory was incomplete after resolving retained alerts"); + expect(alertListRequests).toBe(2); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(0); + }); + + test("preserves an opaque routine-alert ID while validating orphan inventory continuity", async () => { + let alertListRequests = 0; + await sweepIlertWebhookCanaryOrphans({ + apiKey: API_KEY, + fetchFn: async (input) => { + const url = new URL(String(input)); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alerts/count") return Response.json({ count: 1 }); + if (url.pathname === "/api/alerts") { + alertListRequests += 1; + return Response.json([{ + alertKey: "routine-alert", + alertSource: { id: SOURCE_ID }, + id: "provider.alert:0001", + priority: "HIGH", + status: "PENDING", + }]); + } + throw new Error(`unexpected request: ${url}`); + }, + sleep: async () => undefined, + sourceId: SOURCE_ID, + }); + expect(alertListRequests).toBe(4); + }); + + test("rejects attempt 52 for deterministic keys and finalizer sweeps", async () => { + expect(() => canaryAlertKey(RUN_ID, "52")).toThrow( + "GitHub run attempt must be between 1 and 51", + ); + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + runAttempt: "1", + sweepAttempt: "52", + })).rejects.toThrow("GitHub run attempt must be between 1 and 51"); + }); + + test("documents the runtime canary rerun ceiling", async () => { + const architecture = await Bun.file(new URL("../ARCHITECTURE.md", import.meta.url)).text(); + expect(architecture).toContain( + `GitHub's 1 through ${MAX_CANARY_RUN_ATTEMPT} attempt range`, + ); + expect(CANARY_RERUN_LOOKBACK_MS).toBe(32 * 24 * 60 * 60 * 1_000); + expect(architecture).toContain("exact 32-day run-wide report-time window"); + }); + + test("attempt 2 pre-cleans attempt 1 main key before its HIGH alert", async () => { + const service = canaryService({ + initialAlerts: [ + { + alertKey: canaryAlertKey(RUN_ID, "1"), + id: 98, + status: "PENDING", + }, + ], + }); + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + runAttempt: "2", + }); + const highAlertIndex = service.events.findIndex((event) => + event.alertKey === canaryAlertKey(RUN_ID, "2") && event.eventType === "ALERT" + ); + expect(highAlertIndex).toBeGreaterThan(-1); + expect(service.statuses).toEqual(["RESOLVED", "RESOLVED"]); + }); + + test("does not let a delayed prior-attempt event satisfy the current identity", async () => { + const service = canaryService({ + dropAlertCreation: true, + initialAlerts: [ + { + alertKey: canaryAlertKey(RUN_ID, "1"), + id: 11, + status: "PENDING", + }, + ], + }); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + runAttempt: "2", + })).rejects.toBeInstanceOf(AggregateError); + expect(service.statuses[0]).toBe("RESOLVED"); + }); + + test("rejects a non-API source before ALERT", async () => { + const service = canaryService({ sourceIntegrationType: "GITHUB" }); + await expect( + verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + }), + ).rejects.toThrow("integration binding validation failed"); + expect(service.events).toHaveLength(0); + }); + + test("revalidates the source binding after preclean before ALERT", async () => { + const service = canaryService(); + let precleanListed = false; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname === "/api/alerts") precleanListed = true; + if (url.pathname.startsWith("/api/alert-sources/") && precleanListed) { + return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + } + return service.fetchFn(input, init); + }; + let error: unknown; + try { + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn, + runAttempt: "2", + }); + } catch (caught) { + error = caught; + } + expect(errorMessages(error)).toContain("iLert integration binding validation failed"); + expect(service.events).toHaveLength(0); + }); + + test("uses management exact-ID cleanup after the Event API key rotates", async () => { + const service = canaryService(); + let sourceReads = 0; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) { + sourceReads += 1; + if (sourceReads >= 3) return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + } + return service.fetchFn(input, init); + }; + await verifyIlertWebhookCanary({ ...canaryOptions, fetchFn }); + expect(service.events).toEqual([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + eventType: "ALERT", + }]); + expect(service.requests.some((request) => + request.method === "PUT" && /\/alerts\/\d+\/resolve$/u.test(new URL(request.url).pathname) + )).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("retains a discovered exact ID before receiver evidence and uses it after later discovery failure", async () => { + const service = canaryService(); + let inventoryListings = 0; + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname === "/api/alerts" && request.method === "GET") { + inventoryListings += 1; + if (inventoryListings > 2) return new Response(null, { status: 503 }); + } + if ( + url.href.startsWith(`${RECEIVER_ORIGIN}/api/webhooks/ilert?`) && + url.searchParams.get("eventType") === "alert-created" + ) { + return Response.json({ received: false }); + } + return service.fetchFn(input, init); + }, + })).rejects.toThrow("iLert management request failed with HTTP 503"); + expect(service.events.filter((event) => event.eventType === "RESOLVE")).toEqual([]); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/99/resolve") + )).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("reports an Event API binding failure while exact-ID cleanup survives rotation", async () => { + const service = canaryService(); + let rotated = false; + let alertResponseLost = true; + let error: unknown; + try { + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/") && rotated) { + return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + } + const response = await service.fetchFn(input, init); + if ( + alertResponseLost && + request.method === "POST" && + request.url.endsWith("/events") + ) { + alertResponseLost = false; + rotated = true; + return new Response(null, { status: 503 }); + } + return response; + }, + }); + } catch (caught) { + error = caught; + } + expect(errorMessages(error)).toContain("iLert integration binding validation failed"); + expect(service.events.filter((event) => event.eventType === "ALERT")).toHaveLength(1); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/99/resolve") + )).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("uses a same-key Event RESOLVE only while the current alert is undiscovered", async () => { + const service = canaryService({ managementFailsAfterAlert: true }); + await expect( + verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + }), + ).rejects.toThrow("cleanup could not be verified"); + expect(service.events[0]).toEqual({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + eventType: "ALERT", + }); + expect(service.events.some((event) => event.eventType === "RESOLVE")).toBe(true); + expect(new Set(service.events.map((event) => event.alertKey))).toEqual( + new Set([canaryAlertKey(RUN_ID, RUN_ATTEMPT)]), + ); + }); + + test("retries transient RESOLVE and receiver observation failures", async () => { + const service = canaryService({ + receiverObservationFailures: 2, + resolveRequestFailures: 2, + }); + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + }); + expect(service.requests.filter((request) => request.method === "PUT" && /\/alerts\/[1-9][0-9]*\/resolve$/u.test(new URL(request.url).pathname))).toHaveLength(3); + expect(service.receiverObservationRequests).toBeGreaterThanOrEqual(4); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("releases successful Event API and exact-ID resolution bodies before continuing", async () => { + const service = canaryService(); + const lifecycle: string[] = []; + let pendingBody: string | undefined; + let parsedBodyCancellations = 0; + const fetchFn: Fetch = async (input, init) => { + if (pendingBody) throw new Error(`request started before ${pendingBody} body cancellation`); + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + return new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(JSON.stringify(SOURCE))); + controller.close(); + }, + cancel() { + parsedBodyCancellations += 1; + }, + }), { headers: { "content-type": "application/json" } }); + } + const response = await service.fetchFn(input, init); + const label = request.method === "POST" && url.pathname === "/api/events" + ? "event" + : request.method === "PUT" && /\/alerts\/[1-9][0-9]*\/resolve$/u.test(url.pathname) + ? "resolution" + : undefined; + if (!label) return response; + pendingBody = label; + lifecycle.push(`${label}-response`); + return new Response(new ReadableStream({ + cancel() { + lifecycle.push(`${label}-cancelled`); + pendingBody = undefined; + }, + }), { status: response.status }); + }; + + await verifyIlertWebhookCanary({ ...canaryOptions, fetchFn }); + + expect(pendingBody).toBeUndefined(); + expect(parsedBodyCancellations).toBe(0); + expect(lifecycle).toEqual([ + "event-response", + "event-cancelled", + "resolution-response", + "resolution-cancelled", + ]); + }); + + test("preserves a failing Event API status when body cancellation rejects", async () => { + const service = canaryService(); + let cancellations = 0; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + const response = await service.fetchFn(input, init); + if (request.method !== "POST" || new URL(request.url).pathname !== "/api/events") { + return response; + } + const body = await request.clone().json() as { eventType: string }; + if (body.eventType !== "ALERT") return response; + return new Response(new ReadableStream({ + cancel() { + cancellations += 1; + throw new Error("body cancellation failed"); + }, + }), { status: 403 }); + }; + + await expect(verifyIlertWebhookCanary({ ...canaryOptions, fetchFn })) + .rejects.toThrow("iLert event request failed with HTTP 403"); + expect(cancellations).toBe(1); + }); + + test("releases every failing exact-ID resolution body exactly once", async () => { + const service = canaryService({ existing: true }); + let pendingBody = false; + let resolutionRequests = 0; + let cancellations = 0; + const fetchFn: Fetch = async (input, init) => { + if (pendingBody) throw new Error("request started before resolution body cancellation"); + const request = new Request(input, init); + const url = new URL(request.url); + if (request.method !== "PUT" || !/\/alerts\/[1-9][0-9]*\/resolve$/u.test(url.pathname)) { + return service.fetchFn(input, init); + } + resolutionRequests += 1; + pendingBody = true; + return new Response(new ReadableStream({ + cancel() { + cancellations += 1; + pendingBody = false; + }, + }), { status: 409 }); + }; + + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + })).rejects.toThrow("did not verify every discovered alert as RESOLVED"); + expect(pendingBody).toBe(false); + expect(resolutionRequests).toBeGreaterThan(0); + expect(cancellations).toBe(resolutionRequests); + }); + + test("does not retry ALERT after a transient response rotates the source binding", async () => { + const service = canaryService({ alertRetryAfter: "0" }); + let rotated = false; + let error: unknown; + try { + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/") && rotated) { + return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + } + const response = await service.fetchFn(input, init); + if (request.method === "POST" && request.url.endsWith("/events")) rotated = true; + return response; + }, + }); + } catch (caught) { + error = caught; + } + expect(errorMessages(error)).toContain("iLert integration binding validation failed"); + expect(service.events.filter((event) => event.eventType === "ALERT")).toHaveLength(1); + }); + + test("does not dispatch an Event API mutation when validation exhausts the primary deadline", async () => { + const service = canaryService(); + let currentTime = 0; + let sourceReads = 0; + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) { + sourceReads += 1; + if (sourceReads === 2) currentTime = 360_001; + return Response.json(SOURCE); + } + return service.fetchFn(input, init); + }, + now: () => currentTime, + sleep: async () => undefined, + })).rejects.toThrow("cleanup could not be verified"); + expect(service.requests.filter((request) => + request.method === "POST" && new URL(request.url).pathname === "/api/events" + )).toHaveLength(0); + }); + + test("retries management exact-ID resolution after a transient response and Event API key rotation", async () => { + const service = canaryService({ resolveRequestFailures: 1 }); + let rotated = false; + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/") && rotated) { + return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + } + const response = await service.fetchFn(input, init); + if (request.method === "PUT" && /\/alerts\/99\/resolve$/u.test(new URL(request.url).pathname)) { + rotated = true; + } + return response; + }, + }); + expect(service.requests.filter((request) => + request.method === "PUT" && request.url.endsWith("/alerts/99/resolve") + )).toHaveLength(2); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("does not retry Event API RESOLVE after a transient response rotates the source binding", async () => { + const service = canaryService({ eventResolveAlwaysFails: true, managementFailsAfterAlert: true }); + let rotated = false; + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/") && rotated) { + return Response.json({ ...SOURCE, integrationKey: REPLACEMENT_INTEGRATION_KEY }); + } + const response = await service.fetchFn(input, init); + if (request.method === "POST" && request.url.endsWith("/events")) { + const body = await request.clone().json() as { eventType: string }; + if (body.eventType === "RESOLVE") rotated = true; + } + return response; + }, + })).rejects.toThrow("cleanup could not be verified"); + expect(service.events.filter((event) => event.eventType === "RESOLVE")).toHaveLength(1); + }); + + test("requires exact persisted alert-resolved evidence, not action-history growth", async () => { + const service = canaryService({ dropResolvedReceiverEvent: true }); + await expect( + verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + }), + ).rejects.toThrow("did not verify canary resolution"); + expect(service.requests.some((request) => request.url.endsWith("/actions"))).toBe(false); + expect(service.requests.some((request) => request.url.includes("eventType=alert-resolved"))).toBe(true); + }); + + test("requires receiver evidence at the configured alert-source id", async () => { + const service = canaryService({ observedSourceId: SOURCE_ID + 1 }); + await expect( + verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + }), + ).rejects.toThrow("did not confirm persisted Postil webhook delivery"); + const observationRequests = service.requests.filter((request) => + request.url.startsWith(`${RECEIVER_ORIGIN}/api/webhooks/ilert?`), + ); + expect(observationRequests.length).toBeGreaterThan(0); + expect(observationRequests.every((request) => + new URL(request.url).searchParams.get("sourceId") === String(SOURCE_ID), + )).toBe(true); + }); + + test("accepts an exact persisted ACCEPTED canary creation", async () => { + const service = canaryService({ status: "ACCEPTED" }); + await verifyIlertWebhookCanary({ ...canaryOptions, fetchFn: service.fetchFn }); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("keeps discovery open for a canary report time 16 seconds after a 202", async () => { + let currentTime = 0; + const service = canaryService({ + alertReportTime: new Date(16_000).toISOString(), + alertRetryAfter: "10", + }); + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + const discoveryWindows = service.requests + .filter((request) => request.url.includes("/alerts?")) + .map((request) => new URL(request.url).searchParams); + const discoveryUntil = discoveryWindows.map((query) => Date.parse(query.get("until")!)); + expect(discoveryWindows.every((query) => + Date.parse(query.get("until")!) - Date.parse(query.get("from")!) === CANARY_RERUN_LOOKBACK_MS + )).toBe(true); + expect(discoveryUntil.some((until) => until >= 16_000)).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("uses one exact 32-day all-state source-scoped window for every discovery query", async () => { + let currentTime = (30 * 24 * 60 * 60 * 1_000) + (10 * 60 * 1_000); + const service = canaryService({ alertReportTime: new Date(currentTime).toISOString() }); + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + const inventoryQueries = service.requests + .map((request) => new URL(request.url)) + .filter((url) => url.pathname === "/api/alerts" || url.pathname === "/api/alerts/count"); + expect(inventoryQueries.length).toBeGreaterThan(0); + for (const url of inventoryQueries) { + const query = url.searchParams; + expect(query.get("sources")).toBe(String(SOURCE_ID)); + expect(query.getAll("states")).toEqual(["PENDING", "ACCEPTED", "RESOLVED"]); + expect(Date.parse(query.get("until")!) - Date.parse(query.get("from")!)) + .toBe(CANARY_RERUN_LOOKBACK_MS); + if (url.pathname === "/api/alerts") { + expect(query.get("max-results")).toBe("100"); + expect(Number(query.get("start-index"))).toBeGreaterThanOrEqual(0); + } + } + }); + + test("discovers and resolves the exact canary after a slow successful ALERT response", async () => { + let currentTime = 0; + const service = canaryService({ alertReportTime: new Date(0).toISOString() }); + let delayed = false; + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + const response = await service.fetchFn(input, init); + if (!delayed && request.method === "POST" && request.url.endsWith("/events")) { + delayed = true; + currentTime += 6_000; + } + return response; + }, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + const firstDiscovery = service.requests.find((request) => request.url.includes("/alerts?")); + const firstQuery = new URL(firstDiscovery!.url).searchParams; + expect(Date.parse(firstQuery.get("until")!) - Date.parse(firstQuery.get("from")!)) + .toBe(CANARY_RERUN_LOOKBACK_MS); + expect(service.requests.some((request) => request.url.endsWith("/alerts/99/resolve"))).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("discovers and resolves the exact committed canary after a lost first ALERT response", async () => { + let currentTime = 0; + const service = canaryService({ + alertReportTime: new Date(0).toISOString(), + deduplicateAlertKeys: true, + }); + let lost = false; + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + const response = await service.fetchFn(input, init); + if (!lost && request.method === "POST" && request.url.endsWith("/events")) { + lost = true; + currentTime += 8_000; + throw new Error("connection reset after commit"); + } + return response; + }, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + const firstDiscovery = service.requests.find((request) => request.url.includes("/alerts?")); + const firstQuery = new URL(firstDiscovery!.url).searchParams; + expect(Date.parse(firstQuery.get("until")!) - Date.parse(firstQuery.get("from")!)) + .toBe(CANARY_RERUN_LOOKBACK_MS); + expect(service.events.filter((event) => event.eventType === "ALERT")).toHaveLength(2); + expect(service.requests.some((request) => request.url.endsWith("/alerts/99/resolve"))).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("rejects a LOW management result for the HIGH canary", async () => { + const service = canaryService({ mainPriority: "LOW" }); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + })).rejects.toThrow("did not confirm persisted Postil webhook delivery"); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("fails when the retained alert ID stays pending despite empty lists", async () => { + const service = canaryService({ + detailAlwaysPending: true, + emptyListsAfterResolve: true, + }); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: service.fetchFn, + })).rejects.toBeInstanceOf(AggregateError); + expect(service.requests.some((request) => request.url.endsWith("/alerts/99"))).toBe(true); + }); + + test("uses exact alert detail instead of a stale open-alert listing after RESOLVE", async () => { + const service = canaryService({ staleOpenReadsAfterResolve: 1 }); + await verifyIlertWebhookCanary({ ...canaryOptions, fetchFn: service.fetchFn }); + expect(service.requests.some((request) => request.method === "PUT" && request.url.endsWith("/alerts/99/resolve"))).toBe(true); + expect(service.events.filter((event) => event.eventType === "RESOLVE")).toHaveLength(0); + expect(service.requests.some((request) => request.url.endsWith("/alerts/99"))).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("a cleaned handoff still performs an idempotent terminal inventory sweep", async () => { + const service = canaryService(); + await verifyIlertWebhookCanary({ ...canaryOptions, fetchFn: service.fetchFn }); + const eventCount = service.events.length; + await finalizeIlertWebhookCanary({ + ...canaryOptions, + alertSubmitted: "cleaned", + fetchFn: service.fetchFn, + ...finalizerTiming(), + }); + expect(service.events).toHaveLength(eventCount); + expect(service.alertListRequests).toBeGreaterThan(0); + }); + + test("revalidates the management source before accepting empty finalizer scans", async () => { + const service = canaryService(); + let sourceReads = 0; + let error: unknown; + try { + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) { + sourceReads += 1; + if (sourceReads > 1) { + return Response.json({ ...SOURCE, integrationType: "EMAIL" }); + } + } + return service.fetchFn(input, init); + }, + }); + } catch (caught) { + error = caught; + } + expect(errorMessages(error)).toContain( + "iLert management alert-source validation failed", + ); + expect(errorMessages(error)).toContain( + "iLert alert source does not match the configured management identity", + ); + expect(sourceReads).toBe(2); + expect(service.alertListRequests).toBeGreaterThan(0); + expect(service.events).toEqual([]); + }); + + test("finalizer continuity accepts an unrelated opaque routine ID without mutation", async () => { + const requests: Request[] = []; + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) { + return Response.json(SOURCE); + } + if (url.pathname === "/api/alerts/count") return Response.json({ count: 1 }); + if (url.pathname === "/api/alerts") { + return Response.json([{ + alertKey: "routine-alert", + alertSource: { id: SOURCE_ID }, + id: "provider.alert:0001", + priority: "HIGH", + status: "PENDING", + }]); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + }); + expect(requests.filter((request) => request.method === "PUT")).toHaveLength(0); + }); + + test("finalizer resolves only the exact compatibility canary key", async () => { + const service = canaryService({ + initialAlerts: [ + { alertKey: "postil-production-monitor-test", id: 98, status: "PENDING" }, + { alertKey: "postil-production-monitor", id: 99, status: "PENDING" }, + { alertKey: "postil-production-monitor-test-extra", id: 100, status: "PENDING" }, + ], + }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn: service.fetchFn, + }); + expect(service.requests.filter((request) => request.method === "PUT").map((request) => + new URL(request.url).pathname + )).toEqual(["/api/alerts/98/resolve"]); + expect(service.statuses).toEqual(["RESOLVED", "PENDING", "PENDING"]); + }); + + test("a cleaned handoff resolves a delayed duplicate after a retry-ambiguous ALERT", async () => { + let currentTime = 0; + const service = canaryService({ + deferFirstAlertCreation: true, + releaseDeferredAlertAfterResolved: true, + }); + let firstAlertResponseLost = true; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + const response = await service.fetchFn(input, init); + if (firstAlertResponseLost && request.method === "POST" && request.url.endsWith("/events")) { + firstAlertResponseLost = false; + throw new Error("connection reset after Event API acceptance"); + } + return response; + }; + + await verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + expect(service.events.filter((event) => event.eventType === "ALERT")).toHaveLength(2); + expect(service.statuses).toEqual(["RESOLVED"]); + + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + + expect(service.alertListRequests).toBeGreaterThan(1); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/100/resolve") + )).toBe(true); + expect(service.statuses).toEqual(["RESOLVED", "RESOLVED"]); + expect(service.events.filter((event) => event.eventType === "ALERT")).toHaveLength(2); + }); + + test("observes a stale open-state read after an accepted RESOLVE", async () => { + const service = canaryService({ existing: true, staleOpenReadsAfterResolve: 2 }); + await finalizeIlertWebhookCanary({ ...finalizerOptions(), fetchFn: service.fetchFn }); + expect(service.requests.filter((request) => request.method === "PUT" && request.url.endsWith("/alerts/98/resolve")).length).toBeGreaterThanOrEqual(1); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("finalizer resolves a discovered deterministic key by management identity", async () => { + const service = canaryService({ + existing: true, + deferredListings: 0, + staleOpenReadsAfterResolve: 2, + status: "PENDING", + }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: service.fetchFn, + }); + expect(service.requests.some((request) => request.method === "PUT" && request.url.endsWith("/alerts/98/resolve"))).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + for (const alertSubmitted of ["true", "unknown"] as const) { + test(`${alertSubmitted} finalizer finds a deterministic alert that appears after 31 seconds`, async () => { + const service = canaryService({ + deferredListings: 8, + existing: true, + staleOpenReadsAfterResolve: 2, + }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted, + fetchFn: service.fetchFn, + }); + expect( + service.events.some((event) => event.eventType === "RESOLVE") || + service.requests.some((request) => + request.method === "PUT" && /\/alerts\/[1-9][0-9]*\/resolve$/u.test(new URL(request.url).pathname) + ), + ).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + } + + test("management listing failure leaves cleanup required for the attempt key", async () => { + const service = canaryService({ + existing: true, + failAlertListings: 3, + staleOpenReadsAfterResolve: 2, + status: "PENDING", + }); + await expect( + verifyIlertWebhookCanary({ ...canaryOptions, fetchFn: service.fetchFn }), + ).rejects.toThrow("cleanup could not be verified"); + await finalizeIlertWebhookCanary({ ...finalizerOptions(), fetchFn: service.fetchFn }); + expect( + service.events.some((event) => event.eventType === "RESOLVE") || + service.requests.some((request) => + request.method === "PUT" && /\/alerts\/[1-9][0-9]*\/resolve$/u.test(new URL(request.url).pathname) + ), + ).toBe(true); + expect(service.statuses).toEqual(["RESOLVED", "RESOLVED"]); + }); + + test("unknown finalizer rejects an accepted same-key RESOLVE when no alert is observed", async () => { + let currentTime = 0; + const service = canaryService(); + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: service.fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + })).rejects.toThrow("could not account for the accepted current-attempt submission"); + expect(currentTime).toBeGreaterThanOrEqual(CANARY_FINALIZER_BUDGETS.discoveryMs); + expect(service.events).toEqual([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + eventType: "RESOLVE", + }]); + }); + + test("unknown finalizer fails closed when every Event API RESOLVE is rejected and management stays empty", async () => { + const service = canaryService({ eventResolveAlwaysFails: true }); + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: service.fetchFn, + })).rejects.toThrow("could not account for the accepted current-attempt submission"); + expect(service.events.filter((event) => event.eventType === "RESOLVE").length).toBeGreaterThan(1); + expect(service.statuses).toEqual([]); + }); + + test("unknown finalizer recovers through a discovered ID when Event API RESOLVEs fail", async () => { + const service = canaryService({ + deferredListings: 2, + eventResolveAlwaysFails: true, + existing: true, + staleOpenReadsAfterResolve: 2, + }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: service.fetchFn, + }); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + )).toBe(true); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("continues the finalizer after repeated management 503 responses", async () => { + const service = canaryService({ + existing: true, + failAlertListings: 6, + status: "PENDING", + }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: service.fetchFn, + }); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("finds a late discovery alert and resolves its retained ID in reserve", async () => { + const service = canaryService({ lateAlertAtListing: 18 }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: service.fetchFn, + }); + expect(service.statuses).toEqual(["RESOLVED"]); + expect(service.requests.some((request) => request.url.endsWith("/alerts/199"))).toBe(true); + }); + + test("partitions finalizer deadlines and does not starve a later retained ID", async () => { + let currentTime = 0; + let healthyResolved = false; + const requestTimes: Array<{ request: Request; time: number }> = []; + const alerts = Array.from({ length: 10 }, (_, index) => ({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: index + 1, + priority: "HIGH", + status: "PENDING", + })); + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + requestTimes.push({ request: request.clone(), time: currentTime }); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alerts/count") return Response.json({ count: alerts.length }); + if (url.pathname === "/api/alerts") { + return Response.json(alerts.map((alert) => ({ + ...alert, + status: alert.id === 10 && healthyResolved ? "RESOLVED" : "PENDING", + }))); + } + const resolveId = /\/alerts\/([1-9][0-9]*)\/resolve$/u.exec(url.pathname)?.[1]; + if (request.method === "PUT" && resolveId) { + if (resolveId === "10") { + healthyResolved = true; + return new Response(null, { status: 200 }); + } + currentTime += 5_000; + return new Response(null, { status: 503, headers: { "retry-after": "10" } }); + } + const detailId = /\/alerts\/([1-9][0-9]*)$/u.exec(url.pathname)?.[1]; + if (detailId) { + const alert = alerts[Number(detailId) - 1]!; + return Response.json({ + ...alert, + status: alert.id === 10 && healthyResolved ? "RESOLVED" : "PENDING", + }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + })).rejects.toThrow("did not verify every discovered alert as RESOLVED"); + + const terminalInventoryTimes = requestTimes.filter(({ request }) => { + const url = new URL(request.url); + return url.pathname === "/api/alerts" && url.searchParams.getAll("states").includes("RESOLVED"); + }).map(({ time }) => time); + const resolutionTimes = requestTimes.filter(({ request }) => + request.method === "PUT" && request.url.includes("/alerts/") + ).map(({ time }) => time); + expect(healthyResolved).toBe(true); + expect(terminalInventoryTimes).toContain(CANARY_FINALIZER_BUDGETS.discoveryMs); + expect(resolutionTimes.some((time) => + time >= CANARY_FINALIZER_BUDGETS.discoveryMs + + CANARY_FINALIZER_BUDGETS.terminalValidationMs + )).toBe(true); + expect(currentTime).toBe(CANARY_FINALIZER_BUDGETS.operationMs); + expect(requestTimes.every(({ time }) => time < CANARY_FINALIZER_BUDGETS.operationMs)).toBe(true); + }); + + test("uses the 32-day inventory window throughout bounded delayed discovery", async () => { + const runStartedAt = 30 * 24 * 60 * 60 * 1_000 + 10 * 60 * 1_000; + let currentTime = runStartedAt; + let resolved = false; + const inventoryTimes: number[] = []; + const requests: Request[] = []; + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/events") return new Response(null, { status: 202 }); + if (url.pathname === "/api/alerts/count") { + return Response.json({ count: currentTime >= runStartedAt + 81_000 ? 1 : 0 }); + } + if (url.pathname === "/api/alerts") { + inventoryTimes.push(currentTime); + const from = url.searchParams.get("from"); + if (url.searchParams.getAll("states").includes("RESOLVED")) { + expect(Date.parse(from!)).toBeLessThanOrEqual(0); + } + return Response.json(currentTime >= runStartedAt + 81_000 ? [{ + alertKey: canaryAlertKey(RUN_ID, "1"), + alertSource: { id: SOURCE_ID }, + id: 199, + priority: "HIGH", + status: resolved ? "RESOLVED" : "PENDING", + }] : []); + } + if (request.method === "PUT" && url.pathname === "/api/alerts/199/resolve") { + resolved = true; + return new Response(null, { status: 200 }); + } + if (url.pathname === "/api/alerts/199") { + return Response.json({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 199, + priority: "HIGH", + status: resolved ? "RESOLVED" : "PENDING", + }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + runAttempt: "2", + sweepAttempt: "2", + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + expect(inventoryTimes).toContain(runStartedAt + 80_000); + expect(inventoryTimes).toContain(runStartedAt + 90_000); + expect(requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/199/resolve") + )).toBe(true); + expect(resolved).toBe(true); + }); + + test("does not post the current HIGH alert when count-backed preclean loses a shifted prior alert", async () => { + const requests: Request[] = []; + const unrelated = Array.from({ length: 100 }, (_, index) => ({ + alertKey: `unrelated-${index + 1}`, + alertSource: { id: SOURCE_ID }, + id: index + 1, + priority: "HIGH", + status: "PENDING", + })); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + runAttempt: "2", + fetchFn: async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alerts/count") return Response.json({ count: 101 }); + if (url.pathname === "/api/alerts") { + return Response.json(Number(url.searchParams.get("start-index")) === 0 ? unrelated : []); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }, + })).rejects.toThrow("did not stabilize prior-attempt canary cleanup"); + expect(requests.some((request) => + new URL(request.url).pathname === "/api/alerts/count" + )).toBe(true); + expect(requests.some((request) => { + const url = new URL(request.url); + return url.pathname === "/api/alerts" && + url.searchParams.getAll("states").join(",") === "PENDING,ACCEPTED,RESOLVED"; + })).toBe(true); + expect(requests.some((request) => + request.method === "POST" && new URL(request.url).pathname === "/api/events" + )).toBe(false); + }); + + test("accounts for an already resolved accepted current alert through all-state lookup", async () => { + const service = canaryService({ existing: true, status: "RESOLVED" }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "true", + fetchFn: service.fetchFn, + }); + const boundedLookup = service.requests.find((request) => { + if (!request.url.includes("/alerts?")) return false; + const query = new URL(request.url).searchParams; + return query.has("from") && query.has("until") && !query.has("states"); + }); + expect(boundedLookup).toBeDefined(); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("uses the producer attempt for accepted-current finalizer proof", async () => { + const service = canaryService({ existing: true, status: "RESOLVED" }); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "true", + fetchFn: service.fetchFn, + runAttempt: "1", + sweepAttempt: "2", + }); + expect(service.statuses).toEqual(["RESOLVED"]); + }); + + test("finalizer sweeps earlier attempts from a running-attempt fallback identity", async () => { + const service = canaryService({ + initialAlerts: [ + { alertKey: canaryAlertKey(RUN_ID, "1"), id: 98, status: "PENDING" }, + { alertKey: canaryAlertKey(RUN_ID, "2"), id: 99, status: "PENDING" }, + ], + }); + await runCli({ + args: ["--finalize-canary"], + env: cliEnvironment({ + POSTIL_ILERT_CANARY_ALERT_SUBMITTED: "unknown", + POSTIL_ILERT_CANARY_RUN_ATTEMPT: "2", + POSTIL_ILERT_CANARY_SWEEP_ATTEMPT: "2", + }), + fetchFn: service.fetchFn, + ...finalizerTiming(), + log: () => undefined, + }); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + )).toBe(true); + }); + + test("excludes a large resolved history outside the bounded terminal report-time window", async () => { + const requests: Request[] = []; + const historical = Array.from({ length: 1_100 }, (_, index) => ({ + alertKey: `historical-${index}`, + id: index + 1, + status: "RESOLVED", + })); + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (request.url.endsWith("/events")) return new Response(null, { status: 202 }); + if (new URL(request.url).pathname === "/api/alerts/count") { + return Response.json({ count: 0 }); + } + if (request.url.includes("/alerts?")) { + const query = new URL(request.url).searchParams; + if (query.has("from")) return Response.json([]); + const start = Number(query.get("start-index")); + return Response.json(historical.slice(start, start + 100)); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + }); + const alertQueries = requests.filter((request) => request.url.includes("/alerts?")); + const terminalInventoryQueries = alertQueries.filter((request) => { + const states = new URL(request.url).searchParams.getAll("states"); + return states.includes("PENDING") && states.includes("ACCEPTED") && states.includes("RESOLVED"); + }); + expect(terminalInventoryQueries.length).toBeGreaterThanOrEqual(2); + expect(terminalInventoryQueries.every((request) => + new URL(request.url).searchParams.get("start-index") === "0" + )).toBe(true); + expect(terminalInventoryQueries.every((request) => + new URL(request.url).searchParams.has("from") && new URL(request.url).searchParams.has("until") + )).toBe(true); + }); + + for (const status of ["UNKNOWN", undefined] as const) { + test(`finalizer fails closed when a matching alert has ${status ?? "no"} status`, async () => { + const service = canaryService({ existing: true, emptyListsAfterResolve: true }); + let alertListings = 0; + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "unknown", + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (request.url.includes("/alerts?")) { + alertListings += 1; + if (alertListings === 1) return Response.json([]); + if (alertListings === 2) { + return Response.json([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 98, + priority: "HIGH", + ...(status === undefined ? {} : { status }), + }]); + } + } + return service.fetchFn(input, init); + }, + })).rejects.toThrow("invalid status"); + expect(service.statuses).toEqual(["RESOLVED"]); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + )).toBe(true); + }); + } + + for (const status of ["UNKNOWN", undefined] as const) { + test(`primary and finalizer retain an exact ID after ${status ?? "missing"} detail status`, async () => { + let primaryDetailRead = false; + const primary = canaryService(); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (!primaryDetailRead && request.url.endsWith("/alerts/99")) { + primaryDetailRead = true; + return Response.json({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 99, + priority: "HIGH", + ...(status === undefined ? {} : { status }), + }); + } + return primary.fetchFn(input, init); + }, + })).rejects.toThrow("invalid status"); + expect(primary.requests.filter((request) => + request.method === "PUT" && request.url.endsWith("/alerts/99/resolve") + ).length).toBeGreaterThanOrEqual(1); + + let finalizerDetailRead = false; + const finalizer = canaryService({ existing: true }); + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (!finalizerDetailRead && request.url.endsWith("/alerts/98")) { + finalizerDetailRead = true; + return Response.json({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 98, + priority: "HIGH", + ...(status === undefined ? {} : { status }), + }); + } + return finalizer.fetchFn(input, init); + }, + })).rejects.toThrow("invalid status"); + expect(finalizer.requests.filter((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + ).length).toBeGreaterThanOrEqual(1); + }); + + test(`matching list ${status ?? "missing"} status retains the primary exact ID for cleanup`, async () => { + let invalidListingReturned = false; + const service = canaryService(); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (!invalidListingReturned && request.url.includes("/alerts?")) { + invalidListingReturned = true; + return Response.json([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 99, + priority: "HIGH", + ...(status === undefined ? {} : { status }), + }]); + } + return service.fetchFn(input, init); + }, + })).rejects.toThrow("invalid status"); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/99/resolve") + )).toBe(true); + expect(service.events.filter((event) => event.eventType === "RESOLVE")).toEqual([]); + }); + } + + for (const priority of ["UNKNOWN", undefined] as const) { + test(`primary retains the exact ID after ${priority ?? "missing"} list priority`, async () => { + let invalidListingReturned = false; + const service = canaryService({ eventResolveAlwaysFails: true }); + await expect(verifyIlertWebhookCanary({ + ...canaryOptions, + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (!invalidListingReturned && request.url.includes("/alerts?")) { + invalidListingReturned = true; + return Response.json([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 99, + ...(priority === undefined ? {} : { priority }), + status: "PENDING", + }]); + } + return service.fetchFn(input, init); + }, + })).rejects.toThrow("invalid priority"); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/99/resolve") + )).toBe(true); + }); + + test(`finalizer resolves a retained ID after ${priority ?? "missing"} priority despite Event API failure`, async () => { + let alertListings = 0; + const service = canaryService({ existing: true, eventResolveAlwaysFails: true }); + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "true", + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (request.url.includes("/alerts?")) { + alertListings += 1; + if (alertListings <= 2) return Response.json([]); + if (alertListings === 3) { + return Response.json([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: 98, + ...(priority === undefined ? {} : { priority }), + status: "PENDING", + }]); + } + } + return service.fetchFn(input, init); + }, + })).rejects.toThrow("invalid priority"); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + )).toBe(true); + }); + } + + for (const invalidField of ["status", "priority", "id"] as const) { + test(`continues past an invalid attempt-1 ${invalidField} to resolve a later attempt-2 alert`, async () => { + let invalidListingReturned = false; + const attemptOneKey = canaryAlertKey(RUN_ID, "1"); + const attemptTwoKey = canaryAlertKey(RUN_ID, "2"); + const service = canaryService({ + initialAlerts: [ + { alertKey: attemptOneKey, id: 98, priority: "HIGH", status: "PENDING" }, + { alertKey: attemptTwoKey, id: 99, priority: "HIGH", status: "PENDING" }, + ], + }); + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + runAttempt: "3", + sweepAttempt: "3", + fetchFn: async (input, init) => { + const request = new Request(input, init); + if (!invalidListingReturned && request.url.includes("/alerts?")) { + invalidListingReturned = true; + return Response.json([ + { + alertKey: attemptOneKey, + alertSource: { id: SOURCE_ID }, + ...(invalidField === "id" ? {} : { id: 98 }), + priority: invalidField === "priority" ? "UNKNOWN" : "HIGH", + status: invalidField === "status" ? "UNKNOWN" : "PENDING", + }, + { + alertKey: attemptTwoKey, + alertSource: { id: SOURCE_ID }, + id: 99, + priority: "HIGH", + status: "PENDING", + }, + ]); + } + return service.fetchFn(input, init); + }, + })).rejects.toThrow( + invalidField === "id" + ? "without an identity" + : `invalid ${invalidField}`, + ); + for (const id of invalidField === "status" || invalidField === "priority" + ? ["98", "99"] + : ["99"]) { + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith(`/alerts/${id}/resolve`) + )).toBe(true); + } + }); + } + + for (const alertSource of [undefined, { id: SOURCE_ID + 1 }] as const) { + test(`does not resolve a matching alert with a ${alertSource ? "mismatched" : "missing"} source while later trusted alerts clean`, async () => { + const requests: Request[] = []; + let currentTime = 0; + let trustedResolved = false; + const untrustedId = 98; + const trustedId = 99; + const attemptOneKey = canaryAlertKey(RUN_ID, "1"); + const attemptTwoKey = canaryAlertKey(RUN_ID, "2"); + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alerts/count") return Response.json({ count: 2 }); + if (url.pathname === "/api/alerts") { + return Response.json([ + { + alertKey: attemptOneKey, + ...(alertSource ? { alertSource } : {}), + id: untrustedId, + priority: "HIGH", + status: "PENDING", + }, + { + alertKey: attemptTwoKey, + alertSource: { id: SOURCE_ID }, + id: trustedId, + priority: "HIGH", + status: trustedResolved ? "RESOLVED" : "PENDING", + }, + ]); + } + if (request.method === "PUT" && url.pathname === `/api/alerts/${trustedId}/resolve`) { + trustedResolved = true; + return new Response(null, { status: 200 }); + } + if (url.pathname === `/api/alerts/${trustedId}`) { + return Response.json({ + alertKey: attemptTwoKey, + alertSource: { id: SOURCE_ID }, + id: trustedId, + priority: "HIGH", + status: trustedResolved ? "RESOLVED" : "PENDING", + }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + + await expect(finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + runAttempt: "3", + sweepAttempt: "3", + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + })).rejects.toThrow(alertSource ? "different source" : "without a valid source"); + expect(requests.some((request) => + request.method === "PUT" && request.url.endsWith(`/alerts/${untrustedId}/resolve`) + )).toBe(false); + expect(requests.some((request) => + request.method === "PUT" && request.url.endsWith(`/alerts/${trustedId}/resolve`) + )).toBe(true); + expect(trustedResolved).toBe(true); + }); + } + + test("recovers from one incomplete terminal inventory after two complete empty scans", async () => { + let currentTime = 0; + let terminalFailures = 3; + const service = canaryService(); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn: async (input, init) => { + const request = new Request(input, init); + if ( + request.url.includes("/alerts?") && + currentTime >= CANARY_FINALIZER_BUDGETS.discoveryMs && + terminalFailures > 0 + ) { + terminalFailures -= 1; + return new Response(null, { status: 503 }); + } + return service.fetchFn(input, init); + }, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + expect(terminalFailures).toBe(0); + expect(currentTime).toBeGreaterThanOrEqual(100_000); + }); + + test("requires matching all-state counts and repeated inventories before terminal success", async () => { + let currentTime = 0; + let terminalCountCalls = 0; + const service = canaryService(); + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn: async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if ( + url.pathname === "/api/alerts/count" && + currentTime >= CANARY_FINALIZER_BUDGETS.discoveryMs + ) { + terminalCountCalls += 1; + if (terminalCountCalls === 2) return Response.json({ count: 1 }); + } + return service.fetchFn(input, init); + }, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + expect(terminalCountCalls).toBeGreaterThanOrEqual(6); + const terminalLists = service.requests.filter((request) => { + const url = new URL(request.url); + return url.pathname === "/api/alerts" && + url.searchParams.getAll("states").join(",") === "PENDING,ACCEPTED,RESOLVED"; + }); + expect(terminalLists.length).toBeGreaterThanOrEqual(4); + }); + + test("retains a late-page canary when continuity revalidation observes churn", async () => { + let currentTime = 0; + let terminalLists = 0; + let resolved = false; + const canaryId = 202; + const baseline = Array.from({ length: 100 }, (_, index) => ({ + alertKey: `unrelated-${index + 1}`, + alertSource: { id: SOURCE_ID }, + id: index + 1, + priority: "HIGH", + status: "RESOLVED", + })); + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + if (url.pathname.startsWith("/api/alert-sources/")) return Response.json(SOURCE); + if (url.pathname === "/api/alerts/count") return Response.json({ count: 101 }); + if (request.method === "PUT" && url.pathname === `/api/alerts/${canaryId}/resolve`) { + resolved = true; + return new Response(null, { status: 200 }); + } + if (url.pathname === `/api/alerts/${canaryId}`) { + return Response.json({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: canaryId, + priority: "HIGH", + status: resolved ? "RESOLVED" : "PENDING", + }); + } + if (url.pathname === "/api/alerts") { + const states = url.searchParams.getAll("states"); + if (!states.includes("RESOLVED")) return Response.json([]); + terminalLists += 1; + const start = Number(url.searchParams.get("start-index")); + if (start === 0) return Response.json(baseline); + const churned = terminalLists === 4; + if (churned || resolved) { + return Response.json([{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + alertSource: { id: SOURCE_ID }, + id: canaryId, + priority: "HIGH", + status: resolved ? "RESOLVED" : "PENDING", + }]); + } + return Response.json([{ + alertKey: "unrelated-101", + alertSource: { id: SOURCE_ID }, + id: 101, + priority: "HIGH", + status: "RESOLVED", + }]); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + await finalizeIlertWebhookCanary({ + ...finalizerOptions(), + alertSubmitted: "cleaned", + fetchFn, + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }); + expect(terminalLists).toBeGreaterThanOrEqual(8); + expect(resolved).toBe(true); + }); + + test("finalizer CLI is cleanup-only and never reads or mutates alert actions", async () => { + const service = canaryService({ existing: true, staleOpenReadsAfterResolve: 2, status: "PENDING" }); + await runCli({ + args: ["--finalize-canary"], + env: cliEnvironment({ POSTIL_ILERT_RECEIVER_ORIGIN: "not-a-url", POSTIL_ILERT_WEBHOOK_SECRET: "short" }), + fetchFn: service.fetchFn, + ...finalizerTiming(), + log: () => undefined, + }); + expect(service.requests.some((request) => request.url.includes("/alert-actions"))).toBe(false); + expect(service.events.some((event) => event.eventType === "ALERT")).toBe(false); + expect(service.requests.some((request) => request.method === "PUT" && /\/alerts\/[1-9][0-9]*\/resolve$/u.test(new URL(request.url).pathname))).toBe(true); + }); + + test("cleaned finalizer CLI loads credentials and performs management cleanup", async () => { + const service = canaryService(); + await runCli({ + args: ["--finalize-canary"], + env: cliEnvironment({ POSTIL_ILERT_CANARY_ALERT_SUBMITTED: "cleaned" }), + fetchFn: service.fetchFn, + ...finalizerTiming(), + log: () => undefined, + }); + expect(service.alertListRequests).toBeGreaterThan(0); + expect(service.events).toEqual([]); + }); + + test("orphan sweep CLI requires only scoped management cleanup inputs", async () => { + const service = canaryService({ + initialAlerts: [{ + alertKey: canaryAlertKey("67890", "2"), + id: 98, + status: "PENDING", + }], + }); + await runCli({ + args: ["--sweep-canary-orphans"], + env: { + ILERT_API_KEY: API_KEY, + POSTIL_ILERT_ALERT_SOURCE_ID: String(SOURCE_ID), + }, + fetchFn: service.fetchFn, + now: () => 0, + sleep: async () => undefined, + log: () => undefined, + }); + expect(service.events).toEqual([]); + expect(service.requests.some((request) => + request.method === "PUT" && request.url.endsWith("/alerts/98/resolve") + )).toBe(true); + }); + + test("rejects live reconciliation without a recoverable run identity before network access", async () => { + await expect(runCli({ + args: [], + env: {}, + fetchFn: async () => { throw new Error("network access is forbidden"); }, + log: () => undefined, + })).rejects.toThrow("live reconciliation requires --canary"); + }); +}); + +function cliEnvironment(overrides: Record = {}): Record { + return { + ILERT_API_KEY: API_KEY, + ILERT_INTEGRATION_KEY: INTEGRATION_KEY, + POSTIL_ILERT_ALERT_SOURCE_ID: String(SOURCE_ID), + POSTIL_ILERT_CANARY_RUN_ATTEMPT: RUN_ATTEMPT, + POSTIL_ILERT_CANARY_RUN_ID: RUN_ID, + POSTIL_ILERT_RECEIVER_ORIGIN: RECEIVER_ORIGIN, + POSTIL_ILERT_WEBHOOK_SECRET: WEBHOOK_SECRET, + ...overrides, + }; +} + +function finalizerTiming(): { now: () => number; sleep: (milliseconds: number) => Promise } { + let currentTime = 0; + return { + now: () => currentTime, + sleep: async (milliseconds) => { currentTime += milliseconds; }, + }; +} + +function finalizerOptions() { + return { + ...canaryOptions, + ...finalizerTiming(), + }; +} + +function canaryService(options: { + alertReportTime?: string; + alertRetryAfter?: string; + deduplicateAlertKeys?: boolean; + deferFirstAlertCreation?: boolean; + deferredListings?: number; + detailPendingReads?: number; + detailAlwaysPending?: boolean; + dropAlertCreation?: boolean; + dropResolvedReceiverEvent?: boolean; + emptyListsAfterResolve?: boolean; + eventResolveAlwaysFails?: boolean; + existing?: boolean; + failAlertListings?: number; + initialAlerts?: Array<{ + alertKey?: string; + id: number; + priority?: "HIGH" | "LOW"; + sourceId?: number; + status: "PENDING" | "ACCEPTED" | "RESOLVED"; + }>; + lateAlertAtListing?: number; + managementFailsAfterAlert?: boolean; + mainPriority?: "HIGH" | "LOW"; + observedSourceId?: number; + receiverObservationFailures?: number; + releaseDeferredAlertAfterResolved?: boolean; + resolveRequestFailures?: number; + sourceIntegrationType?: string; + staleOpenReadsAfterResolve?: number; + status?: "PENDING" | "ACCEPTED" | "RESOLVED"; +} = {}) { + const events: Array<{ alertKey: string; eventType: string }> = []; + const requests: Request[] = []; + const alerts: Array<{ + alertKey: string; + createdReceived: boolean; + id: number; + priority: "HIGH" | "LOW"; + reportTime?: string; + resolvedReceived: boolean; + sourceId: number; + status: "PENDING" | "ACCEPTED" | "RESOLVED"; + }> = options.initialAlerts + ? options.initialAlerts.map((alert) => ({ + alertKey: alert.alertKey ?? canaryAlertKey(RUN_ID, RUN_ATTEMPT), + createdReceived: true, + id: alert.id, + priority: alert.priority ?? "HIGH", + reportTime: undefined, + resolvedReceived: alert.status === "RESOLVED", + sourceId: alert.sourceId ?? SOURCE_ID, + status: alert.status, + })) + : options.existing + ? [{ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + createdReceived: true, + id: 98, + priority: options.mainPriority ?? "HIGH", + reportTime: undefined, + resolvedReceived: options.status === "RESOLVED", + sourceId: SOURCE_ID, + status: options.status ?? "PENDING", + }] + : []; + let alertSent = false; + let deferredAlert: { alertKey: string; priority: "HIGH" | "LOW"; reportTime?: string } | undefined; + let alertRetryPending = options.alertRetryAfter !== undefined; + let alertListRequests = 0; + let failAlertListings = options.failAlertListings ?? 0; + let detailPendingReads = options.detailPendingReads ?? 0; + let lateAlertAdded = false; + let receiverObservationFailures = options.receiverObservationFailures ?? 0; + let receiverObservationRequests = 0; + let resolveRequestFailures = options.resolveRequestFailures ?? 0; + let staleOpenReadsAfterResolve = options.staleOpenReadsAfterResolve ?? 0; + + const fetchFn: Fetch = async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + if (new URL(request.url).pathname.startsWith("/api/alert-sources/")) { + return Response.json({ + ...SOURCE, + integrationType: options.sourceIntegrationType ?? SOURCE.integrationType, + }); + } + if (request.method === "POST" && request.url.endsWith("/events")) { + const body = await request.json() as { + alertKey: string; + eventType: string; + priority?: "HIGH" | "LOW"; + }; + events.push({ alertKey: body.alertKey, eventType: body.eventType }); + if (body.eventType === "ALERT" && alertRetryPending) { + alertRetryPending = false; + return new Response(null, { + status: 429, + headers: { "retry-after": options.alertRetryAfter! }, + }); + } + if ( + body.eventType === "ALERT" && + options.deferFirstAlertCreation && + !deferredAlert + ) { + deferredAlert = { + alertKey: body.alertKey, + priority: options.mainPriority ?? body.priority ?? "HIGH", + reportTime: options.alertReportTime, + }; + } else if ( + body.eventType === "ALERT" && + !options.dropAlertCreation && + (!options.deduplicateAlertKeys || !alerts.some((alert) => alert.alertKey === body.alertKey)) + ) { + alertSent = true; + const id = alerts.length === 0 + ? 99 + : Math.max(...alerts.map((alert) => alert.id)) + 1; + alerts.push({ + alertKey: body.alertKey, + createdReceived: true, + id, + priority: options.mainPriority ?? body.priority ?? "HIGH", + reportTime: options.alertReportTime, + resolvedReceived: false, + sourceId: SOURCE_ID, + status: options.status === "ACCEPTED" ? "ACCEPTED" : "PENDING", + }); + } + if (body.eventType === "RESOLVE") { + if (options.eventResolveAlwaysFails) return new Response(null, { status: 503 }); + if (resolveRequestFailures > 0) { + resolveRequestFailures -= 1; + return new Response(null, { status: 503 }); + } + for (const alert of alerts.filter((item) => + item.alertKey === body.alertKey && item.status !== "RESOLVED" + )) { + alert.status = "RESOLVED"; + if (!options.dropResolvedReceiverEvent) alert.resolvedReceived = true; + } + } + return new Response(null, { status: 202 }); + } + const resolveId = /\/alerts\/([1-9][0-9]*)\/resolve$/u.exec(new URL(request.url).pathname)?.[1]; + if (request.method === "PUT" && resolveId) { + const alert = alerts.find((item) => String(item.id) === resolveId); + if (!alert) return new Response(null, { status: 404 }); + if (resolveRequestFailures > 0) { + resolveRequestFailures -= 1; + return new Response(null, { status: 503 }); + } + alert.status = "RESOLVED"; + if (!options.dropResolvedReceiverEvent) alert.resolvedReceived = true; + return new Response(null, { status: 200 }); + } + const detailId = /\/alerts\/([1-9][0-9]*)$/u.exec(new URL(request.url).pathname)?.[1]; + if (detailId) { + const alert = alerts.find((item) => String(item.id) === detailId); + if (!alert) return Response.json({ error: "not found" }, { status: 404 }); + const status = options.detailAlwaysPending + ? "PENDING" + : detailPendingReads > 0 + ? (detailPendingReads -= 1, "PENDING" as const) + : alert.status; + return Response.json({ + alertKey: alert.alertKey, + alertSource: { id: alert.sourceId }, + id: alert.id, + priority: alert.priority, + status, + }); + } + if (new URL(request.url).pathname === "/api/alerts/count") { + const query = new URL(request.url).searchParams; + const states = query.getAll("states"); + const from = query.get("from"); + const source = query.get("sources"); + const until = query.get("until"); + const count = alerts.filter((alert) => { + if (alert.reportTime && ((from && alert.reportTime < from) || (until && alert.reportTime > until))) return false; + if (source && source !== String(alert.sourceId)) return false; + return states.length === 0 || states.includes(alert.status); + }).length; + return Response.json({ count }); + } + if (request.url.includes("/alerts?")) { + alertListRequests += 1; + if (failAlertListings > 0) { + failAlertListings -= 1; + return new Response(null, { status: 503 }); + } + if (options.managementFailsAfterAlert && alertSent) { + return new Response(null, { status: 503 }); + } + if ( + deferredAlert && + options.releaseDeferredAlertAfterResolved && + alerts.length > 0 && + alerts.every((alert) => alert.status === "RESOLVED") + ) { + const id = Math.max(...alerts.map((alert) => alert.id)) + 1; + alerts.push({ + alertKey: deferredAlert.alertKey, + createdReceived: true, + id, + priority: deferredAlert.priority, + reportTime: deferredAlert.reportTime, + resolvedReceived: false, + sourceId: SOURCE_ID, + status: "PENDING", + }); + deferredAlert = undefined; + } + if (alertListRequests <= (options.deferredListings ?? 0)) { + return Response.json([]); + } + if ( + options.lateAlertAtListing === alertListRequests && + !lateAlertAdded + ) { + lateAlertAdded = true; + alerts.push({ + alertKey: canaryAlertKey(RUN_ID, RUN_ATTEMPT), + createdReceived: true, + id: 199, + priority: "HIGH", + reportTime: undefined, + resolvedReceived: false, + sourceId: SOURCE_ID, + status: "PENDING", + }); + } + const stale = staleOpenReadsAfterResolve > 0; + if (stale) staleOpenReadsAfterResolve -= 1; + const query = new URL(request.url).searchParams; + const states = query.getAll("states"); + const from = query.get("from"); + const source = query.get("sources"); + const until = query.get("until"); + return Response.json(alerts.flatMap((alert) => { + const status = stale && alert.status === "RESOLVED" ? "PENDING" : alert.status; + if (options.emptyListsAfterResolve && alert.status === "RESOLVED") return []; + if (alert.reportTime && ((from && alert.reportTime < from) || (until && alert.reportTime > until))) return []; + if (source && source !== String(alert.sourceId)) return []; + if (states.length > 0 && !states.includes(status)) return []; + return [{ + alertKey: alert.alertKey, + alertSource: { id: alert.sourceId }, + id: alert.id, + priority: alert.priority, + status, + }]; + })); + } + if (request.url.startsWith(`${RECEIVER_ORIGIN}/api/webhooks/ilert?`)) { + receiverObservationRequests += 1; + if (receiverObservationFailures > 0) { + receiverObservationFailures -= 1; + return new Response(null, { status: 503 }); + } + const query = new URL(request.url).searchParams; + const alert = alerts.find((item) => item.alertKey === query.get("alertKey")); + const received = query.get("sourceId") === String(options.observedSourceId ?? SOURCE_ID) && + (query.get("eventType") === "alert-created" + ? alert?.createdReceived === true + : alert?.resolvedReceived === true); + return Response.json({ received }); + } + if (request.url.endsWith("/actions")) { + return Response.json({ + alertActionId: "72", + history: [{ alertActionId: "72", alertId: 99, id: "untyped", success: true }], + }); + } + throw new Error(`unexpected request: ${request.method} ${request.url}`); + }; + + return { + events, + fetchFn, + requests, + get alertListRequests() { + return alertListRequests; + }, + get receiverObservationRequests() { + return receiverObservationRequests; + }, + get statuses() { + return alerts.map((alert) => alert.status); + }, + }; +} + +function queuedFetch(requests: Request[], responses: Response[]): Fetch { + const actionInventory = new Map(); + return async (input, init) => { + const request = new Request(input, init); + requests.push(request.clone()); + const url = new URL(request.url); + const actionList = url.pathname === "/api/alert-actions"; + if (url.pathname.startsWith("/api/alert-actions") && request.method !== "GET") { + actionInventory.clear(); + } + const cached = actionList ? actionInventory.get(request.url) : undefined; + if (cached) return cached.clone(); + const response = responses.shift(); + if (!response) throw new Error("unexpected request"); + if (actionList) actionInventory.set(request.url, response.clone()); + return response; + }; +} + +function errorMessages(error: unknown, seen = new Set()): string[] { + if (typeof error === "string") return [error]; + if (!error || typeof error !== "object" || seen.has(error)) return []; + seen.add(error); + if (error instanceof AggregateError) { + return [ + error.message, + ...error.errors.flatMap((nested) => errorMessages(nested, seen)), + ]; + } + if (error instanceof Error) { + return [error.message, ...errorMessages(error.cause, seen)]; + } + return []; +} diff --git a/tests/release-database-url.test.ts b/tests/release-database-url.test.ts index d3d077d3..d8c7fc54 100644 --- a/tests/release-database-url.test.ts +++ b/tests/release-database-url.test.ts @@ -191,7 +191,9 @@ describe("release database connection", () => { await runReleaseMigrations( environment, (command) => { - events.push("provider preflight"); + events.push(command.includes("hosted:verify-provider") + ? "provider preflight" + : "managed indexes"); commands.push([...command]); return { exited: Promise.resolve(0) }; }, @@ -203,14 +205,42 @@ describe("release database connection", () => { return true; }, ); - expect(commands).toEqual([["bun", "run", "hosted:verify-provider"]]); + expect(commands).toEqual([ + ["bun", "run", "hosted:verify-provider"], + ["bun", "run", "operational:indexes", "--managed-release"], + ]); expect(events).toEqual([ "compatibility verified", "provider preflight", + "managed indexes", "release prepared", ]); }); + test("stops a managed release before preparation when its index build fails", async () => { + let prepared = false; + await expect( + runReleaseMigrations( + { + POSTIL_MANAGED_RELEASE: "1", + DATABASE_URL: "postgresql://postil@db.internal:5432/postil", + POSTIL_RELEASE_SHA: "a".repeat(40), + POSTIL_COMPATIBLE_SOURCE_RELEASE_SHA: "b".repeat(40), + POSTIL_RELEASE_PROTOCOL: "additive-publication-hosted-v1", + }, + (command) => ({ + exited: Promise.resolve(command.includes("operational:indexes") ? 1 : 0), + }), + async () => undefined, + async () => { + prepared = true; + return true; + }, + ), + ).rejects.toThrow("release managed operational indexes failed with status 1"); + expect(prepared).toBe(false); + }); + test("rejects incompatibility and provider failure before preparation", async () => { const environment = { POSTIL_MANAGED_RELEASE: "1", @@ -354,7 +384,7 @@ describe("release database connection", () => { 'recovery_target_sha="${recovery_targets[0]}"', ); expect(productionMonitorWorkflow).toContain( - "needs: [smoke, release-recovery]", + "needs: [deployment-trust, smoke, release-recovery]", ); expect(productionMonitorWorkflow).toContain( "Postil release recovery failed", diff --git a/tests/workflow-trust.test.ts b/tests/workflow-trust.test.ts new file mode 100644 index 00000000..076c2999 --- /dev/null +++ b/tests/workflow-trust.test.ts @@ -0,0 +1,422 @@ +import { describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse } from "yaml"; + +const root = join(import.meta.dir, ".."); + +describe("deployment workflow trust boundaries", () => { + test("admits only manual deployments and exposes one deploy job", () => { + const workflow = parse( + readFileSync(join(root, ".github/workflows/deploy.yml"), "utf8"), + ) as { + on: Record; + jobs: Record; + }; + + expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); + expect(Object.keys(workflow.jobs)).toEqual(["deploy"]); + expect(workflow.jobs.deploy!.name).toBeUndefined(); + expect(workflow.jobs.deploy!.if).toBe( + "vars.FLY_DEPLOY_ENABLED == 'true' && github.event_name == 'workflow_dispatch'", + ); + }); + + test("derives downstream trust from the triggering deploy attempt's jobs", () => { + const workflow = parse( + readFileSync(join(root, ".github/workflows/production-monitor.yml"), "utf8"), + ) as { + permissions: Record; + jobs: { + "deployment-trust": { + permissions: Record; + "timeout-minutes": number; + steps: Array<{ + id?: string; + env?: Record; + run?: string; + "timeout-minutes"?: number; + }>; + }; + }; + }; + const deployWorkflow = parse( + readFileSync(join(root, ".github/workflows/deploy.yml"), "utf8"), + ) as { jobs: Record }; + const trustJob = workflow.jobs["deployment-trust"]; + const evidenceStep = trustJob.steps.find((step) => step.id === "evidence"); + const run = evidenceStep?.run; + expect(workflow.permissions).toEqual({}); + expect(trustJob.permissions).toEqual({ actions: "read" }); + expect(trustJob["timeout-minutes"]).toBe(5); + expect(evidenceStep?.["timeout-minutes"]).toBe(3); + expect(JSON.stringify(trustJob)).not.toContain("secrets."); + expect(JSON.stringify(trustJob)).not.toContain("id-token"); + expect(run).toBeDefined(); + // GitHub reports a job without a display name under its job identifier. + expect(Object.keys(deployWorkflow.jobs)).toEqual(["deploy"]); + expect(deployWorkflow.jobs.deploy!.name).toBeUndefined(); + expect(run).toContain('.name == "deploy"'); + expect(evidenceStep?.env?.DEPLOY_HEAD_BRANCH).toBe( + "${{ github.event.workflow_run.head_branch }}", + ); + expect(run).toContain("timeout --signal=TERM --kill-after=5s 30s gh api"); + expect(run).toContain("for api_attempt in 1 2 3"); + expect(run).toContain("sleep 2"); + + const temporaryDirectory = mkdtempSync(join(tmpdir(), "postil-workflow-trust-")); + const ghPath = join(temporaryDirectory, "gh"); + const outputPath = join(temporaryDirectory, "output"); + const callsPath = join(temporaryDirectory, "calls"); + const callCountPath = join(temporaryDirectory, "call-count"); + const delaysPath = join(temporaryDirectory, "delays"); + const sleepPath = join(temporaryDirectory, "sleep"); + writeFileSync( + ghPath, + `#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "${callsPath}" +call_count="$(<"${callCountPath}")" +call_count="$((call_count + 1))" +printf '%s\n' "\${call_count}" > "${callCountPath}" +if (( call_count <= MOCK_GH_FAILURES )); then + exit 75 +fi +path="\${!#}" +if [[ ! "\${path}" =~ /attempts/([0-9]+)/jobs\\?per_page=100$ ]]; then + exit 2 +fi +jq -cer --arg attempt "\${BASH_REMATCH[1]}" '.[$attempt] // error("missing mock attempt")' <<<"\${MOCK_ATTEMPTS_JSON}" +`, + ); + chmodSync(ghPath, 0o755); + writeFileSync( + sleepPath, + `#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "${delaysPath}" +`, + ); + chmodSync(sleepPath, 0o755); + + const evaluate = ({ + attempts, + deployEvent = "workflow_dispatch", + deployHeadBranch = "main", + eventName = "workflow_run", + eventRef = "refs/heads/main", + ghFailures = 0, + reconcileAlertStream = "", + runAttempt = 1, + testAlert = "", + }: { + attempts?: Record }>; + deployEvent?: string; + deployHeadBranch?: string; + eventName?: string; + eventRef?: string; + ghFailures?: number; + reconcileAlertStream?: string; + runAttempt?: number; + testAlert?: string; + }) => { + writeFileSync(outputPath, ""); + writeFileSync(callsPath, ""); + writeFileSync(callCountPath, "0\n"); + writeFileSync(delaysPath, ""); + const attemptEvidence = attempts ?? { + "1": { + total_count: 1, + jobs: [{ name: "deploy", conclusion: "success" }], + }, + }; + const result = spawnSync("bash", ["-c", run!], { + encoding: "utf8", + env: { + DEPLOY_EVENT: deployEvent, + DEPLOY_HEAD_BRANCH: deployHeadBranch, + DEPLOY_RUN_ATTEMPT: String(runAttempt), + DEPLOY_RUN_ID: "123", + EVENT_NAME: eventName, + EVENT_REF: eventRef, + GH_TOKEN: "test-token", + GITHUB_OUTPUT: outputPath, + GITHUB_REPOSITORY: "postil-dev/postil", + MOCK_ATTEMPTS_JSON: JSON.stringify(attemptEvidence), + MOCK_GH_FAILURES: String(ghFailures), + NODE_ENV: "test", + PATH: `${temporaryDirectory}:${process.env.PATH}`, + RECONCILE_ALERT_STREAM: reconcileAlertStream, + TEST_ALERT: testAlert, + } as NodeJS.ProcessEnv, + }); + const outputs = Object.fromEntries( + readFileSync(outputPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => line.split("=", 2) as [string, string]), + ); + const calls = readFileSync(callsPath, "utf8").trim().split("\n").filter(Boolean); + const delays = readFileSync(delaysPath, "utf8").trim().split("\n").filter(Boolean); + return { calls, delays, outputs, status: result.status, stderr: result.stderr }; + }; + const evidence = (deployConclusion: string) => ({ + "1": { + total_count: 1, + jobs: [{ name: "deploy", conclusion: deployConclusion }], + }, + }); + + try { + // The finalizer resolves the key the operator test alert uses, so one + // dispatch may request either manual alert path but never both. + expect(evidenceStep?.env).toMatchObject({ + RECONCILE_ALERT_STREAM: "${{ inputs.reconcile_alert_stream }}", + TEST_ALERT: "${{ inputs.test_alert }}", + }); + const conflictingInputs = evaluate({ + eventName: "workflow_dispatch", + reconcileAlertStream: "true", + testAlert: "true", + }); + expect(conflictingInputs).toMatchObject({ + status: 1, + outputs: { conclusion: "", trusted: "false" }, + calls: [], + }); + expect(conflictingInputs.stderr).toContain( + "test_alert and reconcile_alert_stream cannot be combined", + ); + for (const [testAlert, reconcileAlertStream] of [ + ["true", "false"], + ["false", "true"], + ["false", "false"], + ] as const) { + expect(evaluate({ + eventName: "workflow_dispatch", + reconcileAlertStream, + testAlert, + })).toMatchObject({ status: 0, outputs: { trusted: "true" } }); + } + expect(evaluate({ + eventName: "workflow_dispatch", + eventRef: "refs/heads/feature", + reconcileAlertStream: "true", + testAlert: "true", + }).outputs.trusted).toBe("false"); + // Scheduled and deployment-triggered runs carry no inputs. + expect(evaluate({ eventName: "schedule" })).toMatchObject({ + status: 0, + outputs: { trusted: "true" }, + }); + + for (const conclusion of ["success", "failure", "cancelled", "timed_out"]) { + expect(evaluate({ attempts: evidence(conclusion) })).toMatchObject({ + status: 0, + outputs: { conclusion, trusted: "true" }, + }); + } + expect(evaluate({ attempts: evidence("skipped") })).toMatchObject({ + status: 0, + outputs: { conclusion: "skipped", trusted: "false" }, + }); + + const rerun = evaluate({ + runAttempt: 2, + attempts: { + "1": { + total_count: 1, + jobs: [{ name: "deploy", conclusion: "failure" }], + }, + "2": { + total_count: 1, + jobs: [{ name: "deploy", conclusion: "cancelled" }], + }, + }, + }); + expect(rerun).toMatchObject({ + status: 0, + outputs: { conclusion: "cancelled", trusted: "true" }, + }); + expect(rerun.calls).toEqual([ + "api --method GET repos/postil-dev/postil/actions/runs/123/attempts/2/jobs?per_page=100", + ]); + + const transientApiFailure = evaluate({ ghFailures: 2 }); + expect(transientApiFailure).toMatchObject({ + status: 0, + outputs: { conclusion: "success", trusted: "true" }, + delays: ["2", "2"], + }); + expect(transientApiFailure.calls).toHaveLength(3); + + const exhaustedApiFailure = evaluate({ ghFailures: 3 }); + expect(exhaustedApiFailure.status).not.toBe(0); + expect(exhaustedApiFailure.outputs.trusted).toBe("false"); + expect(exhaustedApiFailure.calls).toHaveLength(3); + expect(exhaustedApiFailure.delays).toEqual(["2", "2"]); + expect(exhaustedApiFailure.stderr).toContain( + "Deployment job evidence remained unavailable after bounded retries.", + ); + + const duplicate = evaluate({ + attempts: { + "1": { + total_count: 2, + jobs: [ + { name: "deploy", conclusion: "success" }, + { name: "deploy", conclusion: "failure" }, + ], + }, + }, + }); + expect(duplicate.status).not.toBe(0); + expect(duplicate.outputs.trusted).toBe("false"); + + const missingDeployment = evaluate({ + attempts: { + "1": { + total_count: 1, + jobs: [{ name: "Deploy production", conclusion: "success" }], + }, + }, + }); + expect(missingDeployment.status).not.toBe(0); + expect(missingDeployment.outputs.trusted).toBe("false"); + + const truncated = evaluate({ + attempts: { + "1": { + total_count: 2, + jobs: [{ name: "deploy", conclusion: "success" }], + }, + }, + }); + expect(truncated.status).not.toBe(0); + expect(truncated.outputs.trusted).toBe("false"); + + for (const deployEvent of ["pull_request", "push", "workflow_run"]) { + expect(evaluate({ deployEvent })).toMatchObject({ + calls: [], + status: 0, + outputs: { conclusion: "", trusted: "false" }, + }); + } + expect(evaluate({ deployHeadBranch: "feature" })).toMatchObject({ + calls: [], + status: 0, + outputs: { conclusion: "", trusted: "false" }, + }); + expect(evaluate({ + eventName: "workflow_dispatch", + eventRef: "refs/heads/main", + })).toMatchObject({ calls: [], status: 0, outputs: { conclusion: "", trusted: "true" } }); + expect(evaluate({ + eventName: "workflow_dispatch", + eventRef: "refs/heads/feature", + })).toMatchObject({ calls: [], status: 0, outputs: { conclusion: "", trusted: "false" } }); + expect(evaluate({ eventName: "schedule" })).toMatchObject({ + calls: [], + status: 0, + outputs: { conclusion: "", trusted: "true" }, + }); + } finally { + rmSync(temporaryDirectory, { recursive: true, force: true }); + } + }); + + test("gates every production credential path on deployment evidence", () => { + const source = readFileSync( + join(root, ".github/workflows/production-monitor.yml"), + "utf8", + ); + const workflow = parse(source) as { + jobs: Record; + }>; + }; + const protectedJobs = Object.entries(workflow.jobs).filter(([, job]) => + JSON.stringify(job).includes("secrets.") || + job.steps?.some((step) => + step.uses?.startsWith("Infisical/secrets-action@") || + step.uses === "./.github/actions/ilert-event" + ) + ); + + expect(protectedJobs.map(([name]) => name).sort()).toEqual([ + "alert-stream", + "alert-stream-finalize", + "canary-orphan-sweep", + "notify", + "release-recovery", + "resolve", + "resolve-release-recovery", + "resolve-test-alert", + "smoke", + ]); + for (const [, job] of protectedJobs) { + const needs = Array.isArray(job.needs) ? job.needs : [job.needs]; + expect(needs).toContain("deployment-trust"); + expect(job.if).toContain("needs.deployment-trust.outputs.trusted == 'true'"); + } + expect(source).not.toContain("github.event.workflow_run.conclusion"); + }); + + test("runs the orphan canary sweep only after an explicit repository opt-in", () => { + const workflow = parse( + readFileSync(join(root, ".github/workflows/production-monitor.yml"), "utf8"), + ) as { + jobs: Record; + }>; + }; + const optIn = "vars.POSTIL_ILERT_CANARY_SWEEP_ENABLED"; + expect(workflow.jobs["canary-orphan-sweep"]!.if).toContain( + `needs.deployment-trust.outputs.trusted == 'true' && ${optIn} == 'true' && `, + ); + // Jobs that exist on the released monitor stay independent of the opt-in. + for (const name of ["smoke", "release-recovery", "notify", "resolve", "resolve-release-recovery"]) { + expect(workflow.jobs[name]!.if).not.toContain(optIn); + } + + const report = workflow.jobs["deployment-trust"]!.steps.find( + (step) => step.name === "Report the orphaned iLert canary sweep state", + ); + expect(report?.uses).toBeUndefined(); + expect(report?.if).toBe( + `\${{ steps.evidence.outputs.trusted == 'true' && github.event_name != 'workflow_run' && ${optIn} != 'true' }}`, + ); + const summaryDirectory = mkdtempSync(join(tmpdir(), "postil-sweep-state-")); + try { + const summaryPath = join(summaryDirectory, "summary"); + writeFileSync(summaryPath, ""); + const result = spawnSync("bash", ["-c", report!.run!], { + encoding: "utf8", + env: { + GITHUB_STEP_SUMMARY: summaryPath, + NODE_ENV: "test", + PATH: process.env.PATH, + } as NodeJS.ProcessEnv, + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("::notice title=Orphaned iLert canary sweep is disabled::"); + expect(readFileSync(summaryPath, "utf8")).toContain( + "makes no iLert management request and resolves no orphaned canary alert", + ); + } finally { + rmSync(summaryDirectory, { recursive: true, force: true }); + } + }); +});