From fe007e2efc72b2eef7ec923b269b3bede0ec4dc4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 02:51:49 +0100 Subject: [PATCH 01/11] feat(dns): adopt the zone's DNS records through a guarded plan and apply Declare the records this repository owns in dns.tf and add a main-only "DNS plan and guarded apply" workflow against the authoritative R2 state. It imports a declared record only when exactly one unowned live record matches it, refuses deletes, replacements and changing imports, encrypts state before and after every apply, and proves a clean plan from a fresh runner. External-dns records, including the SES bounce records, stay with external-dns. The email-only import proposal is superseded and removed. Part of #21. Part of #65. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dns-apply.yaml | 153 ++++++++++++++++ .github/workflows/email-dns-proposal.yaml | 84 --------- README.md | 9 +- dns.tf | 164 +++++++++++++++++ docs/email-dns-proposal.md | 32 ---- docs/runbooks/bootstrap-and-recovery.md | 52 ++++++ scripts/apply-dns.sh | 213 ++++++++++++++++++++++ scripts/check-dns-plan.jq | 26 +++ scripts/check-email-dns-plan.jq | 33 ---- scripts/dns-imports.jq | 29 +++ scripts/email-dns-proposal.jq | 79 -------- scripts/plan-email-dns.sh | 121 ------------ scripts/test.sh | 3 +- scripts/tests/test-dns-apply.sh | 101 ++++++++++ scripts/tests/test-email-dns-plan.sh | 168 ----------------- scripts/tests/test-email-dns-proposal.sh | 105 ----------- 16 files changed, 746 insertions(+), 626 deletions(-) create mode 100644 .github/workflows/dns-apply.yaml delete mode 100644 .github/workflows/email-dns-proposal.yaml create mode 100644 dns.tf delete mode 100644 docs/email-dns-proposal.md create mode 100755 scripts/apply-dns.sh create mode 100644 scripts/check-dns-plan.jq delete mode 100644 scripts/check-email-dns-plan.jq create mode 100644 scripts/dns-imports.jq delete mode 100644 scripts/email-dns-proposal.jq delete mode 100644 scripts/plan-email-dns.sh create mode 100755 scripts/tests/test-dns-apply.sh delete mode 100644 scripts/tests/test-email-dns-plan.sh delete mode 100644 scripts/tests/test-email-dns-proposal.sh diff --git a/.github/workflows/dns-apply.yaml b/.github/workflows/dns-apply.yaml new file mode 100644 index 0000000..f5bd76b --- /dev/null +++ b/.github/workflows/dns-apply.yaml @@ -0,0 +1,153 @@ +name: DNS plan and guarded apply + +"on": + workflow_dispatch: + inputs: + mode: + description: plan shows the reviewed DNS change; apply makes it + type: choice + options: [plan, apply] + default: plan + required: true + confirmation: + description: Type PLAN_DNS for plan, or APPLY_DNS for apply + type: string + required: true + +permissions: + contents: read + +concurrency: + group: cloudflare-bootstrap-migration + cancel-in-progress: false + +jobs: + change: + name: Plan or apply the reviewed DNS change + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Require the confirmed first-attempt main dispatch + env: + DNS_MODE: ${{ inputs.mode }} + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/cloudflare || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$GITHUB_WORKFLOW_REF" != devantler-tech/cloudflare/.github/workflows/dns-apply.yaml@refs/heads/main ]]; then + echo 'CLOUDFLARE_DNS_CHANGE=UNVERIFIED stage=dispatch' >&2 + exit 2 + fi + if [[ ! ( "$DNS_MODE" == plan && "$CONFIRMATION" == PLAN_DNS ) && ! ( "$DNS_MODE" == apply && "$CONFIRMATION" == APPLY_DNS ) ]]; then + echo 'CLOUDFLARE_DNS_CHANGE=UNVERIFIED stage=dispatch' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Install the pinned OpenTofu CLI + uses: opentofu/setup-opentofu@a1320f892987e89d278cc92dc5adc984fb93aca4 # v2.0.2 + with: + tofu_version: 1.12.6 + tofu_wrapper: false + + - name: Install the pinned encryption tool + run: | + curl --disable --fail --silent --show-error --proto '=https' --location --max-redirs 3 --connect-timeout 10 --max-time 60 --output "$RUNNER_TEMP/age.tar.gz" https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-amd64.tar.gz + printf '%s %s\n' cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 "$RUNNER_TEMP/age.tar.gz" | sha256sum --check --status + tar -xzf "$RUNNER_TEMP/age.tar.gz" -C "$RUNNER_TEMP" + echo "$RUNNER_TEMP/age" >> "$GITHUB_PATH" + + - name: Plan, check the boundary and apply in apply mode + id: change + env: + DNS_MODE: ${{ inputs.mode }} + DNS_PHASE: change + CONFIRMATION: ${{ inputs.confirmation }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_BOOTSTRAP_CONFIGURATION: ${{ secrets.CLOUDFLARE_BOOTSTRAP_CONFIGURATION }} + OUTPUT_DIR: ${{ runner.temp }}/dns-change + run: bash scripts/apply-dns.sh "$GITHUB_WORKSPACE" + + - name: Retain encrypted state captures and evidence + if: ${{ always() && steps.change.outcome != 'skipped' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: encrypted-dns-change + path: | + ${{ runner.temp }}/dns-change/before/manifest.json + ${{ runner.temp }}/dns-change/before/state.json.age + ${{ runner.temp }}/dns-change/before/configuration.json.age + ${{ runner.temp }}/dns-change/after/manifest.json + ${{ runner.temp }}/dns-change/after/state.json.age + ${{ runner.temp }}/dns-change/after/configuration.json.age + ${{ runner.temp }}/dns-change/evidence.tar.gz.age + if-no-files-found: warn # Early refusals have no private capture. + retention-days: 7 + compression-level: 0 + overwrite: false + include-hidden-files: false + + recover: + name: Recover from R2 on a fresh runner and require no changes + needs: change + if: ${{ always() && !cancelled() && inputs.mode == 'apply' && needs.change.result != 'skipped' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Require the confirmed first-attempt main dispatch + env: + CONFIRMATION: ${{ inputs.confirmation }} + run: | + if [[ "$GITHUB_EVENT_NAME" != workflow_dispatch || "$GITHUB_REF" != refs/heads/main || "$GITHUB_REPOSITORY" != devantler-tech/cloudflare || "$GITHUB_ACTOR" != devantler || "$GITHUB_RUN_ATTEMPT" != 1 || "$RUNNER_ENVIRONMENT" != github-hosted || "$GITHUB_WORKFLOW_REF" != devantler-tech/cloudflare/.github/workflows/dns-apply.yaml@refs/heads/main || "$CONFIRMATION" != APPLY_DNS ]]; then + echo 'CLOUDFLARE_DNS_CHANGE=UNVERIFIED stage=dispatch' >&2 + exit 2 + fi + + - name: Check out the confirmed immutable source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Install the pinned OpenTofu CLI + uses: opentofu/setup-opentofu@a1320f892987e89d278cc92dc5adc984fb93aca4 # v2.0.2 + with: + tofu_version: 1.12.6 + tofu_wrapper: false + + - name: Install the pinned encryption tool + run: | + curl --disable --fail --silent --show-error --proto '=https' --location --max-redirs 3 --connect-timeout 10 --max-time 60 --output "$RUNNER_TEMP/age.tar.gz" https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-amd64.tar.gz + printf '%s %s\n' cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 "$RUNNER_TEMP/age.tar.gz" | sha256sum --check --status + tar -xzf "$RUNNER_TEMP/age.tar.gz" -C "$RUNNER_TEMP" + echo "$RUNNER_TEMP/age" >> "$GITHUB_PATH" + + - name: Recover without copied state and require a clean plan + id: recovery + env: + DNS_MODE: apply + DNS_PHASE: recover + CONFIRMATION: ${{ inputs.confirmation }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_BOOTSTRAP_CONFIGURATION: ${{ secrets.CLOUDFLARE_BOOTSTRAP_CONFIGURATION }} + OUTPUT_DIR: ${{ runner.temp }}/dns-recovery + run: bash scripts/apply-dns.sh "$GITHUB_WORKSPACE" + + - name: Retain the independently recovered capture + if: ${{ always() && steps.recovery.outcome != 'skipped' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: encrypted-dns-recovery + path: | + ${{ runner.temp }}/dns-recovery/recovered/manifest.json + ${{ runner.temp }}/dns-recovery/recovered/state.json.age + ${{ runner.temp }}/dns-recovery/recovered/configuration.json.age + ${{ runner.temp }}/dns-recovery/evidence.tar.gz.age + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + overwrite: false + include-hidden-files: false diff --git a/.github/workflows/email-dns-proposal.yaml b/.github/workflows/email-dns-proposal.yaml deleted file mode 100644 index e6e4346..0000000 --- a/.github/workflows/email-dns-proposal.yaml +++ /dev/null @@ -1,84 +0,0 @@ -name: Email DNS import proposal (encrypted, read-only) - -"on": - workflow_dispatch: - inputs: - confirmation: - description: Type PLAN_EMAIL_DNS to import existing Email DNS locally and require no resource changes - required: true - type: string - -permissions: - contents: read - -concurrency: - group: cloudflare-bootstrap-migration - cancel-in-progress: false - -jobs: - plan: - name: Import existing Email DNS without provider changes - runs-on: ubuntu-latest - timeout-minutes: 12 - steps: - - name: Require reviewed main, the approved operator and a first attempt - env: - CONFIRMATION: ${{ inputs.confirmation }} - run: | - if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [ "$GITHUB_REF" != refs/heads/main ] || [ "$GITHUB_REPOSITORY" != devantler-tech/cloudflare ] || [ "$GITHUB_ACTOR" != devantler ] || [ "$GITHUB_RUN_ATTEMPT" != 1 ] || [ "$GITHUB_WORKFLOW_REF" != devantler-tech/cloudflare/.github/workflows/email-dns-proposal.yaml@refs/heads/main ] || [ "$RUNNER_ENVIRONMENT" != github-hosted ] || [ "$CONFIRMATION" != PLAN_EMAIL_DNS ]; then - echo 'EMAIL_DNS_PROPOSAL=UNVERIFIED stage=dispatch' >&2 - exit 2 - fi - - - name: Check out the confirmed revision - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.sha }} - persist-credentials: false - - - name: Install the pinned OpenTofu CLI - uses: opentofu/setup-opentofu@a1320f892987e89d278cc92dc5adc984fb93aca4 # v2.0.2 - with: - tofu_version: 1.12.6 - tofu_wrapper: false - - - name: Install the pinned encryption tool - run: | - curl --disable --fail --silent --show-error --proto '=https' --location --max-redirs 3 --connect-timeout 10 --max-time 60 --output "$RUNNER_TEMP/age.tar.gz" https://github.com/FiloSottile/age/releases/download/v1.3.2/age-v1.3.2-linux-amd64.tar.gz - printf '%s %s\n' cbe24006683f8eb669266162894b9a522a1af52f2665fbc63a4bb032ed26ac10 "$RUNNER_TEMP/age.tar.gz" | sha256sum --check --status - tar -xzf "$RUNNER_TEMP/age.tar.gz" -C "$RUNNER_TEMP" - echo "$RUNNER_TEMP/age" >> "$GITHUB_PATH" - - - name: Import locally, require an unchanged plan and encrypt it - id: proposal - env: - CONFIRMATION: ${{ inputs.confirmation }} - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - OUTPUT_DIR: ${{ runner.temp }}/email-dns-proposal - DIAGNOSTICS_DIR: ${{ runner.temp }}/email-dns-diagnostics - run: bash scripts/plan-email-dns.sh "$GITHUB_WORKSPACE" - - - name: Export only ciphertext and binding metadata - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: encrypted-email-dns-proposal - path: | - ${{ runner.temp }}/email-dns-proposal/proposal.tar.gz.age - ${{ runner.temp }}/email-dns-proposal/manifest.json - if-no-files-found: error - retention-days: 1 - compression-level: 0 - overwrite: false - include-hidden-files: false - - - name: Retain encrypted evidence of a refused proposal - if: ${{ failure() && steps.proposal.outcome == 'failure' }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: encrypted-email-dns-diagnostics - path: ${{ runner.temp }}/email-dns-diagnostics/diagnostics.tar.gz.age - if-no-files-found: warn # Early dispatch and credential refusals have no private capture. - retention-days: 1 - compression-level: 0 - overwrite: false - include-hidden-files: false diff --git a/README.md b/README.md index 8af09a3..22dcf7e 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,13 @@ the local checks, and what a pull request needs before it can merge. The configuration is deliberately import-first. The manual state-bootstrap workflow defaults to checking independent encrypted recovery publication. Its apply mode accepts only the frozen, reviewed first proposal and requires a -successful custody check from that same main revision. General applies remain -disabled until inventory, imports, remote locking and recovery pass their gates. +successful custody check from that same main revision. + +DNS records are declared in [`dns.tf`](dns.tf) and change only through the +manual **DNS plan and guarded apply** workflow: plan first, then apply the same +reviewed revision. It imports declared records that already exist, never +deletes or replaces anything, and keeps an encrypted state copy from before and +after every apply. Other resources still have no general apply path. The ownership decision and dependency boundary are recorded in [`docs/adr/0001-cluster-independent-cloudflare-owner.md`](docs/adr/0001-cluster-independent-cloudflare-owner.md). diff --git a/dns.tf b/dns.tf new file mode 100644 index 0000000..b9afa55 --- /dev/null +++ b/dns.tf @@ -0,0 +1,164 @@ +# DNS records this repository owns. Every value here is already published in +# public DNS, so the records are reviewed in the open; the account and zone +# identifiers stay in the encrypted configuration. +# +# Records that the platform's external-dns owns stay with it, together with +# their `_externaldns.` TXT registry. That includes the Email sender's SES +# `bounce` records. Certificate challenges stay with their issuer. +# `scripts/apply-dns.sh` imports a declared record when exactly one live +# record has the same name, type and content, and refuses an import that would +# change it. + +# The site entry points. They stay separate resources so a reviewed change can +# hand one to external-dns with a `removed` block that keeps the record. +resource "cloudflare_dns_record" "apex" { + zone_id = cloudflare_zone.primary.id + name = "devantler.tech" + type = "CNAME" + content = "devantler.github.io" + ttl = 1 + proxied = false + comment = "GitHub Pages Personal" + settings = { flatten_cname = false } +} + +resource "cloudflare_dns_record" "www" { + zone_id = cloudflare_zone.primary.id + name = "www.devantler.tech" + type = "CNAME" + content = "devantler.github.io" + ttl = 1 + proxied = true + comment = "GitHub Pages Personal" + settings = { flatten_cname = false } +} + +resource "cloudflare_dns_record" "ksail_docs" { + zone_id = cloudflare_zone.primary.id + name = "ksail.devantler.tech" + type = "CNAME" + content = "devantler-tech.github.io" + ttl = 1 + proxied = false + comment = "GitHub Pages" + settings = { flatten_cname = false } +} + +# Company Email (iCloud). These must not change while the mail transport moves. +resource "cloudflare_dns_record" "mail_mx_01" { + zone_id = cloudflare_zone.primary.id + name = "devantler.tech" + type = "MX" + content = "mx01.mail.icloud.com" + priority = 10 + ttl = 3600 + proxied = false + settings = {} +} + +resource "cloudflare_dns_record" "mail_mx_02" { + zone_id = cloudflare_zone.primary.id + name = "devantler.tech" + type = "MX" + content = "mx02.mail.icloud.com" + priority = 10 + ttl = 3600 + proxied = false + settings = {} +} + +resource "cloudflare_dns_record" "mail_spf" { + zone_id = cloudflare_zone.primary.id + name = "devantler.tech" + type = "TXT" + content = "\"v=spf1 include:icloud.com ~all\"" + ttl = 3600 + proxied = false + settings = {} +} + +resource "cloudflare_dns_record" "mail_dmarc" { + zone_id = cloudflare_zone.primary.id + name = "_dmarc.devantler.tech" + type = "TXT" + content = "\"v=DMARC1; p=none; rua=mailto:b5a283adfc26471c8ee5facda309570b@dmarc-reports.cloudflare.net\"" + ttl = 1 + proxied = false + settings = {} +} + +resource "cloudflare_dns_record" "mail_dkim_sig1" { + zone_id = cloudflare_zone.primary.id + name = "sig1._domainkey.devantler.tech" + type = "CNAME" + content = "sig1.dkim.devantler.tech.at.icloudmailadmin.com" + ttl = 3600 + proxied = false + settings = { flatten_cname = false } +} + +resource "cloudflare_dns_record" "apple_domain" { + zone_id = cloudflare_zone.primary.id + name = "devantler.tech" + type = "TXT" + content = "\"apple-domain=e9xblGFwUZq601pe\"" + ttl = 3600 + proxied = false + settings = {} +} + +# Domain ownership proofs for external services. +resource "cloudflare_dns_record" "apple_business_verification" { + zone_id = cloudflare_zone.primary.id + name = "devantler.tech" + type = "TXT" + content = "\"apple-domain-verification=4dQV9V0uTcnCM2Vu\"" + ttl = 1 + proxied = false + comment = "Apple Business identity verification" + settings = {} +} + +resource "cloudflare_dns_record" "github_organization_verification" { + zone_id = cloudflare_zone.primary.id + name = "_gh-devantler-tech-o.devantler.tech" + type = "TXT" + content = "\"05ddf7694b\"" + ttl = 1 + proxied = false + comment = "GitHub Organization Challenge" + settings = {} +} + +resource "cloudflare_dns_record" "github_pages_verification_personal" { + zone_id = cloudflare_zone.primary.id + name = "_github-pages-challenge-devantler.devantler.tech" + type = "TXT" + content = "\"543a342b74e6ce2f3f31b118a5ead2\"" + ttl = 1 + proxied = false + comment = "GitHub Pages Challenge" + settings = {} +} + +resource "cloudflare_dns_record" "github_pages_verification_organization" { + zone_id = cloudflare_zone.primary.id + name = "_github-pages-challenge-devantler-tech.devantler.tech" + type = "TXT" + content = "\"95af3e964364c659e04e4fbc58cce9\"" + ttl = 1 + proxied = false + comment = "GitHub Pages Challenge" + settings = {} +} + +resource "cloudflare_dns_record" "visual_studio_marketplace_verification" { + zone_id = cloudflare_zone.primary.id + name = "_visual-studio-marketplace-devantler.devantler.tech" + type = "TXT" + content = "\"e403e2c7-8fd4-4721-847b-23a083f773a6\"" + ttl = 1 + proxied = false + comment = "Visual Studio Marketplace" + settings = {} +} diff --git a/docs/email-dns-proposal.md b/docs/email-dns-proposal.md deleted file mode 100644 index 559fbe7..0000000 --- a/docs/email-dns-proposal.md +++ /dev/null @@ -1,32 +0,0 @@ -# Existing Email DNS import proposal - -Run **Email DNS import proposal (encrypted, read-only)** from reviewed `main` -with the confirmation `PLAN_EMAIL_DNS`. It uses the existing repository -credential on a GitHub-hosted runner. It needs no new credential and performs -no provider apply or authoritative-state adoption. - -The workflow reads the complete current DNS inventory, selects existing root -MX, SPF, DMARC, DKIM and Apple ownership records, and refuses ambiguous records -or competing controller ownership. Runtime-managed records and temporary -certificate challenges stay outside its ownership. Values, comments, tags and -provider settings are preserved. An absent comment stays absent; tags are -explicit when desired instead of being added by an import default. - -The existing zone, independent state bucket and selected records are imported -only into a private local backend. The native provider plan must contain every -expected object with a known unchanged action. Creates, updates, deletes, -replacements, drift and deferred evaluation refuse the proposal. Existing -company Email routing is preserved throughout. - -The one-day artifact contains an encrypted archive and a source/run binding -manifest. The archive includes the inventory, desired variables, import list, -saved native plan, JSON plan and bounded diagnostics. Authenticate its ciphertext -and plaintext checksums, then compare its internal source/run metadata against -the workflow before reviewing it with the independently held recovery identity. -Keep decrypted material in private temporary storage. A refused run publishes -only a separate encrypted diagnostic archive when capture is possible. - -This package has a distinct proposal kind and cannot enter the state-bootstrap -apply path. A successful result proves a read-only import proposal. Authoritative -state adoption and Email routing changes remain separate reviewed steps, with -transport and recovery acceptance required before routing moves. diff --git a/docs/runbooks/bootstrap-and-recovery.md b/docs/runbooks/bootstrap-and-recovery.md index b556e1e..6d2eb81 100644 --- a/docs/runbooks/bootstrap-and-recovery.md +++ b/docs/runbooks/bootstrap-and-recovery.md @@ -336,6 +336,58 @@ retention, route cutovers or user notification delivery. Those remain under [the alerting migration](https://github.com/devantler-tech/platform/issues/4683). Do not configure the infrastructure-state bucket as the production journal. +## DNS changes + +The records this repository owns are declared in `dns.tf`. Records that the +platform's external-dns owns stay with it, together with their `_externaldns.` +TXT registry. That includes the Email sender's SES `bounce` records. +Certificate challenges stay with their issuer. Never declare the typeless +`_externaldns.devantler.tech`: external-dns would read it as owning the apex, +then delete the apex MX and TXT records that company Email depends on. The +plan check refuses it. + +Every change is a reviewed pull request, then two dispatches of **DNS plan and +guarded apply** from `main` at that merged revision: + +1. `mode=plan`, confirmation `PLAN_DNS`. This changes nothing. +2. `mode=apply`, confirmation `APPLY_DNS`, after reading the plan's summary. + +Both runs work against the authoritative R2 state through a 15-minute storage +session derived from the existing `CLOUDFLARE_API_TOKEN`; the configuration +comes from `CLOUDFLARE_BOOTSTRAP_CONFIGURATION`. The live state must still be +exactly the retained `recovery/backend-source/` generation, so an earlier apply +that was not yet retained refuses every new run. A declared record that the +plan would create is imported when exactly one live record has the same name, +type and content. It stays a real create only when no live record has that +name and type. A changed content, more than one match, or a record that +external-dns owns refuses the run. + +The saved plan must then pass `scripts/assert-no-destructive-plan.sh` and +`scripts/check-dns-plan.jq`. Only DNS records may change. Imports must leave +their record exactly as it is. Creates, updates and `removed` blocks with +`destroy = false` are admitted; deletes and replacements are not. The public +log shows only `DNS_PLAN` lines, with addresses, actions and changed attribute +names; `DNS_DRIFT` lines; and `DNS_UNMANAGED` lines for live records that +nothing owns. Values, identifiers and tool output stay in the encrypted +evidence. + +In apply mode the run encrypts the state before and after the apply, even +when the apply fails, and then requires a clean plan. A second job initializes +from R2 on a fresh runner and must also find nothing to change. Afterwards, +download the `encrypted-dns-change` and `encrypted-dns-recovery` artifacts, +authenticate the `after` and `recovered` packages with the independent +identity (`check-state-recovery-package.sh --identity-file`), and retain the +`after` package in `recovery/backend-source/` through a signed, data-only pull +request. The configuration does not change, so the configuration secret stays +as it is. The next DNS change is possible only after that pull request merges. + +To hand a record to external-dns without a gap, create its ownership TXT +record in one change, then replace the record's resource with a `removed` +block that keeps the record (`lifecycle { destroy = false }`). To undo an +apply, revert its pull request and run the same two dispatches; a revert that +would delete a record is refused, so restore deleted values by declaring them +again. + ## Recovery Retrieve short-lived Cloudflare and R2 state credentials from the independent diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh new file mode 100755 index 0000000..7d208ef --- /dev/null +++ b/scripts/apply-dns.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# Reviewed DNS changes against the authoritative R2 state. The change phase +# imports declared records that already exist, refuses deletes, replacements +# and changing imports, and in apply mode captures encrypted state before and +# after the apply. The recover phase plans from a fresh runner and must find +# nothing to do. Only fixed verdict lines reach the log; values stay encrypted. +set +x +set -euo pipefail +umask 077 +stage=dispatch scratch='' output_created=false +stop() { exit 2; } +finish() { + local rc=$? + if [[ -n $scratch ]]; then + if [[ $output_created == true && -d $scratch/evidence ]]; then + tar -czf "$scratch/evidence.tar.gz" -C "$scratch" evidence >/dev/null 2>&1 && + age -r "$recipient" -o "$OUTPUT_DIR/evidence.tar.gz.age" "$scratch/evidence.tar.gz" 2>/dev/null || rc=2 + fi + rm -rf "$scratch" + fi + if [[ $rc != 0 ]]; then + printf 'CLOUDFLARE_DNS_CHANGE=UNVERIFIED stage=%s\n' "$stage" >&2 + exit 2 + fi + printf 'CLOUDFLARE_DNS_CHANGE=VERIFIED mode=%s phase=%s\n' "$DNS_MODE" "$DNS_PHASE" +} +trap finish EXIT +trap 'exit 2' INT TERM +[[ ${BASH_VERSINFO[0]} -ge 5 && $# == 1 && -d $1 && ! -L $1 && + ${GITHUB_EVENT_NAME:-} == workflow_dispatch && ${GITHUB_REPOSITORY:-} == devantler-tech/cloudflare && + ${GITHUB_REF:-} == refs/heads/main && ${GITHUB_ACTOR:-} == devantler && + ${GITHUB_RUN_ATTEMPT:-} == 1 && ${RUNNER_ENVIRONMENT:-} == github-hosted && + ${GITHUB_WORKFLOW_REF:-} == devantler-tech/cloudflare/.github/workflows/dns-apply.yaml@refs/heads/main && + ${GITHUB_SHA:-} =~ ^[a-f0-9]{40}$ && ${GITHUB_RUN_ID:-} =~ ^[1-9][0-9]{5,15}$ && + ( ( ${DNS_MODE:-} == plan && ${CONFIRMATION:-} == PLAN_DNS && ${DNS_PHASE:-} == change ) || + ( ${DNS_MODE:-} == apply && ${CONFIRMATION:-} == APPLY_DNS && + ( ${DNS_PHASE:-} == change || ${DNS_PHASE:-} == recover ) ) ) ]] || stop +root=$1 +# shellcheck source=scripts/r2-backend-common.sh +source "$root/scripts/r2-backend-common.sh" +stage=configuration +[[ -n ${CLOUDFLARE_BOOTSTRAP_CONFIGURATION:-} && ${#CLOUDFLARE_BOOTSTRAP_CONFIGURATION} -le 8388608 && + -n ${OUTPUT_DIR:-} && $OUTPUT_DIR == /* && ! -e $OUTPUT_DIR && ! -L $OUTPUT_DIR && + $(findmnt -n -o FSTYPE --target /dev/shm) == tmpfs ]] || stop +for tool in tofu timeout jq curl age tar shasum; do command -v "$tool" >/dev/null 2>&1 || stop; done +scratch=$(mktemp -d /dev/shm/cloudflare-dns.XXXXXXXX) +mkdir "$scratch/evidence" "$scratch/home" "$scratch/work" +recipient=$(state_recipient "$root") +printf '%s' "$CLOUDFLARE_BOOTSTRAP_CONFIGURATION" | backend_configuration_input > "$scratch/configuration.json" || stop +unset CLOUDFLARE_BOOTSTRAP_CONFIGURATION +backend_configuration "$root" "$scratch/configuration.json" || stop +jq -e '.zone_name=="devantler.tech"' "$scratch/configuration.json" >/dev/null 2>&1 || stop +account=$(jq -er .account_id "$scratch/configuration.json") +retained="$root/recovery/backend-source/manifest.json" +mkdir "$OUTPUT_DIR" +output_created=true +stage=credential +root_token=${CLOUDFLARE_API_TOKEN:-} +unset CLOUDFLARE_API_TOKEN +[[ ${#root_token} -ge 20 && ${#root_token} -le 512 && $root_token =~ ^[A-Za-z0-9._~+/-]+=*$ ]] || stop +# The account token reaches curl only through an inherited input pipe. +request() { + local method=$1 url=$2 response=$3 body=${4:-} status rc=0 + local -a args=(--request "$method" --url "$url" --output "$response") + if [[ -n $body ]]; then args+=(--header 'Content-Type: application/json' --data-binary "@$body"); fi + status=$(printf 'header = "Authorization: Bearer %s"\n' "$root_token" | + curl --disable --config - --proto '=https' --noproxy '*' --max-redirs 0 \ + --connect-timeout 10 --max-time 30 --max-filesize 32768 --silent --fail-with-body \ + --write-out '%{http_code}' "${args[@]}" 2>/dev/null) || rc=$? + if [[ $status =~ ^[0-9]{3}$ ]]; then printf 'status=%s curl_exit=%s\n' "$status" "$rc" >> "$scratch/evidence/http.log"; fi + jq -r '.errors[]?.code | select(type=="number") | "provider_code=\(.)"' "$response" >> "$scratch/evidence/http.log" 2>/dev/null || true + [[ $rc == 0 && $status == 200 ]] && state_unique_json "$response" && + jq -e '.success==true and (.errors==null or .errors==[])' "$response" >/dev/null 2>&1 +} +request GET "https://api.cloudflare.com/client/v4/accounts/$account/tokens/verify" "$scratch/parent.json" || stop +parent=$(jq -er 'if .result.status=="active" and (.result.id|type=="string" and test("^[a-f0-9]{32}$")) then .result.id else error("invalid parent") end' "$scratch/parent.json" 2>/dev/null) || stop +jq -n --arg parent "$parent" '{parentAccessKeyId:$parent,bucket:"cloudflare-bootstrap-state",permission:"object-read-write",ttlSeconds:900,prefixes:["cloudflare/"]}' > "$scratch/request.json" +stage=session +request POST "https://api.cloudflare.com/client/v4/accounts/$account/r2/temp-access-credentials" "$scratch/session.json" "$scratch/request.json" || stop +backend_session "$scratch/session.json" || stop +unset parent +# OpenTofu gets only the short-lived state session, plus the account token for +# provider steps. Nothing else from the runner environment reaches it. +native() ( + local provider_mode=$1; shift + while IFS= read -r name; do [[ $name == PATH ]] || unset "$name"; done < <(compgen -e) + export HOME="$scratch/home" TF_IN_AUTOMATION=1 CHECKPOINT_DISABLE=1 AWS_EC2_METADATA_DISABLED=true + AWS_ACCESS_KEY_ID=$(jq -er .result.accessKeyId "$scratch/session.json") + AWS_SECRET_ACCESS_KEY=$(jq -er .result.secretAccessKey "$scratch/session.json") + AWS_SESSION_TOKEN=$(jq -er .result.sessionToken "$scratch/session.json") + export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + if [[ $provider_mode == provider ]]; then export CLOUDFLARE_API_TOKEN="$root_token"; fi + "$@" +) +tf() { native "$1" timeout --kill-after=10s 300 tofu -chdir="$scratch/work" "${@:2}"; } +# Plan to a saved file and render it as JSON in private evidence. +plan_json() { + local name=$1 rc=0 + tf provider plan -input=false -lock-timeout=0s -no-color -out="$name.tfplan" > "$scratch/evidence/$name-plan.log" 2>&1 || rc=$? + [[ $rc == 0 ]] || return 1 + tf local show -json "$name.tfplan" > "$scratch/evidence/$name.json" 2> "$scratch/evidence/$name-show.log" || return 1 + state_unique_json "$scratch/evidence/$name.json" +} +# Bind the live state to the retained recovery generation: an apply that was +# not yet retained, or a different lineage, refuses every further change. +capture() { + local name=$1 + tf local state pull > "$scratch/evidence/$name-state.json" 2> "$scratch/evidence/$name-pull.log" || return 1 + bash "$root/scripts/prepare-state-recovery-package.sh" "$root" "$scratch/evidence/$name-state.json" \ + "$scratch/configuration.json" "$OUTPUT_DIR/$name" "$GITHUB_SHA" "$GITHUB_RUN_ID" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$scratch/evidence/$name-package.log" 2>&1 +} +stage=source +for file in "$root"/*.tf "$root/.terraform.lock.hcl"; do + [[ -f $file && ! -L $file && $file != *_override.tf ]] || stop + cp "$file" "$scratch/work/" +done +cp "$scratch/configuration.json" "$scratch/work/production.auto.tfvars.json" +cat > "$scratch/backend.hcl" < "$scratch/evidence/version.json" 2>/dev/null || stop +jq -e '.terraform_version=="1.12.6"' "$scratch/evidence/version.json" >/dev/null 2>&1 || stop +stage=initialize +tf local init -input=false -lockfile=readonly -no-color -backend-config="$scratch/backend.hcl" > "$scratch/evidence/init.log" 2>&1 || stop +stage=recovery-generation +tf local state pull > "$scratch/evidence/start-state.json" 2> "$scratch/evidence/start-pull.log" || stop +state_unique_json "$scratch/evidence/start-state.json" || stop +if [[ $DNS_PHASE == change ]]; then + jq -e --slurpfile retained "$retained" '.lineage==$retained[0].lineage and .serial==$retained[0].serial' \ + "$scratch/evidence/start-state.json" >/dev/null 2>&1 || stop +else + # The change job just applied; recovery must find the same lineage, at least as new. + jq -e --slurpfile retained "$retained" '.lineage==$retained[0].lineage and .serial>=$retained[0].serial' \ + "$scratch/evidence/start-state.json" >/dev/null 2>&1 || stop + stage=recovery-capture + capture recovered || stop + stage=fresh-plan + tf provider plan -input=false -lock-timeout=0s -detailed-exitcode -no-color > "$scratch/evidence/fresh-plan.log" 2>&1 || stop + unset root_token + exit 0 +fi +stage=inventory +# shellcheck disable=SC2034 # collect_dns_snapshot reads it. +provider_token=$root_token +# shellcheck source=scripts/dns-snapshot-common.sh +source "$root/scripts/dns-snapshot-common.sh" +collect_dns_snapshot "$scratch" +unset provider_token +mv "$scratch/snapshot.json" "$scratch/evidence/snapshot.json" +rm -f "$scratch"/page-*.json "$scratch/records.json" "$scratch/zone.json" +jq -e --arg account "$account" '.zone.account.id==$account and .zone.name=="devantler.tech"' "$scratch/evidence/snapshot.json" >/dev/null 2>&1 || stop +stage=discover +plan_json discover || stop +jq --slurpfile snapshot "$scratch/evidence/snapshot.json" -f "$root/scripts/dns-imports.jq" \ + "$scratch/evidence/discover.json" > "$scratch/evidence/resolved.json" 2>/dev/null || stop +jq -r '.refused[]|"DNS_REFUSED address=\(.address) reason=\(.refused)"' "$scratch/evidence/resolved.json" +jq -e '.refused==[]' "$scratch/evidence/resolved.json" >/dev/null 2>&1 || stop +jq '.imports' "$scratch/evidence/resolved.json" > "$scratch/evidence/imports.json" +jq -r '.[]|"import {\n to = \(.address)\n id = \(.id|tojson)\n}\n"' "$scratch/evidence/imports.json" > "$scratch/work/imports.tf" +stage=plan +plan_json change || stop +# Public summary: addresses, actions and changed attribute names, never values. +jq -r '.resource_changes[] | + (if .change.importing then "import" else (.change.actions|join("-")) end) as $action | + select($action!="no-op") | + ([(.change.before // {}), (.change.after // {})] as [$b,$a] | + ([$b,$a]|map(keys)|add|unique) | map(select($b[.]!=$a[.]))) as $attributes | + "DNS_PLAN address=\(.address) action=\($action)" + + (if $action=="update" or ($action=="import" and .change.actions!=["no-op"]) then " attributes=\($attributes|join(","))" else "" end) +' "$scratch/evidence/change.json" +jq -r '.resource_drift[]? | "DNS_DRIFT address=\(.address) action=\(.change.actions|join("-"))"' "$scratch/evidence/change.json" +# Live records that nothing owns: not declared here, not external-dns, not a challenge. +jq -r --slurpfile plan "$scratch/evidence/change.json" --slurpfile imports "$scratch/evidence/imports.json" ' + ([$plan[0].prior_state.values.root_module.resources[]? | select(.type=="cloudflare_dns_record") | .values.id] + + [$imports[0][] | .id | split("/")[1]]) as $managed | + .records as $records | + .records[] | . as $r | + select((.id|IN($managed[]))|not) | + select((.name|startswith("_externaldns.") or startswith("_acme-challenge."))|not) | + select(any($records[]; .type=="TXT" and (.name=="_externaldns."+$r.name or .name=="_externaldns."+($r.type|ascii_downcase)+"-"+$r.name))|not) | + "DNS_UNMANAGED name=\(.name) type=\(.type)" +' "$scratch/evidence/snapshot.json" +stage=boundary +bash "$root/scripts/assert-no-destructive-plan.sh" "$scratch/evidence/change.json" > "$scratch/evidence/destructive.log" 2>&1 || stop +jq -e --slurpfile imports "$scratch/evidence/imports.json" -f "$root/scripts/check-dns-plan.jq" "$scratch/evidence/change.json" >/dev/null 2>&1 || stop +if [[ $DNS_MODE == plan ]]; then unset root_token; exit 0; fi +stage=before-capture +capture before || stop +stage=apply +apply_rc=0 +tf provider apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? +# Capture whatever the apply left behind before judging it. +stage=after-capture +capture after || stop +stage=apply +[[ $apply_rc == 0 ]] || stop +stage=convergence +rm "$scratch/work/imports.tf" +tf provider plan -input=false -lock-timeout=0s -detailed-exitcode -no-color > "$scratch/evidence/convergence-plan.log" 2>&1 || stop +unset root_token diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq new file mode 100644 index 0000000..c6a01d8 --- /dev/null +++ b/scripts/check-dns-plan.jq @@ -0,0 +1,26 @@ +# Input: the saved plan JSON. $imports[0]: the resolved import list. +# Only DNS records may change. Nothing is deleted or replaced. Every import is +# known and leaves its record exactly as it is. The typeless apex registry name +# is refused outright: external-dns would read it as owning the apex mail +# records and delete them. +def changed: (.change.before != .change.after) or any(.change.after_unknown|..; .==true); +. as $plan | $imports[0] as $wanted | +.errored!=true and +((.deferred_changes // [])|length)==0 and +(.resource_changes|type=="array") and +($wanted|type=="array") and +(($wanted|map(.address)|length)==($wanted|map(.address)|unique|length)) and +all($wanted[]; + . as $import | + [$plan.resource_changes[]|select(.address==$import.address)] as $matches | + ($matches|length)==1 and $matches[0].change.importing.id==$import.id) and +all(.resource_changes[]; + . as $resource | + .mode=="managed" and + (.change.actions|IN(["no-op"],["create"],["update"],["forget"])) and + (if .change.importing then + any($wanted[]; .address==$resource.address) and .change.actions==["no-op"] and (changed|not) + else true end) and + (if .type=="cloudflare_dns_record" then + ((.change.after // {}).name != "_externaldns.devantler.tech") + else .change.actions==["no-op"] end)) diff --git a/scripts/check-email-dns-plan.jq b/scripts/check-email-dns-plan.jq deleted file mode 100644 index 7d3310c..0000000 --- a/scripts/check-email-dns-plan.jq +++ /dev/null @@ -1,33 +0,0 @@ -# A proposal may import existing objects locally; every provider action must be -# known and unchanged. Extra, missing, duplicate and deferred objects are HOLD. -def type_for($address): - if $address=="cloudflare_zone.primary" then "cloudflare_zone" - elif $address=="cloudflare_r2_bucket.buckets[\"cloudflare-bootstrap-state\"]" then "cloudflare_r2_bucket" - elif ($address|test("^cloudflare_dns_record\\.records\\[\\\"email_[a-f0-9]{32}\\\"\\]$")) then "cloudflare_dns_record" - else "refused" end; -def valid_import: - type_for(.address) as $type | - if $type=="cloudflare_zone" then (.id|test("^[a-f0-9]{32}$")) - elif $type=="cloudflare_r2_bucket" then (.id|test("^[a-f0-9]{32}/cloudflare-bootstrap-state/default$")) - elif $type=="cloudflare_dns_record" then (.id|test("^[a-f0-9]{32}/[a-f0-9]{32}$")) - else false end; -. as $plan | $wanted[0] as $imports | -.errored!=true and -((.deferred_changes // [])|length)==0 and -((.resource_drift // [])|length)==0 and -($imports|type=="array" and length>=7 and length<=32) and -all($imports[];valid_import) and -(($imports|map(.address)|length)==($imports|map(.address)|unique|length)) and -(.resource_changes|type=="array") and -((.resource_changes|map(.address)|sort)==($imports|map(.address)|sort)) and -all(.resource_changes[]; - . as $resource | - ([$imports[]|select(.address==$resource.address)]) as $expected | - ($expected|length)==1 and - .mode=="managed" and .type==type_for(.address) and - .change.actions==["no-op"] and .change.before==.change.after and - (any(.change.after_unknown|..; .==true)|not) and - .change.after.id==( - if .type=="cloudflare_zone" then $expected[0].id - else ($expected[0].id|split("/")[1]) end - )) diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq new file mode 100644 index 0000000..e57248e --- /dev/null +++ b/scripts/dns-imports.jq @@ -0,0 +1,29 @@ +# Input: a plan JSON made before imports. $snapshot[0]: the complete live zone. +# A declared record the plan would create is imported when exactly one live +# record has the same name, type and content. It stays a create only when no +# live record has that name and type. Anything else is refused, so a typo or +# drift can never add a second mail record or take over an external-dns record. +def registry: .type=="TXT" and (.name|startswith("_externaldns.")); +def owned($records): + . as $r | any($records[]; + .type=="TXT" and + (.name=="_externaldns."+$r.name or .name=="_externaldns."+($r.type|ascii_downcase)+"-"+$r.name)); +$snapshot[0] as $live | +[.resource_changes[] | + select(.mode=="managed" and .type=="cloudflare_dns_record" and .change.actions==["create"]) | + .change.after as $after | + [$live.records[] | select(.name==$after.name and .type==$after.type)] as $same | + [$same[] | select(.content==$after.content)] as $exact | + if ($exact|length)==1 then + if ($exact[0]|registry) or ($exact[0]|owned($live.records)) then {address, refused:"external-dns-owned"} + else {address, id:($live.zone.id+"/"+$exact[0].id)} end + elif ($exact|length)>1 then {address, refused:"ambiguous"} + elif ($same|length)>0 then {address, refused:"content-differs"} + else empty end +] as $resolved | +([$resolved[]|select(has("id"))|.id]) as $ids | +{ + imports:[$resolved[]|select(has("id"))]|sort_by(.address), + refused:([$resolved[]|select(has("refused"))] + + (if ($ids|length)==($ids|unique|length) then [] else [{address:"*",refused:"duplicate-target"}] end)) +} diff --git a/scripts/email-dns-proposal.jq b/scripts/email-dns-proposal.jq deleted file mode 100644 index 1d590a7..0000000 --- a/scripts/email-dns-proposal.jq +++ /dev/null @@ -1,79 +0,0 @@ -# Input is a bounded complete snapshot; the caller authenticates its revision -# and freshness. This filter preserves values and refuses competing ownership. -def identifier: type=="string" and test("^[a-f0-9]{32}$"); -def integer: type=="number" and .==floor; -def text_value: type=="string" and length>0 and length<=8192 and (test("[[:cntrl:]]")|not); -def unquote: ltrimstr("\"") | rtrimstr("\""); -def spf: .type=="TXT" and (.content|type=="string") and (.content|unquote|ascii_downcase|startswith("v=spf1 ")); -def apple: .type=="TXT" and (.content|type=="string") and (.content|unquote|startswith("apple-domain=")); -def dkim($zone): - (.type=="CNAME" or .type=="TXT") and - (.name|endswith("._domainkey."+$zone)) and - (.name|rtrimstr("._domainkey."+$zone)|test("^[a-z0-9_-]{1,63}$")); -def email($zone): - (.name==$zone and (.type=="MX" or spf or apple)) or - (.name=="_dmarc."+$zone and .type=="TXT") or dkim($zone); -def record_shape: - (.id|identifier) and (.content|text_value) and - (.ttl|integer and (.==1 or (.>=30 and .<=86400))) and - .proxied==false and - (.priority==null or (.priority|integer and .>=0 and .<=65535)) and - (if .type=="MX" then .priority!=null else .priority==null end) and - (.comment==null or (.comment|type=="string" and length<=1024)) and - ((.tags // [])|type=="array" and length<=100 and - all(.[];type=="string" and length<=200) and (length==(unique|length))) and - ((.settings // {})|type=="object" and - ((keys-["flatten_cname","ipv4_only","ipv6_only"])|length)==0 and - all(.[];type=="boolean")) and - (.private_routing==null or (.private_routing|type=="boolean")) and - (.data==null or .data=={}) and - ((.meta // {})|type=="object" and - ((keys-["auto_added","managed_by_apps","managed_by_argo_tunnel"])|length)==0); -def free_of_controller: - ((.tags // [])|all(.[]; - (test("external-dns";"i")|not) and - (if test("^(owner|managed-by|controller):";"i") then - .=="owner:cloudflare-repo" or .=="managed-by:cloudflare-repo" or .=="controller:cloudflare-repo" - else true end))) and - ((.comment // "")|test("external-dns";"i")|not) and - ((.meta // {}) as $meta | - all(["auto_added","managed_by_apps","managed_by_argo_tunnel"][]; - . as $key | ($meta[$key]==null or $meta[$key]==false))); -. as $snapshot | -if .schema!=1 or .source_revision!=$revision or - ($revision|test("^[a-f0-9]{40}$")|not) or - .zone.name!=$zone or .zone.status!="active" or - (.zone.id|identifier|not) or (.zone.account.id|identifier|not) or - (.records|type!="array" or length<1 or length>10000) or - ((.records|length)!=(.records|map(.id)|unique|length)) or - (all(.records[];(.id|identifier) and (.name|type=="string") and - (.name==$zone or (.name|endswith("."+$zone))) and (.type|type=="string"))|not) -then error("email-dns-proposal-refused") -else - [.records[]|select(email($zone))] as $selected | - if ($selected|length)<5 or ($selected|length)>30 or - (all($selected[];record_shape and free_of_controller)|not) or - ([$selected[]|select(.type=="MX")]|length)<1 or - ([$selected[]|select(spf)]|length)!=1 or - ([$selected[]|select(apple)]|length)!=1 or - ([$selected[]|select(.name=="_dmarc."+$zone)]|length)!=1 or - ([$selected[]|select(dkim($zone))]|length)<1 or - (([$selected[]|{name,type,content,priority}]|unique|length)!=($selected|length)) or - (all($selected[]; - . as $candidate | - any($snapshot.records[]; - .type=="TXT" and - (.name=="_externaldns."+$candidate.name or - .name=="_externaldns."+($candidate.type|ascii_downcase)+"-"+$candidate.name))|not)|not) - then error("email-dns-proposal-refused") - else - { - dns_records:(reduce $selected[] as $r ({}; .["email_"+$r.id]={ - name:$r.name,type:$r.type,content:$r.content,ttl:$r.ttl, - proxied:$r.proxied,priority:$r.priority,comment:$r.comment, - tags:($r.tags // []),settings:($r.settings // {}),private_routing:$r.private_routing - })), - imports:([$selected[]|{address:("cloudflare_dns_record.records["+("email_"+.id|tojson)+"]"),id:($snapshot.zone.id+"/"+.id)}]|sort_by(.address)) - } - end -end diff --git a/scripts/plan-email-dns.sh b/scripts/plan-email-dns.sh deleted file mode 100644 index 4567ce2..0000000 --- a/scripts/plan-email-dns.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env bash -# Import only into isolated local state; publish an encrypted no-change proposal. -set +x -set -euo pipefail -umask 077 -# Emit only the caller's closed failure stage, never a provider response. -stop() { failure_stage=$1; printf 'EMAIL_DNS_PROPOSAL=UNVERIFIED stage=%s\n' "$1" >&2; exit 2; } -[[ ${GITHUB_EVENT_NAME:-} == workflow_dispatch && ${GITHUB_REF:-} == refs/heads/main && - ${GITHUB_REPOSITORY:-} == devantler-tech/cloudflare && ${GITHUB_ACTOR:-} == devantler && - ${GITHUB_RUN_ATTEMPT:-} == 1 && ${RUNNER_ENVIRONMENT:-} == github-hosted && - ${GITHUB_WORKFLOW_REF:-} == devantler-tech/cloudflare/.github/workflows/email-dns-proposal.yaml@refs/heads/main && - ${GITHUB_SHA:-} =~ ^[a-f0-9]{40}$ && ${GITHUB_RUN_ID:-} =~ ^[1-9][0-9]{0,15}$ && - ${CONFIRMATION:-} == PLAN_EMAIL_DNS ]] || stop dispatch -[[ $# == 1 && -d $1 && ! -L $1 ]] || stop root -root=$1 -provider_token=${CLOUDFLARE_API_TOKEN:-} -unset CLOUDFLARE_API_TOKEN -[[ ${#provider_token} -ge 20 && ${#provider_token} -le 512 && $provider_token =~ ^[A-Za-z0-9._~+/-]+=*$ ]] || stop credential -while IFS= read -r variable; do - case $variable in - TF_*|TOFU_*|CLOUDFLARE_*|AWS_*|HTTP_PROXY|HTTPS_PROXY|ALL_PROXY|NO_PROXY|http_proxy|https_proxy|all_proxy|no_proxy) unset "$variable";; - esac -done < <(compgen -e) -for tool in jq curl age tofu shasum tar; do command -v "$tool" >/dev/null 2>&1 || stop tools;done -[[ -n ${OUTPUT_DIR:-} && $OUTPUT_DIR == /* && ! -e $OUTPUT_DIR && ! -L $OUTPUT_DIR ]] || stop destination -[[ -n ${DIAGNOSTICS_DIR:-} && $DIAGNOSTICS_DIR == /* && ! -e $DIAGNOSTICS_DIR && ! -L $DIAGNOSTICS_DIR && $DIAGNOSTICS_DIR != "$OUTPUT_DIR" ]] || stop diagnostics-destination -script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) -# shellcheck source=scripts/state-recovery-common.sh -source "$script_dir/state-recovery-common.sh" -# shellcheck source=scripts/dns-snapshot-common.sh -source "$script_dir/dns-snapshot-common.sh" -recipient=$(state_recipient "$root") -scratch=$(mktemp -d) -owned_output=false complete=false failure_stage=unexpected - -# Only fixed bounded files enter the encrypted archive; credentials, provider -# executables and the environment never enter either success or failure output. -pack_private() { - local kind=$1 file - local files=(metadata.json) - jq -n --arg kind "$kind" --arg source "$GITHUB_SHA" --argjson run "$GITHUB_RUN_ID" --arg stage "$failure_stage" \ - '{schemaVersion:1,kind:$kind,sourceCommit:$source,sourceRun:$run,providerWrites:0,stage:$stage}' > "$scratch/metadata.json" || return 1 - if [[ $kind == cloudflare-email-dns-proposal ]]; then - for file in snapshot.json bucket.json proposal.json imports.json plan.json work/production.auto.tfvars.json work/migration.tfplan; do - [[ -f $scratch/$file && ! -L $scratch/$file && -s $scratch/$file && $(wc -c < "$scratch/$file") -le 16777216 ]] || return 1 - done - fi - for file in snapshot.json bucket.json proposal.json imports.json plan.json init.log import.log plan.log show.log work/production.auto.tfvars.json work/migration.tfplan; do - if [[ -f $scratch/$file && ! -L $scratch/$file && $(wc -c < "$scratch/$file") -le 16777216 ]];then files+=("$file");fi - done - tar -czf "$scratch/bundle.tar.gz" -C "$scratch" "${files[@]}" 2>/dev/null || return 1 - [[ $(wc -c < "$scratch/bundle.tar.gz") -le 33554432 ]] || return 1 - age -r "$recipient" -o "$scratch/bundle.tar.gz.age" "$scratch/bundle.tar.gz" 2>/dev/null -} -# Encrypt bounded adverse evidence and remove only this invocation's scratch -# and any incomplete output directory it created. -cleanup() { - local rc=$? - if [[ $rc != 0 ]];then - if pack_private cloudflare-email-dns-diagnostics && mkdir -m 700 "$DIAGNOSTICS_DIR" && - mv "$scratch/bundle.tar.gz.age" "$DIAGNOSTICS_DIR/diagnostics.tar.gz.age";then - printf '%s\n' 'EMAIL_DNS_DIAGNOSTICS=ENCRYPTED' - else printf '%s\n' 'EMAIL_DNS_DIAGNOSTICS=UNVERIFIED' >&2;fi - fi - rm -rf "$scratch" - if "$owned_output" && ! "$complete";then rm -rf "$OUTPUT_DIR";fi -} -trap cleanup EXIT -trap 'exit 2' INT TERM -printf 'recipient validation\n' | age -r "$recipient" -o "$scratch/probe.age" 2>/dev/null || stop recipient -collect_dns_snapshot "$scratch" -state_unique_json "$scratch/snapshot.json" || stop snapshot -jq -e --arg source "$GITHUB_SHA" --argjson now "$(date -u +%s)" \ - '(.captured_at|fromdateiso8601) as $captured|.source_revision==$source and $captured<=$now and ($now-$captured)<=300' \ - "$scratch/snapshot.json" >/dev/null 2>&1 || stop snapshot-freshness -jq --arg zone devantler.tech --arg revision "$GITHUB_SHA" -f "$script_dir/email-dns-proposal.jq" "$scratch/snapshot.json" > "$scratch/proposal.json" 2>/dev/null || stop ownership -account_id=$(jq -er .zone.account.id "$scratch/snapshot.json") -zone_id=$(jq -er .zone.id "$scratch/snapshot.json") -read_cloudflare "https://api.cloudflare.com/client/v4/accounts/$account_id/r2/buckets/cloudflare-bootstrap-state" > "$scratch/bucket.json" || stop bucket-read -state_unique_json "$scratch/bucket.json" || stop bucket -jq -e '.success==true and .result.name=="cloudflare-bootstrap-state" and .result.jurisdiction=="default" and (.result.storage_class=="Standard" or .result.storage_class=="InfrequentAccess")' "$scratch/bucket.json" >/dev/null 2>&1 || stop bucket -mkdir "$scratch/work" -for file in main.tf variables.tf versions.tf providers.tf outputs.tf .terraform.lock.hcl;do - [[ -f $root/$file && ! -L $root/$file ]] || stop source - cp "$root/$file" "$scratch/work/$file" || stop source -done -printf 'terraform {\n backend "local" {}\n}\n' > "$scratch/work/backend_override.tf" -printf '\n' > "$scratch/provider.rc" -jq -n --arg account "$account_id" --slurpfile dns "$scratch/proposal.json" --slurpfile bucket "$scratch/bucket.json" \ - '{account_id:$account,zone_name:"devantler.tech",dns_records:$dns[0].dns_records,r2_buckets:{"cloudflare-bootstrap-state":{jurisdiction:"default",storage_class:$bucket[0].result.storage_class}},r2_lifecycle_rules:{},account_tokens:{}}' > "$scratch/work/production.auto.tfvars.json" -jq --arg zone "$zone_id" --arg account "$account_id" \ - '.imports + [{address:"cloudflare_zone.primary",id:$zone},{address:"cloudflare_r2_bucket.buckets[\"cloudflare-bootstrap-state\"]",id:($account+"/cloudflare-bootstrap-state/default")}]|sort_by(.address)' "$scratch/proposal.json" > "$scratch/imports.json" -# Expose the provider credential only to the isolated OpenTofu process. -provider_tofu() ( - export CLOUDFLARE_API_TOKEN="$provider_token" TF_IN_AUTOMATION=true CHECKPOINT_DISABLE=1 TF_CLI_CONFIG_FILE="$scratch/provider.rc" - tofu -chdir="$scratch/work" "$@" -) -provider_tofu init -input=false -lockfile=readonly > "$scratch/init.log" 2>&1 || stop init -while IFS=$'\t' read -r address identity;do - provider_tofu import -input=false "$address" "$identity" >> "$scratch/import.log" 2>&1 || stop import -done < <(jq -r '.[]|[.address,.id]|@tsv' "$scratch/imports.json") -plan_rc=0 -provider_tofu plan -input=false -detailed-exitcode -out=migration.tfplan > "$scratch/plan.log" 2>&1 || plan_rc=$? -[[ $plan_rc == 0 || $plan_rc == 2 ]] || stop plan -provider_tofu show -json migration.tfplan > "$scratch/plan.json" 2> "$scratch/show.log" || stop show -state_unique_json "$scratch/plan.json" || stop plan-json -jq -e --slurpfile wanted "$scratch/imports.json" -f "$script_dir/check-email-dns-plan.jq" "$scratch/plan.json" >/dev/null 2>&1 || stop change-boundary -[[ $plan_rc == 0 ]] || stop changed-plan -unset provider_token account_id zone_id -failure_stage=complete -pack_private cloudflare-email-dns-proposal || stop encryption -cipher_hash=$(state_hash "$scratch/bundle.tar.gz.age") || stop checksum -plain_hash=$(state_hash "$scratch/bundle.tar.gz") || stop checksum -[[ $cipher_hash =~ ^[a-f0-9]{64}$ && $plain_hash =~ ^[a-f0-9]{64}$ ]] || stop checksum -mkdir -m 700 "$OUTPUT_DIR" || stop destination -owned_output=true -mv "$scratch/bundle.tar.gz.age" "$OUTPUT_DIR/proposal.tar.gz.age" -jq -n --arg source "$GITHUB_SHA" --argjson run "$GITHUB_RUN_ID" --arg ciphertext "$cipher_hash" --arg plaintext "$plain_hash" \ - '{schemaVersion:1,kind:"cloudflare-email-dns-proposal",sourceCommit:$source,sourceRun:$run,providerWrites:0,ciphertextSha256:$ciphertext,plaintextSha256:$plaintext,authoritativeStateAdopted:false}' > "$OUTPUT_DIR/manifest.json" -complete=true -printf '%s\n' 'EMAIL_DNS_PROPOSAL=ENCRYPTED_UNCHANGED provider_writes=0' diff --git a/scripts/test.sh b/scripts/test.sh index f6bedee..24d7fd4 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -13,8 +13,7 @@ bash scripts/tests/test-recovery-package.sh bash scripts/tests/test-state-recovery-package.sh bash scripts/tests/test-state-capture.sh bash scripts/tests/test-bootstrap-plan.sh -bash scripts/tests/test-email-dns-proposal.sh -bash scripts/tests/test-email-dns-plan.sh +bash scripts/tests/test-dns-apply.sh bash scripts/tests/test-alert-journal-native.sh bash scripts/tests/test-bootstrap-apply-input.sh bash scripts/tests/test-bootstrap-state-publication.sh diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh new file mode 100755 index 0000000..52ee5c5 --- /dev/null +++ b/scripts/tests/test-dns-apply.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT + +# 1. Every unreviewed dispatch stops before credentials, state or provider access. +for scenario in ref actor repository event attempt runner workflow confirmation plan-recover mode; do + ( + export GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF=refs/heads/main + export GITHUB_ACTOR=devantler GITHUB_REPOSITORY=devantler-tech/cloudflare + export GITHUB_RUN_ATTEMPT=1 GITHUB_RUN_ID=123456789 GITHUB_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + export RUNNER_ENVIRONMENT=github-hosted + export GITHUB_WORKFLOW_REF=devantler-tech/cloudflare/.github/workflows/dns-apply.yaml@refs/heads/main + export DNS_MODE=apply DNS_PHASE=change CONFIRMATION=APPLY_DNS + case $scenario in + ref) GITHUB_REF=refs/heads/unreviewed ;; + actor) GITHUB_ACTOR=unapproved ;; + repository) GITHUB_REPOSITORY=other/cloudflare ;; + event) GITHUB_EVENT_NAME=pull_request ;; + attempt) GITHUB_RUN_ATTEMPT=2 ;; + runner) RUNNER_ENVIRONMENT=self-hosted ;; + workflow) GITHUB_WORKFLOW_REF=devantler-tech/cloudflare/.github/workflows/other.yaml@refs/heads/main ;; + confirmation) DNS_MODE=plan ;; + plan-recover) DNS_MODE=plan CONFIRMATION=PLAN_DNS DNS_PHASE=recover ;; + mode) DNS_MODE=destroy ;; + esac + rc=0 + bash "$root/scripts/apply-dns.sh" "$root" > "$scratch/refusal" 2>&1 || rc=$? + [[ $rc == 2 ]] + grep -qx 'CLOUDFLARE_DNS_CHANGE=UNVERIFIED stage=dispatch' "$scratch/refusal" + ) +done +printf '%s\n' 'PASS: DNS changes refuse unreviewed dispatches before credentials or state' + +# 2. Import resolution: exact single match imports, unknown names stay creates, +# and drift, ambiguity or external-dns ownership refuses. +zone=11111111111111111111111111111111 +jq -n --arg zone "$zone" '{zone:{id:$zone}, records:[ + {id:"00000000000000000000000000000001",name:"devantler.tech",type:"MX",content:"mx01.mail.icloud.com"}, + {id:"00000000000000000000000000000002",name:"devantler.tech",type:"TXT",content:"\"v=spf1 include:icloud.com ~all\""}, + {id:"00000000000000000000000000000003",name:"devantler.tech",type:"TXT",content:"\"apple-domain=x\""}, + {id:"00000000000000000000000000000004",name:"dup.devantler.tech",type:"TXT",content:"\"same\""}, + {id:"00000000000000000000000000000005",name:"dup.devantler.tech",type:"TXT",content:"\"same\""}, + {id:"00000000000000000000000000000006",name:"portal.devantler.tech",type:"A",content:"192.0.2.1"}, + {id:"00000000000000000000000000000007",name:"_externaldns.a-portal.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""} +]}' > "$scratch/snapshot.json" +create() { jq -n --arg address "$1" --arg name "$2" --arg type "$3" --arg content "$4" \ + '{address:$address,mode:"managed",type:"cloudflare_dns_record",change:{actions:["create"],after:{name:$name,type:$type,content:$content}}}'; } +resolve() { jq -s '{resource_changes:.}' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq"; } +result=$( { create cloudflare_dns_record.mail_mx_01 devantler.tech MX mx01.mail.icloud.com + create cloudflare_dns_record.mail_spf devantler.tech TXT '"v=spf1 include:icloud.com ~all"' + create cloudflare_dns_record.marker _externaldns.cname-www.devantler.tech TXT '"heritage=external-dns"'; } | resolve) +jq -e --arg zone "$zone" '.refused==[] and .imports==[ + {address:"cloudflare_dns_record.mail_mx_01",id:($zone+"/00000000000000000000000000000001")}, + {address:"cloudflare_dns_record.mail_spf",id:($zone+"/00000000000000000000000000000002")}]' <<<"$result" >/dev/null +for refusal in content-differs:devantler.tech:TXT:'"v=spf1 -all"' ambiguous:dup.devantler.tech:TXT:'"same"' \ + external-dns-owned:portal.devantler.tech:A:192.0.2.1 external-dns-owned:_externaldns.a-portal.devantler.tech:TXT:'"heritage=external-dns"'; do + IFS=: read -r reason name type content <<<"$refusal" + result=$(create cloudflare_dns_record.case "$name" "$type" "$content" | resolve) + jq -e --arg reason "$reason" '.imports==[] and .refused==[{address:"cloudflare_dns_record.case",refused:$reason}]' <<<"$result" >/dev/null +done +result=$( { create cloudflare_dns_record.one devantler.tech MX mx01.mail.icloud.com + create cloudflare_dns_record.two devantler.tech MX mx01.mail.icloud.com; } | resolve) +jq -e '.refused==[{address:"*",refused:"duplicate-target"}]' <<<"$result" >/dev/null +printf '%s\n' 'PASS: declared records import only from one exact unowned live record' + +# 3. Plan boundary: imports must be unchanged, and nothing is deleted or replaced. +import_id="$zone/00000000000000000000000000000001" +printf '[{"address":"cloudflare_dns_record.mail_mx_01","id":"%s"}]\n' "$import_id" > "$scratch/imports.json" +record='{"name":"devantler.tech","type":"MX","content":"mx01.mail.icloud.com","ttl":3600}' +change() { jq -n --arg address "$1" --arg type "$2" --argjson actions "$3" --argjson before "$4" --argjson after "$5" --argjson importing "$6" \ + '{address:$address,mode:"managed",type:$type,change:({actions:$actions,before:$before,after:$after,after_unknown:{}} + (if $importing then {importing:$importing} else {} end))}'; } +plan() { jq -s '{resource_changes:.}'; } +check() { jq -e --slurpfile imports "$scratch/imports.json" -f "$root/scripts/check-dns-plan.jq" >/dev/null; } +noop_import=$(change cloudflare_dns_record.mail_mx_01 cloudflare_dns_record '["no-op"]' "$record" "$record" "{\"id\":\"$import_id\"}") +zone_noop=$(change cloudflare_zone.primary cloudflare_zone '["no-op"]' '{}' '{}' null) +marker='{"name":"_externaldns.cname-www.devantler.tech","type":"TXT","content":"x"}' +{ printf '%s\n' "$noop_import" "$zone_noop" + change cloudflare_dns_record.marker cloudflare_dns_record '["create"]' null "$marker" null + change cloudflare_dns_record.apex cloudflare_dns_record '["update"]' '{"content":"a"}' '{"content":"b"}' null + change cloudflare_dns_record.www cloudflare_dns_record '["forget"]' '{"content":"a"}' null null +} | plan | check +reject() { if plan | check; then printf 'FAIL: %s was admitted\n' "$1" >&2; exit 1; fi; } +change cloudflare_dns_record.mail_mx_01 cloudflare_dns_record '["no-op"]' "$record" '{"name":"devantler.tech","type":"MX","content":"mx01.mail.icloud.com","ttl":300}' "{\"id\":\"$import_id\"}" | reject changing-import +change cloudflare_dns_record.mail_mx_01 cloudflare_dns_record '["update"]' "$record" "$record" "{\"id\":\"$import_id\"}" | reject updating-import +change cloudflare_dns_record.mail_mx_01 cloudflare_dns_record '["no-op"]' "$record" "$record" '{"id":"other/id"}' | reject wrong-import-target +printf '%s\n' "$zone_noop" | reject missing-import +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.mail_spf cloudflare_dns_record '["delete"]' "$record" null null; } | reject delete +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.mail_spf cloudflare_dns_record '["delete","create"]' "$record" "$record" null; } | reject replace +{ printf '%s\n' "$noop_import"; change cloudflare_zone.primary cloudflare_zone '["update"]' '{"a":1}' '{"a":2}' null; } | reject zone-update +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.bad cloudflare_dns_record '["create"]' null '{"name":"_externaldns.devantler.tech","type":"TXT","content":"x"}' null; } | reject typeless-apex-registry +printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if check; then exit 1; fi +printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi +printf '%s\n' 'PASS: DNS plans admit unchanged imports, creates, updates and forgets only' + +# 4. The declared mail records are the ones the company Email depends on. +grep -q 'content = "mx01.mail.icloud.com"' "$root/dns.tf" +grep -q 'content = "mx02.mail.icloud.com"' "$root/dns.tf" +if grep -q '"_externaldns.devantler.tech"' "$root/dns.tf"; then exit 1; fi +printf '%s\n' 'PASS: declared DNS keeps the Email records and never the typeless apex registry' diff --git a/scripts/tests/test-email-dns-plan.sh b/scripts/tests/test-email-dns-plan.sh deleted file mode 100644 index d6710cc..0000000 --- a/scripts/tests/test-email-dns-plan.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 -root=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -mkdir "$scratch/bin" "$scratch/root" "$scratch/root/.github" -for file in main.tf variables.tf versions.tf providers.tf outputs.tf .terraform.lock.hcl; do cp "$root/$file" "$scratch/root/$file"; done -age-keygen -o "$scratch/key" 2>/dev/null -age-keygen -y "$scratch/key" > "$scratch/root/.github/dns-recovery-recipient.txt" -cat > "$scratch/bin/curl" <<'CURL' -#!/usr/bin/env bash -set -euo pipefail -[[ -z ${CLOUDFLARE_API_TOKEN:-} ]] -url='' -while [[ $# -gt 0 ]]; do - case $1 in - --disable|--silent|--fail) shift ;; - --config) [[ $2 == - ]]; shift 2 ;; - --request) [[ $2 == GET ]]; shift 2 ;; - --proto) [[ $2 == '=https' ]]; shift 2 ;; - --noproxy) [[ $2 == '*' ]]; shift 2 ;; - --max-redirs) [[ $2 == 0 ]]; shift 2 ;; - --connect-timeout|--max-time|--max-filesize) shift 2 ;; - --url) url=$2;shift 2 ;; - *) exit 80 ;; - esac -done -IFS= read -r header -[[ $header == 'header = "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"' ]] -printf '%s\n' "$url" >> "$FIXTURE_DIR/api" -case $url in -'https://api.cloudflare.com/client/v4/zones?name=devantler.tech&per_page=50') - jq -n '{success:true,result:[{id:"11111111111111111111111111111111",name:"devantler.tech",status:"active",account:{id:"22222222222222222222222222222222"}}],result_info:{total_count:1,total_pages:1}}' ;; -'https://api.cloudflare.com/client/v4/zones/11111111111111111111111111111111/dns_records?per_page=1000&page=1') - jq -n '{success:true,result: [ - {id:"00000000000000000000000000000001",name:"devantler.tech",type:"MX",content:"mx.example.test",priority:10}, - {id:"00000000000000000000000000000002",name:"devantler.tech",type:"TXT",content:"\"v=spf1 -all\""}, - {id:"00000000000000000000000000000003",name:"_dmarc.devantler.tech",type:"TXT",content:"\"v=DMARC1; p=reject\""}, - {id:"00000000000000000000000000000004",name:"sig1._domainkey.devantler.tech",type:"CNAME",content:"sig1.keys.example.test"}, - {id:"00000000000000000000000000000005",name:"devantler.tech",type:"TXT",content:"\"apple-domain=PRIVATE_FIXTURE_DNS\""} - ]|map({ttl:300,proxied:false,comment:null,tags:[],settings:{}}+.),result_info:{page:1,per_page:1000,count:5,total_count:5,total_pages:1}}' ;; -'https://api.cloudflare.com/client/v4/accounts/22222222222222222222222222222222/r2/buckets/cloudflare-bootstrap-state') - [[ $FIXTURE_CASE != bucket-denied ]] || exit 22 - jq -n '{success:true,result:{name:"cloudflare-bootstrap-state",jurisdiction:"default",location:"EEUR",storage_class:"Standard"}}' ;; -*)exit 81 ;; -esac -CURL -cat > "$scratch/bin/tofu" <<'TOFU' -#!/usr/bin/env bash -set -euo pipefail -[[ $1 == -chdir=* ]] -work=${1#-chdir=};shift -[[ ${CLOUDFLARE_API_TOKEN:-} == AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA && -z ${TF_LOG:-} && -z ${TF_CLI_ARGS_plan:-} && -z ${CLOUDFLARE_API_BASE_URL:-} ]] -printf '%s\n' "$1" >> "$FIXTURE_DIR/cli" -printf '%s\n' PRIVATE_FIXTURE_PROVIDER_OUTPUT >&2 -case $1 in - init) - [[ $* == 'init -input=false -lockfile=readonly' ]] - [[ $(cat "$work/backend_override.tf") == $'terraform {\n backend "local" {}\n}' ]] ;; - import) - [[ $# == 4 && $2 == -input=false ]] - jq -n --arg address "$3" --arg id "$4" '{address:$address,id:$id}' >> "$work/imports.executed.jsonl" ;; - plan) - [[ $* == 'plan -input=false -detailed-exitcode -out=migration.tfplan' ]] - jq -s -e 'length==7' "$work/imports.executed.jsonl" >/dev/null - printf '%s' PRIVATE_FIXTURE_SAVED_PLAN > "$work/migration.tfplan" - jq -s --arg scenario "$FIXTURE_CASE" '{ - errored:false, - resource_changes:[.[]|{address,mode:"managed",type:(if .address=="cloudflare_zone.primary" then "cloudflare_zone" elif (.address|startswith("cloudflare_r2_bucket.")) then "cloudflare_r2_bucket" else "cloudflare_dns_record" end),change:{actions:["no-op"],before:{id:(if .address=="cloudflare_zone.primary" then .id else (.id|split("/")[1]) end)},after:{id:(if .address=="cloudflare_zone.primary" then .id else (.id|split("/")[1]) end)},after_unknown:{}}}] - }|if $scenario=="plan-update" then .resource_changes[0].change.actions=["update"] else . end' "$work/imports.executed.jsonl" > "$work/fixture-plan.json" ;; - show) - [[ $* == 'show -json migration.tfplan' ]] - cat "$work/fixture-plan.json" ;; - *)exit 82 ;; -esac -TOFU -chmod +x "$scratch/bin/curl" "$scratch/bin/tofu" -# Exercise real encryption transport around controlled provider and CLI boundaries. -run_case(){ - local name=$1 expected=$2 ref=refs/heads/main confirmation=PLAN_EMAIL_DNS token=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA rc=0 - local dir="$scratch/$name" - mkdir "$dir"; : > "$dir/api"; : > "$dir/cli" - case $name in - wrong-ref)ref=refs/heads/feature;; - wrong-confirmation)confirmation=APPLY;; - missing-token)token='';; - esac - env PATH="$scratch/bin:$PATH" FIXTURE_DIR="$dir" FIXTURE_CASE="$name" \ - GITHUB_EVENT_NAME=workflow_dispatch GITHUB_REF="$ref" GITHUB_REPOSITORY=devantler-tech/cloudflare GITHUB_ACTOR=devantler GITHUB_RUN_ATTEMPT=1 GITHUB_RUN_ID=12345 GITHUB_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ - GITHUB_WORKFLOW_REF=devantler-tech/cloudflare/.github/workflows/email-dns-proposal.yaml@refs/heads/main RUNNER_ENVIRONMENT=github-hosted \ - CONFIRMATION="$confirmation" CLOUDFLARE_API_TOKEN="$token" TF_LOG=TRACE TF_CLI_ARGS_plan=-destroy CLOUDFLARE_API_BASE_URL=https://untrusted.invalid \ - OUTPUT_DIR="$dir/output" DIAGNOSTICS_DIR="$dir/diagnostics" bash "$root/scripts/plan-email-dns.sh" "$scratch/root" > "$dir/log" 2>&1 || rc=$? - [[ $rc == "$expected" ]] || { echo "FAIL: Email proposal $name exit=$rc"; cat "$dir/log"; exit 1; } - if grep -Eq 'AAAAAAAAAAAA|111111111111|222222222222|PRIVATE_FIXTURE_|Authorization:' "$dir/log"; then echo 'FAIL: private proposal detail escaped';exit 1;fi - if [[ $expected == 0 ]]; then - [[ $(wc -l < "$dir/api") -eq 3 && $(wc -l < "$dir/cli") -eq 10 ]] - grep -q '^EMAIL_DNS_PROPOSAL=ENCRYPTED_UNCHANGED provider_writes=0$' "$dir/log" - [[ $(find "$dir/output" -type f | wc -l) -eq 2 ]] - age --decrypt -i "$scratch/key" "$dir/output/proposal.tar.gz.age" > "$dir/bundle.tar.gz" 2>/dev/null - mkdir "$dir/plain";tar -xzf "$dir/bundle.tar.gz" -C "$dir/plain" - jq -e '.kind=="cloudflare-email-dns-proposal" and .providerWrites==0 and .sourceRun==12345' "$dir/output/manifest.json" >/dev/null - jq -e '.kind=="cloudflare-email-dns-proposal" and .providerWrites==0' "$dir/plain/metadata.json" >/dev/null - jq -e '.dns_records|length==5' "$dir/plain/work/production.auto.tfvars.json" >/dev/null - [[ $(cat "$dir/plain/work/migration.tfplan") == PRIVATE_FIXTURE_SAVED_PLAN ]] - [[ ! -e "$dir/diagnostics" ]] - else - [[ ! -e "$dir/output" ]] - case $name in wrong-ref|wrong-confirmation|missing-token) [[ ! -s "$dir/api" && ! -s "$dir/cli" ]];;esac - fi -} -run_case success 0 -run_case wrong-ref 2 -run_case wrong-confirmation 2 -run_case missing-token 2 -run_case bucket-denied 2 -run_case plan-update 2 -printf '%s\n' 'PASS: private native Email DNS proposal lifecycle and refusal controls' - -mkdir "$scratch/parse" -workflow="$root/.github/workflows/email-dns-proposal.yaml" -expression=$(jq -nr --arg path "$workflow" '"jsonencode(yamldecode(file(" + ($path|tojson) + ")))"') -printf '%s\n' "$expression" | tofu -chdir="$scratch/parse" console -no-color > "$scratch/workflow.string.json" -jq -r . "$scratch/workflow.string.json" > "$scratch/workflow.json" -# Check declared workflow custody and reject privilege, trigger and export changes. -check_workflow(){ - jq -e ' - .permissions=={contents:"read"} and (.on|keys)==["workflow_dispatch"] and - (.jobs|keys)==["plan"] and .jobs.plan["runs-on"]=="ubuntu-latest" and .jobs.plan["timeout-minutes"]==12 and - .concurrency=={group:"cloudflare-bootstrap-migration","cancel-in-progress":false} and - (.env // {})=={} and (.jobs.plan.env // {})=={} and (.jobs.plan.permissions // {})=={} and - (.jobs.plan.steps|length)==7 and - all(.jobs.plan.steps[];(has("continue-on-error")|not)) and - .jobs.plan.steps[0].env=={CONFIRMATION:"${{ inputs.confirmation }}"} and - (.jobs.plan.steps[0].run|contains("${{")|not) and - .jobs.plan.steps[1].with=={ref:"${{ github.sha }}","persist-credentials":false} and - .jobs.plan.steps[2].with=={tofu_version:"1.12.6",tofu_wrapper:false} and - .jobs.plan.steps[4].run=="bash scripts/plan-email-dns.sh \"$GITHUB_WORKSPACE\"" and - .jobs.plan.steps[4].env=={CONFIRMATION:"${{ inputs.confirmation }}",CLOUDFLARE_API_TOKEN:"${{ secrets.CLOUDFLARE_API_TOKEN }}",OUTPUT_DIR:"${{ runner.temp }}/email-dns-proposal",DIAGNOSTICS_DIR:"${{ runner.temp }}/email-dns-diagnostics"} and - .jobs.plan.steps[5].uses=="actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9" and - .jobs.plan.steps[5].with.path=="${{ runner.temp }}/email-dns-proposal/proposal.tar.gz.age\n${{ runner.temp }}/email-dns-proposal/manifest.json\n" and - .jobs.plan.steps[5].with["retention-days"]==1 and - .jobs.plan.steps[6].with.path=="${{ runner.temp }}/email-dns-diagnostics/diagnostics.tar.gz.age" and - .jobs.plan.steps[6].with["retention-days"]==1 - ' "$1" >/dev/null -} -check_workflow "$scratch/workflow.json" -jq -r '.jobs.plan.steps[0].run' "$scratch/workflow.json" > "$scratch/guard.sh" -for scenario in accepted wrong-ref wrong-workflow wrong-runner wrong-actor retry wrong-event wrong-confirmation;do - ref=refs/heads/main workflow_ref=devantler-tech/cloudflare/.github/workflows/email-dns-proposal.yaml@refs/heads/main runner=github-hosted actor=devantler attempt=1 event=workflow_dispatch confirmation=PLAN_EMAIL_DNS - case $scenario in - wrong-ref)ref=refs/heads/feature;; - wrong-workflow)workflow_ref=devantler-tech/cloudflare/.github/workflows/other.yaml@refs/heads/main;; - wrong-runner)runner=self-hosted;; - wrong-actor)actor=foreign;; - retry)attempt=2;; - wrong-event)event=pull_request;; - wrong-confirmation)confirmation='PLAN_EMAIL_DNS"; echo unexpected';; - esac - rc=0 - env GITHUB_EVENT_NAME="$event" GITHUB_REF="$ref" GITHUB_REPOSITORY=devantler-tech/cloudflare GITHUB_ACTOR="$actor" GITHUB_RUN_ATTEMPT="$attempt" GITHUB_WORKFLOW_REF="$workflow_ref" RUNNER_ENVIRONMENT="$runner" CONFIRMATION="$confirmation" bash "$scratch/guard.sh" > "$scratch/guard.log" 2>&1 || rc=$? - if [[ $scenario == accepted ]];then [[ $rc == 0 ]];else [[ $rc == 2 ]];fi -done -for mutation in '.permissions.contents="write"' '.on.push={}' '.jobs.plan.steps[5].with.path="/tmp"' '.jobs.plan.steps[4].run="tofu apply"' '.jobs.plan.steps[0].env.CLOUDFLARE_API_TOKEN="early"';do - jq "$mutation" "$scratch/workflow.json" > "$scratch/workflow-mutant.json" - if check_workflow "$scratch/workflow-mutant.json";then echo 'FAIL: unsafe Email proposal workflow was accepted' >&2;exit 1;fi -done -printf '%s\n' 'PASS: main-only Email proposal workflow guards and ciphertext boundaries' diff --git a/scripts/tests/test-email-dns-proposal.sh b/scripts/tests/test-email-dns-proposal.sh deleted file mode 100644 index 88d50c6..0000000 --- a/scripts/tests/test-email-dns-proposal.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -umask 077 -root=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd) -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -cp "$root/variables.tf" "$scratch/variables.tf" -jq -n '{account_id:"11111111111111111111111111111111",zone_name:"example.test",dns_records:{email:{name:"example.test",type:"TXT",content:"\"v=spf1 -all\"",comment:null}}}' > "$scratch/fixture.tfvars.json" -printf '%s\n' 'jsonencode({comment=var.dns_records["email"].comment,tags=var.dns_records["email"].tags})' | - tofu -chdir="$scratch" console -no-color -var-file=fixture.tfvars.json > "$scratch/result.string.json" -jq -r . "$scratch/result.string.json" > "$scratch/result.json" -jq -e '.comment==null and .tags==[]' "$scratch/result.json" >/dev/null || { - printf '%s\n' 'FAIL: existing Email DNS metadata would be changed by defaults' >&2 - exit 1 -} -printf '%s\n' 'PASS: imported Email DNS keeps absent metadata' - -jq -n '{ - schema:1,source_revision:"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",captured_at:"2026-11-03T00:00:00Z", - zone:{id:"11111111111111111111111111111111",name:"example.test",status:"active",account:{id:"22222222222222222222222222222222"}}, - records: [ - {id:"00000000000000000000000000000001",name:"example.test",type:"MX",content:"mx1.example.test",priority:10}, - {id:"00000000000000000000000000000002",name:"example.test",type:"MX",content:"mx2.example.test",priority:10}, - {id:"00000000000000000000000000000003",name:"example.test",type:"TXT",content:"\"v=spf1 -all\""}, - {id:"00000000000000000000000000000004",name:"_dmarc.example.test",type:"TXT",content:"\"v=DMARC1; p=reject\""}, - {id:"00000000000000000000000000000005",name:"sig1._domainkey.example.test",type:"CNAME",content:"sig1.keys.example.test",settings:{flatten_cname:false}}, - {id:"00000000000000000000000000000006",name:"example.test",type:"TXT",content:"\"apple-domain=private-fixture\""}, - {id:"00000000000000000000000000000007",name:"app.example.test",type:"A",content:"192.0.2.1"}, - {id:"00000000000000000000000000000008",name:"_acme-challenge.example.test",type:"TXT",content:"temporary-fixture"}, - {id:"00000000000000000000000000000009",name:"_externaldns.a-app.example.test",type:"TXT",content:"heritage=external-dns,external-dns/owner=example.test"} - ] | map({ttl:300,proxied:false,comment:null,tags:[],settings:{}}+.) -}' > "$scratch/snapshot.json" -# Exercise the production selector against complete synthetic DNS snapshots. -build() { - jq --arg zone example.test --arg revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -f "$root/scripts/email-dns-proposal.jq" "$1" -} -build "$scratch/snapshot.json" > "$scratch/proposal.json" -jq -e ' - (.dns_records|length)==6 and (.imports|length)==6 and - .dns_records.email_00000000000000000000000000000003.content=="\"v=spf1 -all\"" and - .dns_records.email_00000000000000000000000000000005.settings.flatten_cname==false and - .dns_records.email_00000000000000000000000000000001.priority==10 and - all(.dns_records[];.comment==null and .tags==[]) and - all(.imports[];.address|startswith("cloudflare_dns_record.records[\"email_")) -' "$scratch/proposal.json" >/dev/null -for mutation in \ - '.records += [.records[0]]' \ - '.zone.name="foreign.test"' \ - '.source_revision="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"' \ - '.records[0].priority=null' \ - '.records[2].proxied=true' \ - '.records[4].settings.unsupported=true' \ - '.records[4].meta={managed_by_apps:true}' \ - '.records[4].tags=["owner:external-dns"]' \ - '.records[4].tags=["owner:other-controller"]' \ - '.records[4].meta={managed_by_future_controller:true}' \ - '.records[8].name="_externaldns.cname-sig1._domainkey.example.test"' \ - '.records[8].name="_externaldns.sig1._domainkey.example.test"' \ - '.records[2].content="unrecognized"' \ - '.records[1].id="../foreign"' \ - '.records[3].ttl=0'; do - jq "$mutation" "$scratch/snapshot.json" > "$scratch/mutant.json" - if build "$scratch/mutant.json" > "$scratch/rejected.json" 2> "$scratch/rejected.log"; then - echo 'FAIL: ambiguous or controller-owned Email DNS was accepted' >&2; exit 1 - fi - if grep -q 'private-fixture' "$scratch/rejected.log"; then - echo 'FAIL: DNS proposal exposed record content' >&2; exit 1 - fi -done -printf '%s\n' 'PASS: only existing unambiguous static Email DNS is proposed' -jq '.records[4].tags=["owner:cloudflare-repo","env:production"]|.records[4].comment="Preserve this human note"' "$scratch/snapshot.json" > "$scratch/known-owner.json" -build "$scratch/known-owner.json" > "$scratch/known-owner-proposal.json" -jq -e '.dns_records.email_00000000000000000000000000000005.tags==["owner:cloudflare-repo","env:production"] and .dns_records.email_00000000000000000000000000000005.comment=="Preserve this human note"' "$scratch/known-owner-proposal.json" >/dev/null - -jq --slurpfile proposal "$scratch/proposal.json" '.dns_records=$proposal[0].dns_records' "$scratch/fixture.tfvars.json" > "$scratch/proposal.tfvars.json" -printf '%s\n' 'jsonencode(var.dns_records)' | tofu -chdir="$scratch" console -no-color -var-file=proposal.tfvars.json > "$scratch/roundtrip.string.json" -jq -r . "$scratch/roundtrip.string.json" > "$scratch/roundtrip.json" -jq -e '.email_00000000000000000000000000000005.settings.flatten_cname==false and .email_00000000000000000000000000000003.content=="\"v=spf1 -all\""' "$scratch/roundtrip.json" >/dev/null || { - echo 'FAIL: declared DNS variables discarded existing provider settings' >&2; exit 1 -} -printf '%s\n' 'PASS: native variables preserve the complete proposed DNS values' - -jq '.imports + [{address:"cloudflare_zone.primary",id:"11111111111111111111111111111111"},{address:"cloudflare_r2_bucket.buckets[\"cloudflare-bootstrap-state\"]",id:"22222222222222222222222222222222/cloudflare-bootstrap-state/default"}]' "$scratch/proposal.json" > "$scratch/imports.json" -jq -n --slurpfile wanted "$scratch/imports.json" '{errored:false,resource_changes:[$wanted[0][]|. as $r|{address,mode:"managed",type:(if .address=="cloudflare_zone.primary" then "cloudflare_zone" elif (.address|startswith("cloudflare_r2_bucket.")) then "cloudflare_r2_bucket" else "cloudflare_dns_record" end),change:{actions:["no-op"],before:{id:(if .address=="cloudflare_zone.primary" then .id else (.id|split("/")[1]) end)},after:{id:(if .address=="cloudflare_zone.primary" then .id else (.id|split("/")[1]) end)},after_unknown:{}}}]}' > "$scratch/plan.json" -# Exercise the production no-change guard against explicit imported-plan fixtures. -check_plan() { jq -e --slurpfile wanted "$scratch/imports.json" -f "$root/scripts/check-email-dns-plan.jq" "$1" >/dev/null; } -check_plan "$scratch/plan.json" -for mutation in \ - '.resource_changes[0].change.actions=["update"]' \ - '.resource_changes[0].change.actions=["delete","create"]' \ - '.resource_changes[0].change.actions=["create"]' \ - '.resource_changes[0].change.after.id="foreign"' \ - '.resource_changes[0].change.before.id="foreign"' \ - '.resource_changes[0].change.after_unknown.id=true' \ - '.resource_changes[0].mode="data"' \ - '.resource_changes += [.resource_changes[0]]' \ - 'del(.resource_changes[0])' \ - '.resource_changes[0].address="foreign"' \ - '.deferred_changes=[{}]' \ - '.resource_drift=[{}]' \ - '.errored=true'; do - jq "$mutation" "$scratch/plan.json" > "$scratch/mutant-plan.json" - if check_plan "$scratch/mutant-plan.json" 2>/dev/null; then echo 'FAIL: an incomplete or changing provider plan was accepted' >&2; exit 1; fi -done -printf '%s\n' 'PASS: only the complete unchanged imported provider plan is accepted' From a53b0eaf4c5136939c1ac946a89d6c21bbeb6193 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 02:53:05 +0100 Subject: [PATCH 02/11] fix(dns): recover on a fresh runner only after a successful apply Co-Authored-By: Claude Opus 5.5 --- .github/workflows/dns-apply.yaml | 2 +- docs/runbooks/bootstrap-and-recovery.md | 2 +- scripts/apply-dns.sh | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dns-apply.yaml b/.github/workflows/dns-apply.yaml index f5bd76b..ad1b8bc 100644 --- a/.github/workflows/dns-apply.yaml +++ b/.github/workflows/dns-apply.yaml @@ -93,7 +93,7 @@ jobs: recover: name: Recover from R2 on a fresh runner and require no changes needs: change - if: ${{ always() && !cancelled() && inputs.mode == 'apply' && needs.change.result != 'skipped' }} + if: ${{ inputs.mode == 'apply' && needs.change.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 15 steps: diff --git a/docs/runbooks/bootstrap-and-recovery.md b/docs/runbooks/bootstrap-and-recovery.md index 6d2eb81..5ca1018 100644 --- a/docs/runbooks/bootstrap-and-recovery.md +++ b/docs/runbooks/bootstrap-and-recovery.md @@ -372,7 +372,7 @@ nothing owns. Values, identifiers and tool output stay in the encrypted evidence. In apply mode the run encrypts the state before and after the apply, even -when the apply fails, and then requires a clean plan. A second job initializes +when the apply fails, and then requires a clean plan. After a successful apply, a second job initializes from R2 on a fresh runner and must also find nothing to change. Afterwards, download the `encrypted-dns-change` and `encrypted-dns-recovery` artifacts, authenticate the `after` and `recovered` packages with the independent diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 7d208ef..3de15bd 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -101,8 +101,7 @@ plan_json() { tf local show -json "$name.tfplan" > "$scratch/evidence/$name.json" 2> "$scratch/evidence/$name-show.log" || return 1 state_unique_json "$scratch/evidence/$name.json" } -# Bind the live state to the retained recovery generation: an apply that was -# not yet retained, or a different lineage, refuses every further change. +# Encrypt the current state with the unchanged configuration as one package. capture() { local name=$1 tf local state pull > "$scratch/evidence/$name-state.json" 2> "$scratch/evidence/$name-pull.log" || return 1 @@ -136,6 +135,8 @@ jq -e '.terraform_version=="1.12.6"' "$scratch/evidence/version.json" >/dev/null stage=initialize tf local init -input=false -lockfile=readonly -no-color -backend-config="$scratch/backend.hcl" > "$scratch/evidence/init.log" 2>&1 || stop stage=recovery-generation +# Bind the live state to the retained recovery generation: an apply that was +# not yet retained, or a different lineage, refuses every further change. tf local state pull > "$scratch/evidence/start-state.json" 2> "$scratch/evidence/start-pull.log" || stop state_unique_json "$scratch/evidence/start-state.json" || stop if [[ $DNS_PHASE == change ]]; then From df8da4b3b461a01c0fc46598d6af1011d553d618 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 02:55:23 +0100 Subject: [PATCH 03/11] feat(dns): hand www to the platform's external-dns without a gap Create external-dns's ownership record for the existing www CNAME and forget the CNAME here, so external-dns replaces it with the gateway addresses in one batch. Part of #21. Co-Authored-By: Claude Opus 5.5 --- dns.tf | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/dns.tf b/dns.tf index b9afa55..0192f2f 100644 --- a/dns.tf +++ b/dns.tf @@ -22,15 +22,26 @@ resource "cloudflare_dns_record" "apex" { settings = { flatten_cname = false } } -resource "cloudflare_dns_record" "www" { +# www moves to the platform: external-dns serves it from the business site's +# public route. This ownership record lets external-dns take over the existing +# CNAME and replace it with the gateway addresses in one batch, so www always +# resolves. The CNAME itself stays in Cloudflare; this repository only forgets it. +resource "cloudflare_dns_record" "www_external_dns_owner" { zone_id = cloudflare_zone.primary.id - name = "www.devantler.tech" - type = "CNAME" - content = "devantler.github.io" + name = "_externaldns.cname-www.devantler.tech" + type = "TXT" + content = "\"heritage=external-dns,external-dns/owner=platform.devantler.tech,external-dns/resource=httproute/business-site/business-site-public\"" ttl = 1 - proxied = true - comment = "GitHub Pages Personal" - settings = { flatten_cname = false } + proxied = false + settings = {} +} + +removed { + from = cloudflare_dns_record.www + + lifecycle { + destroy = false + } } resource "cloudflare_dns_record" "ksail_docs" { From b9b3b57a9b953ea3c2ce73ff37ab7524fa5985e3 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 03:21:12 +0100 Subject: [PATCH 04/11] fix(dns): close the review's ownership, name and emergency-state gaps - Refuse a create that an external-dns ownership record already claims, even when the claimed record itself is missing. - Compare record names in full lower-case form, so a relative or differently written `_externaldns` name cannot pass the apex guard. - Keep OpenTofu's errored.tfstate in the encrypted evidence when an apply cannot save its state to R2. Co-Authored-By: Claude Opus 5.5 --- scripts/apply-dns.sh | 6 +++++- scripts/check-dns-plan.jq | 9 +++++++-- scripts/dns-imports.jq | 22 ++++++++++++++-------- scripts/tests/test-dns-apply.sh | 9 +++++++-- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 3de15bd..2d939b6 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -203,7 +203,11 @@ capture before || stop stage=apply apply_rc=0 tf provider apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? -# Capture whatever the apply left behind before judging it. +# Capture whatever the apply left behind before judging it. When OpenTofu +# cannot save to R2 it writes errored.tfstate; keep that in the encrypted +# evidence first, because the remote copy may then be older than the zone. +stage=errored-state +if [[ -f $scratch/work/errored.tfstate ]]; then cp "$scratch/work/errored.tfstate" "$scratch/evidence/errored.tfstate"; fi stage=after-capture capture after || stop stage=apply diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq index c6a01d8..5a988d8 100644 --- a/scripts/check-dns-plan.jq +++ b/scripts/check-dns-plan.jq @@ -1,8 +1,13 @@ -# Input: the saved plan JSON. $imports[0]: the resolved import list. +# Input: the saved plan JSON. $imports[0]: the resolved import list. Names are +# compared in their full, lower-case form, so a relative name cannot slip by. # Only DNS records may change. Nothing is deleted or replaced. Every import is # known and leaves its record exactly as it is. The typeless apex registry name # is refused outright: external-dns would read it as owning the apex mail # records and delete them. +def fqdn: ascii_downcase | rtrimstr(".") | + if .=="@" then "devantler.tech" + elif .=="devantler.tech" or endswith(".devantler.tech") then . + else .+".devantler.tech" end; def changed: (.change.before != .change.after) or any(.change.after_unknown|..; .==true); . as $plan | $imports[0] as $wanted | .errored!=true and @@ -22,5 +27,5 @@ all(.resource_changes[]; any($wanted[]; .address==$resource.address) and .change.actions==["no-op"] and (changed|not) else true end) and (if .type=="cloudflare_dns_record" then - ((.change.after // {}).name != "_externaldns.devantler.tech") + (((.change.after // {}).name // "")|fqdn) != "_externaldns.devantler.tech" else .change.actions==["no-op"] end)) diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq index e57248e..0fe756e 100644 --- a/scripts/dns-imports.jq +++ b/scripts/dns-imports.jq @@ -1,20 +1,26 @@ # Input: a plan JSON made before imports. $snapshot[0]: the complete live zone. # A declared record the plan would create is imported when exactly one live # record has the same name, type and content. It stays a create only when no -# live record has that name and type. Anything else is refused, so a typo or -# drift can never add a second mail record or take over an external-dns record. -def registry: .type=="TXT" and (.name|startswith("_externaldns.")); +# live record has that name and type and no external-dns ownership record +# claims it. Anything else is refused, so a typo or drift can never add a +# second mail record or take over an external-dns record. +def fqdn: ascii_downcase | rtrimstr(".") | + if .=="@" then "devantler.tech" + elif .=="devantler.tech" or endswith(".devantler.tech") then . + else .+".devantler.tech" end; +def registry: .type=="TXT" and (.name|fqdn|startswith("_externaldns.")); def owned($records): - . as $r | any($records[]; - .type=="TXT" and - (.name=="_externaldns."+$r.name or .name=="_externaldns."+($r.type|ascii_downcase)+"-"+$r.name)); + (.name|fqdn) as $name | (.type|ascii_downcase) as $type | any($records[]; + .type=="TXT" and (.name|fqdn) as $marker | + ($marker=="_externaldns."+$name or $marker=="_externaldns."+$type+"-"+$name)); $snapshot[0] as $live | [.resource_changes[] | select(.mode=="managed" and .type=="cloudflare_dns_record" and .change.actions==["create"]) | .change.after as $after | - [$live.records[] | select(.name==$after.name and .type==$after.type)] as $same | + [$live.records[] | select((.name|fqdn)==($after.name|fqdn) and .type==$after.type)] as $same | [$same[] | select(.content==$after.content)] as $exact | - if ($exact|length)==1 then + if ($after|registry|not) and ($after|owned($live.records)) then {address, refused:"external-dns-owned"} + elif ($exact|length)==1 then if ($exact[0]|registry) or ($exact[0]|owned($live.records)) then {address, refused:"external-dns-owned"} else {address, id:($live.zone.id+"/"+$exact[0].id)} end elif ($exact|length)>1 then {address, refused:"ambiguous"} diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh index 52ee5c5..ec26a13 100755 --- a/scripts/tests/test-dns-apply.sh +++ b/scripts/tests/test-dns-apply.sh @@ -43,7 +43,8 @@ jq -n --arg zone "$zone" '{zone:{id:$zone}, records:[ {id:"00000000000000000000000000000004",name:"dup.devantler.tech",type:"TXT",content:"\"same\""}, {id:"00000000000000000000000000000005",name:"dup.devantler.tech",type:"TXT",content:"\"same\""}, {id:"00000000000000000000000000000006",name:"portal.devantler.tech",type:"A",content:"192.0.2.1"}, - {id:"00000000000000000000000000000007",name:"_externaldns.a-portal.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""} + {id:"00000000000000000000000000000007",name:"_externaldns.a-portal.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""}, + {id:"00000000000000000000000000000008",name:"_externaldns.a-ghost.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""} ]}' > "$scratch/snapshot.json" create() { jq -n --arg address "$1" --arg name "$2" --arg type "$3" --arg content "$4" \ '{address:$address,mode:"managed",type:"cloudflare_dns_record",change:{actions:["create"],after:{name:$name,type:$type,content:$content}}}'; } @@ -55,7 +56,8 @@ jq -e --arg zone "$zone" '.refused==[] and .imports==[ {address:"cloudflare_dns_record.mail_mx_01",id:($zone+"/00000000000000000000000000000001")}, {address:"cloudflare_dns_record.mail_spf",id:($zone+"/00000000000000000000000000000002")}]' <<<"$result" >/dev/null for refusal in content-differs:devantler.tech:TXT:'"v=spf1 -all"' ambiguous:dup.devantler.tech:TXT:'"same"' \ - external-dns-owned:portal.devantler.tech:A:192.0.2.1 external-dns-owned:_externaldns.a-portal.devantler.tech:TXT:'"heritage=external-dns"'; do + external-dns-owned:portal.devantler.tech:A:192.0.2.1 external-dns-owned:_externaldns.a-portal.devantler.tech:TXT:'"heritage=external-dns"' \ + external-dns-owned:ghost.devantler.tech:A:192.0.2.9 content-differs:Devantler.tech.:TXT:'"v=spf1 -all"' content-differs:@:MX:mx99.example.test; do IFS=: read -r reason name type content <<<"$refusal" result=$(create cloudflare_dns_record.case "$name" "$type" "$content" | resolve) jq -e --arg reason "$reason" '.imports==[] and .refused==[{address:"cloudflare_dns_record.case",refused:$reason}]' <<<"$result" >/dev/null @@ -90,6 +92,9 @@ printf '%s\n' "$zone_noop" | reject missing-import { printf '%s\n' "$noop_import"; change cloudflare_dns_record.mail_spf cloudflare_dns_record '["delete","create"]' "$record" "$record" null; } | reject replace { printf '%s\n' "$noop_import"; change cloudflare_zone.primary cloudflare_zone '["update"]' '{"a":1}' '{"a":2}' null; } | reject zone-update { printf '%s\n' "$noop_import"; change cloudflare_dns_record.bad cloudflare_dns_record '["create"]' null '{"name":"_externaldns.devantler.tech","type":"TXT","content":"x"}' null; } | reject typeless-apex-registry +for name in _externaldns _EXTERNALDNS.devantler.tech. _externaldns.Devantler.Tech; do + { printf '%s\n' "$noop_import"; change cloudflare_dns_record.bad cloudflare_dns_record '["create"]' null "{\"name\":\"$name\",\"type\":\"TXT\",\"content\":\"x\"}" null; } | reject "typeless-apex-registry $name" +done printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if check; then exit 1; fi printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi printf '%s\n' 'PASS: DNS plans admit unchanged imports, creates, updates and forgets only' From 1446a879b593ff193b6f703e024010c8b473397d Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 03:26:41 +0100 Subject: [PATCH 05/11] fix(dns): bind the ownership marker name before testing it jq 1.8 parses `X and Y as $v | body` with the whole and-expression as the binding, so the external-dns ownership check always found no owner on the Ubuntu 26.04 runner. Bind the name first; the tests pass on jq 1.7 and 1.8. Co-Authored-By: Claude Opus 5.5 --- scripts/dns-imports.jq | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq index 0fe756e..810c4b3 100644 --- a/scripts/dns-imports.jq +++ b/scripts/dns-imports.jq @@ -10,9 +10,9 @@ def fqdn: ascii_downcase | rtrimstr(".") | else .+".devantler.tech" end; def registry: .type=="TXT" and (.name|fqdn|startswith("_externaldns.")); def owned($records): - (.name|fqdn) as $name | (.type|ascii_downcase) as $type | any($records[]; - .type=="TXT" and (.name|fqdn) as $marker | - ($marker=="_externaldns."+$name or $marker=="_externaldns."+$type+"-"+$name)); + (.name|fqdn) as $name | (.type|ascii_downcase) as $type | + any($records[]; (.name|fqdn) as $marker | + .type=="TXT" and ($marker=="_externaldns."+$name or $marker=="_externaldns."+$type+"-"+$name)); $snapshot[0] as $live | [.resource_changes[] | select(.mode=="managed" and .type=="cloudflare_dns_record" and .change.actions==["create"]) | From afc1a56d7c57a8c2f54edb9c050a86d6985e0520 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 03:31:51 +0100 Subject: [PATCH 06/11] fix(dns): log only a count of unmanaged records Keep the names of live records that nothing owns in the encrypted evidence and print a count by type, so the public log never lists hostnames. Co-Authored-By: Claude Opus 5.5 --- docs/runbooks/bootstrap-and-recovery.md | 5 +++-- scripts/apply-dns.sh | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/runbooks/bootstrap-and-recovery.md b/docs/runbooks/bootstrap-and-recovery.md index 5ca1018..69371b6 100644 --- a/docs/runbooks/bootstrap-and-recovery.md +++ b/docs/runbooks/bootstrap-and-recovery.md @@ -367,8 +367,9 @@ The saved plan must then pass `scripts/assert-no-destructive-plan.sh` and their record exactly as it is. Creates, updates and `removed` blocks with `destroy = false` are admitted; deletes and replacements are not. The public log shows only `DNS_PLAN` lines, with addresses, actions and changed attribute -names; `DNS_DRIFT` lines; and `DNS_UNMANAGED` lines for live records that -nothing owns. Values, identifiers and tool output stay in the encrypted +names; `DNS_DRIFT` lines; and one `DNS_UNMANAGED` count, by type, of live +records that nothing owns. Their names, values, identifiers and tool output +stay in the encrypted evidence. In apply mode the run encrypts the state before and after the apply, even diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 2d939b6..70ac6ea 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -183,17 +183,19 @@ jq -r '.resource_changes[] | (if $action=="update" or ($action=="import" and .change.actions!=["no-op"]) then " attributes=\($attributes|join(","))" else "" end) ' "$scratch/evidence/change.json" jq -r '.resource_drift[]? | "DNS_DRIFT address=\(.address) action=\(.change.actions|join("-"))"' "$scratch/evidence/change.json" -# Live records that nothing owns: not declared here, not external-dns, not a challenge. -jq -r --slurpfile plan "$scratch/evidence/change.json" --slurpfile imports "$scratch/evidence/imports.json" ' +# Live records that nothing owns: not declared here, not external-dns, not a +# challenge. Their names stay in the encrypted evidence; the log gets counts. +jq --slurpfile plan "$scratch/evidence/change.json" --slurpfile imports "$scratch/evidence/imports.json" ' ([$plan[0].prior_state.values.root_module.resources[]? | select(.type=="cloudflare_dns_record") | .values.id] + [$imports[0][] | .id | split("/")[1]]) as $managed | .records as $records | - .records[] | . as $r | - select((.id|IN($managed[]))|not) | - select((.name|startswith("_externaldns.") or startswith("_acme-challenge."))|not) | - select(any($records[]; .type=="TXT" and (.name=="_externaldns."+$r.name or .name=="_externaldns."+($r.type|ascii_downcase)+"-"+$r.name))|not) | - "DNS_UNMANAGED name=\(.name) type=\(.type)" -' "$scratch/evidence/snapshot.json" + [.records[] | . as $r | + select((.id|IN($managed[]))|not) | + select((.name|startswith("_externaldns.") or startswith("_acme-challenge."))|not) | + select(any($records[]; .type=="TXT" and (.name=="_externaldns."+$r.name or .name=="_externaldns."+($r.type|ascii_downcase)+"-"+$r.name))|not) | + {name, type}] +' "$scratch/evidence/snapshot.json" > "$scratch/evidence/unmanaged.json" || stop +jq -r '"DNS_UNMANAGED count=\(length) types=\(group_by(.type)|map("\(.[0].type):\(length)")|join(","))"' "$scratch/evidence/unmanaged.json" stage=boundary bash "$root/scripts/assert-no-destructive-plan.sh" "$scratch/evidence/change.json" > "$scratch/evidence/destructive.log" 2>&1 || stop jq -e --slurpfile imports "$scratch/evidence/imports.json" -f "$root/scripts/check-dns-plan.jq" "$scratch/evidence/change.json" >/dev/null 2>&1 || stop From fc82a9b8581c77b97f28394311b96a8e6edca73b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 04:15:19 +0100 Subject: [PATCH 07/11] fix(dns): refuse double adoption and markers that claim managed records From a fresh local review: - Never import a live record that a resource already manages, and leave managed siblings out of the content-differs check. - Let an `_externaldns.` ownership record claim only a record that the same plan stops managing, read both typed and typeless, so no marker can hand a mail record such as the DKIM CNAME to external-dns. - Refuse a create or update of a record external-dns owns, or one without a known name. - Print addresses only for the records in dns.tf; anything keyed by the private configuration appears as a count. - Stop the apply with SIGTERM only, so a timeout cannot leave the R2 lock. Co-Authored-By: Claude Opus 5.5 --- docs/runbooks/bootstrap-and-recovery.md | 39 +++++++++++++--------- scripts/apply-dns.sh | 19 +++++++---- scripts/check-dns-plan.jq | 44 +++++++++++++++++++------ scripts/dns-imports.jq | 14 +++++--- scripts/tests/test-dns-apply.sh | 44 +++++++++++++++++++------ 5 files changed, 113 insertions(+), 47 deletions(-) diff --git a/docs/runbooks/bootstrap-and-recovery.md b/docs/runbooks/bootstrap-and-recovery.md index 69371b6..9956bda 100644 --- a/docs/runbooks/bootstrap-and-recovery.md +++ b/docs/runbooks/bootstrap-and-recovery.md @@ -357,34 +357,41 @@ session derived from the existing `CLOUDFLARE_API_TOKEN`; the configuration comes from `CLOUDFLARE_BOOTSTRAP_CONFIGURATION`. The live state must still be exactly the retained `recovery/backend-source/` generation, so an earlier apply that was not yet retained refuses every new run. A declared record that the -plan would create is imported when exactly one live record has the same name, -type and content. It stays a real create only when no live record has that -name and type. A changed content, more than one match, or a record that -external-dns owns refuses the run. +plan would create is imported when exactly one live record that nothing here +manages yet has the same name, type and content. It stays a real create only +when no such record has that name and type. A changed content, more than one +match, a record already managed here, or a record that external-dns owns +refuses the run. The saved plan must then pass `scripts/assert-no-destructive-plan.sh` and `scripts/check-dns-plan.jq`. Only DNS records may change. Imports must leave their record exactly as it is. Creates, updates and `removed` blocks with -`destroy = false` are admitted; deletes and replacements are not. The public -log shows only `DNS_PLAN` lines, with addresses, actions and changed attribute -names; `DNS_DRIFT` lines; and one `DNS_UNMANAGED` count, by type, of live -records that nothing owns. Their names, values, identifiers and tool output -stay in the encrypted -evidence. +`destroy = false` are admitted; deletes and replacements are not. A created or +updated record must have a known name and must not be owned by external-dns. +An `_externaldns.` ownership record may only claim a record that the same plan +stops managing, so it can never hand a mail record to external-dns. The public +log shows `DNS_PLAN` and `DNS_DRIFT` lines, with the `dns.tf` addresses, +actions and changed attribute names, plus counts for anything keyed by the +private configuration and one `DNS_UNMANAGED` count, by type, of live records +that nothing owns. Names, values, identifiers and tool output stay in the +encrypted evidence. In apply mode the run encrypts the state before and after the apply, even -when the apply fails, and then requires a clean plan. After a successful apply, a second job initializes -from R2 on a fresh runner and must also find nothing to change. Afterwards, +when the apply fails, and then requires a clean plan. The script never force-kills +the apply, because a killed apply would leave the R2 lock and unrecorded +changes behind. After a successful apply, a second job initializes from R2 on +a fresh runner and must also find nothing to change. Afterwards, download the `encrypted-dns-change` and `encrypted-dns-recovery` artifacts, authenticate the `after` and `recovered` packages with the independent identity (`check-state-recovery-package.sh --identity-file`), and retain the -`after` package in `recovery/backend-source/` through a signed, data-only pull -request. The configuration does not change, so the configuration secret stays +`after` package in `recovery/backend-source/` through a signed pull request: +a data-only one, or the next DNS change. The configuration does not change, so the configuration secret stays as it is. The next DNS change is possible only after that pull request merges. To hand a record to external-dns without a gap, create its ownership TXT -record in one change, then replace the record's resource with a `removed` -block that keeps the record (`lifecycle { destroy = false }`). To undo an +record and replace the record's resource with a `removed` block that keeps the +record (`lifecycle { destroy = false }`), in the same change. Forget the +ownership record in the next change, because external-dns then owns it. To undo an apply, revert its pull request and run the same two dispatches; a revert that would delete a record is refused, so restore deleted values by declaring them again. diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 70ac6ea..942679f 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -173,16 +173,21 @@ jq '.imports' "$scratch/evidence/resolved.json" > "$scratch/evidence/imports.jso jq -r '.[]|"import {\n to = \(.address)\n id = \(.id|tojson)\n}\n"' "$scratch/evidence/imports.json" > "$scratch/work/imports.tf" stage=plan plan_json change || stop -# Public summary: addresses, actions and changed attribute names, never values. -jq -r '.resource_changes[] | +# Public summary: actions and changed attribute names, never values. Only the +# addresses written in dns.tf are printed; resources keyed by the private +# configuration (tokens, buckets, configured records) appear only as a count. +jq -r 'def public: .address|startswith("cloudflare_dns_record.") and (startswith("cloudflare_dns_record.records[")|not); + (.resource_changes[] | select(public) | (if .change.importing then "import" else (.change.actions|join("-")) end) as $action | select($action!="no-op") | ([(.change.before // {}), (.change.after // {})] as [$b,$a] | ([$b,$a]|map(keys)|add|unique) | map(select($b[.]!=$a[.]))) as $attributes | "DNS_PLAN address=\(.address) action=\($action)" + - (if $action=="update" or ($action=="import" and .change.actions!=["no-op"]) then " attributes=\($attributes|join(","))" else "" end) + (if $action=="update" or ($action=="import" and .change.actions!=["no-op"]) then " attributes=\($attributes|join(","))" else "" end)), + "DNS_PLAN_PRIVATE changes=\([.resource_changes[] | select(public|not) | select(.change.actions!=["no-op"])]|length)", + (.resource_drift[]? | select(public) | "DNS_DRIFT address=\(.address) action=\(.change.actions|join("-"))"), + "DNS_DRIFT_PRIVATE changes=\([.resource_drift[]? | select(public|not)]|length)" ' "$scratch/evidence/change.json" -jq -r '.resource_drift[]? | "DNS_DRIFT address=\(.address) action=\(.change.actions|join("-"))"' "$scratch/evidence/change.json" # Live records that nothing owns: not declared here, not external-dns, not a # challenge. Their names stay in the encrypted evidence; the log gets counts. jq --slurpfile plan "$scratch/evidence/change.json" --slurpfile imports "$scratch/evidence/imports.json" ' @@ -198,13 +203,15 @@ jq --slurpfile plan "$scratch/evidence/change.json" --slurpfile imports "$scratc jq -r '"DNS_UNMANAGED count=\(length) types=\(group_by(.type)|map("\(.[0].type):\(length)")|join(","))"' "$scratch/evidence/unmanaged.json" stage=boundary bash "$root/scripts/assert-no-destructive-plan.sh" "$scratch/evidence/change.json" > "$scratch/evidence/destructive.log" 2>&1 || stop -jq -e --slurpfile imports "$scratch/evidence/imports.json" -f "$root/scripts/check-dns-plan.jq" "$scratch/evidence/change.json" >/dev/null 2>&1 || stop +jq -e --slurpfile imports "$scratch/evidence/imports.json" --slurpfile snapshot "$scratch/evidence/snapshot.json" -f "$root/scripts/check-dns-plan.jq" "$scratch/evidence/change.json" >/dev/null 2>&1 || stop if [[ $DNS_MODE == plan ]]; then unset root_token; exit 0; fi stage=before-capture capture before || stop stage=apply apply_rc=0 -tf provider apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? +# No hard kill mid-apply: a killed apply leaves the R2 lock and unrecorded +# changes. The job timeout is the outer bound. +native provider timeout 600 tofu -chdir="$scratch/work" apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? # Capture whatever the apply left behind before judging it. When OpenTofu # cannot save to R2 it writes errored.tfstate; keep that in the encrypted # evidence first, because the remote copy may then be older than the zone. diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq index 5a988d8..e01166d 100644 --- a/scripts/check-dns-plan.jq +++ b/scripts/check-dns-plan.jq @@ -1,19 +1,36 @@ -# Input: the saved plan JSON. $imports[0]: the resolved import list. Names are -# compared in their full, lower-case form, so a relative name cannot slip by. +# Input: the saved plan JSON. $imports[0]: the resolved import list. +# $snapshot[0]: the live zone. Names are compared in their full, lower-case +# form, so a relative or differently written name cannot slip by. # Only DNS records may change. Nothing is deleted or replaced. Every import is -# known and leaves its record exactly as it is. The typeless apex registry name -# is refused outright: external-dns would read it as owning the apex mail -# records and delete them. +# known and leaves its record exactly as it is. A created or updated record +# has a known name, and external-dns does not own it. An external-dns +# ownership record may only claim a record this plan stops managing: one that +# claims a record still managed here, such as a mail record, would let +# external-dns delete it. The typeless apex registry name is always refused, +# because it claims every apex record, the Email MX and TXT records included. def fqdn: ascii_downcase | rtrimstr(".") | if .=="@" then "devantler.tech" elif .=="devantler.tech" or endswith(".devantler.tech") then . else .+".devantler.tech" end; def changed: (.change.before != .change.after) or any(.change.after_unknown|..; .==true); -. as $plan | $imports[0] as $wanted | +def owned($records): + (.name|fqdn) as $name | (.type|ascii_downcase) as $type | + any($records[]; (.name|fqdn) as $marker | + .type=="TXT" and ($marker=="_externaldns."+$name or $marker=="_externaldns."+$type+"-"+$name)); +# Every (name, type) an ownership record could claim, read both typed and typeless. +def claims: + (.name|fqdn|ltrimstr("_externaldns.")) as $rest | + [{name:$rest, type:null}] + + [$rest|capture("^(?[a-z0-9]+)-(?.+)$")|{name, type}]; +def writes: .change.actions==["create"] or .change.actions==["update"]; +. as $plan | $imports[0] as $wanted | $snapshot[0].records as $live | +[.resource_changes[] | select(.type=="cloudflare_dns_record" and .change.actions!=["forget"]) | + .change.after // .change.before | select(type=="object") | + {name:(.name // "" | fqdn), type:(.type // "" | ascii_downcase)}] as $kept | .errored!=true and ((.deferred_changes // [])|length)==0 and (.resource_changes|type=="array") and -($wanted|type=="array") and +($wanted|type=="array") and ($live|type=="array") and (($wanted|map(.address)|length)==($wanted|map(.address)|unique|length)) and all($wanted[]; . as $import | @@ -26,6 +43,13 @@ all(.resource_changes[]; (if .change.importing then any($wanted[]; .address==$resource.address) and .change.actions==["no-op"] and (changed|not) else true end) and - (if .type=="cloudflare_dns_record" then - (((.change.after // {}).name // "")|fqdn) != "_externaldns.devantler.tech" - else .change.actions==["no-op"] end)) + (if .type!="cloudflare_dns_record" then .change.actions==["no-op"] + elif writes then + .change.after as $after | + ($after.name|type=="string") and ($after.type|type=="string") and + ($after.name|fqdn) != "_externaldns.devantler.tech" and + (if ($after.type=="TXT" and ($after.name|fqdn|startswith("_externaldns."))) then + all($after|claims[]; . as $claim | + all($kept[]; .name!=$claim.name or ($claim.type!=null and .type!=$claim.type))) + else ($after|owned($live)|not) end) + else true end)) diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq index 810c4b3..19cb164 100644 --- a/scripts/dns-imports.jq +++ b/scripts/dns-imports.jq @@ -1,9 +1,10 @@ # Input: a plan JSON made before imports. $snapshot[0]: the complete live zone. # A declared record the plan would create is imported when exactly one live -# record has the same name, type and content. It stays a create only when no -# live record has that name and type and no external-dns ownership record -# claims it. Anything else is refused, so a typo or drift can never add a -# second mail record or take over an external-dns record. +# record that no resource manages yet has the same name, type and content. It +# stays a create only when no unmanaged live record has that name and type and +# no external-dns ownership record claims it. Anything else is refused, so a +# typo or drift can never add a second mail record, adopt a record twice, or +# take over an external-dns record. def fqdn: ascii_downcase | rtrimstr(".") | if .=="@" then "devantler.tech" elif .=="devantler.tech" or endswith(".devantler.tech") then . @@ -14,12 +15,15 @@ def owned($records): any($records[]; (.name|fqdn) as $marker | .type=="TXT" and ($marker=="_externaldns."+$name or $marker=="_externaldns."+$type+"-"+$name)); $snapshot[0] as $live | +[.prior_state.values.root_module.resources[]? | select(.type=="cloudflare_dns_record") | .values.id] as $managed | [.resource_changes[] | select(.mode=="managed" and .type=="cloudflare_dns_record" and .change.actions==["create"]) | .change.after as $after | - [$live.records[] | select((.name|fqdn)==($after.name|fqdn) and .type==$after.type)] as $same | + [$live.records[] | select((.name|fqdn)==($after.name|fqdn) and .type==$after.type)] as $named | + [$named[] | select(.id|IN($managed[])|not)] as $same | [$same[] | select(.content==$after.content)] as $exact | if ($after|registry|not) and ($after|owned($live.records)) then {address, refused:"external-dns-owned"} + elif any($named[]; (.id|IN($managed[])) and .content==$after.content) then {address, refused:"already-managed"} elif ($exact|length)==1 then if ($exact[0]|registry) or ($exact[0]|owned($live.records)) then {address, refused:"external-dns-owned"} else {address, id:($live.zone.id+"/"+$exact[0].id)} end diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh index ec26a13..c64adc8 100755 --- a/scripts/tests/test-dns-apply.sh +++ b/scripts/tests/test-dns-apply.sh @@ -65,23 +65,37 @@ done result=$( { create cloudflare_dns_record.one devantler.tech MX mx01.mail.icloud.com create cloudflare_dns_record.two devantler.tech MX mx01.mail.icloud.com; } | resolve) jq -e '.refused==[{address:"*",refused:"duplicate-target"}]' <<<"$result" >/dev/null +# A record already managed here is never adopted a second time, and managed +# siblings do not block a genuinely new record at the same name and type. +managed() { jq -s --argjson ids "$1" '{prior_state:{values:{root_module:{resources:[$ids[]|{type:"cloudflare_dns_record",values:{id:.}}]}}},resource_changes:.}'; } +result=$(create cloudflare_dns_record.mail_mx_03 devantler.tech MX mx01.mail.icloud.com | + managed '["00000000000000000000000000000001"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") +jq -e '.imports==[] and .refused==[{address:"cloudflare_dns_record.mail_mx_03",refused:"already-managed"}]' <<<"$result" >/dev/null +result=$(create cloudflare_dns_record.new_txt devantler.tech TXT '"new-proof"' | + managed '["00000000000000000000000000000002","00000000000000000000000000000003"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") +jq -e '.imports==[] and .refused==[]' <<<"$result" >/dev/null printf '%s\n' 'PASS: declared records import only from one exact unowned live record' -# 3. Plan boundary: imports must be unchanged, and nothing is deleted or replaced. +# 3. Plan boundary: imports must be unchanged, nothing is deleted or replaced, +# and an ownership record may only claim a record this plan stops managing. import_id="$zone/00000000000000000000000000000001" printf '[{"address":"cloudflare_dns_record.mail_mx_01","id":"%s"}]\n' "$import_id" > "$scratch/imports.json" record='{"name":"devantler.tech","type":"MX","content":"mx01.mail.icloud.com","ttl":3600}' +dkim='{"name":"sig1._domainkey.devantler.tech","type":"CNAME","content":"dkim.example.test"}' +www='{"name":"www.devantler.tech","type":"CNAME","content":"pages.example.test"}' change() { jq -n --arg address "$1" --arg type "$2" --argjson actions "$3" --argjson before "$4" --argjson after "$5" --argjson importing "$6" \ '{address:$address,mode:"managed",type:$type,change:({actions:$actions,before:$before,after:$after,after_unknown:{}} + (if $importing then {importing:$importing} else {} end))}'; } plan() { jq -s '{resource_changes:.}'; } -check() { jq -e --slurpfile imports "$scratch/imports.json" -f "$root/scripts/check-dns-plan.jq" >/dev/null; } +check() { jq -e --slurpfile imports "$scratch/imports.json" --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/check-dns-plan.jq" >/dev/null; } noop_import=$(change cloudflare_dns_record.mail_mx_01 cloudflare_dns_record '["no-op"]' "$record" "$record" "{\"id\":\"$import_id\"}") zone_noop=$(change cloudflare_zone.primary cloudflare_zone '["no-op"]' '{}' '{}' null) -marker='{"name":"_externaldns.cname-www.devantler.tech","type":"TXT","content":"x"}' -{ printf '%s\n' "$noop_import" "$zone_noop" - change cloudflare_dns_record.marker cloudflare_dns_record '["create"]' null "$marker" null - change cloudflare_dns_record.apex cloudflare_dns_record '["update"]' '{"content":"a"}' '{"content":"b"}' null - change cloudflare_dns_record.www cloudflare_dns_record '["forget"]' '{"content":"a"}' null null +dkim_kept=$(change cloudflare_dns_record.mail_dkim_sig1 cloudflare_dns_record '["no-op"]' "$dkim" "$dkim" null) +marker() { change cloudflare_dns_record.marker cloudflare_dns_record '["create"]' null "{\"name\":\"$1\",\"type\":\"TXT\",\"content\":\"x\"}" null; } +# The www handover: the marker claims www, which the same plan forgets. +{ printf '%s\n' "$noop_import" "$zone_noop" "$dkim_kept" + marker _externaldns.cname-www.devantler.tech + change cloudflare_dns_record.apex cloudflare_dns_record '["update"]' '{"name":"devantler.tech","type":"CNAME","content":"a"}' '{"name":"devantler.tech","type":"CNAME","content":"b"}' null + change cloudflare_dns_record.www cloudflare_dns_record '["forget"]' "$www" null null } | plan | check reject() { if plan | check; then printf 'FAIL: %s was admitted\n' "$1" >&2; exit 1; fi; } change cloudflare_dns_record.mail_mx_01 cloudflare_dns_record '["no-op"]' "$record" '{"name":"devantler.tech","type":"MX","content":"mx01.mail.icloud.com","ttl":300}' "{\"id\":\"$import_id\"}" | reject changing-import @@ -91,10 +105,20 @@ printf '%s\n' "$zone_noop" | reject missing-import { printf '%s\n' "$noop_import"; change cloudflare_dns_record.mail_spf cloudflare_dns_record '["delete"]' "$record" null null; } | reject delete { printf '%s\n' "$noop_import"; change cloudflare_dns_record.mail_spf cloudflare_dns_record '["delete","create"]' "$record" "$record" null; } | reject replace { printf '%s\n' "$noop_import"; change cloudflare_zone.primary cloudflare_zone '["update"]' '{"a":1}' '{"a":2}' null; } | reject zone-update -{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.bad cloudflare_dns_record '["create"]' null '{"name":"_externaldns.devantler.tech","type":"TXT","content":"x"}' null; } | reject typeless-apex-registry -for name in _externaldns _EXTERNALDNS.devantler.tech. _externaldns.Devantler.Tech; do - { printf '%s\n' "$noop_import"; change cloudflare_dns_record.bad cloudflare_dns_record '["create"]' null "{\"name\":\"$name\",\"type\":\"TXT\",\"content\":\"x\"}" null; } | reject "typeless-apex-registry $name" +for name in _externaldns.devantler.tech _externaldns _EXTERNALDNS.devantler.tech. _externaldns.Devantler.Tech; do + { printf '%s\n' "$noop_import"; marker "$name"; } | reject "typeless-apex-registry $name" done +# Markers that would hand a record still managed here, such as mail, to external-dns. +for name in _externaldns.cname-sig1._domainkey.devantler.tech _externaldns.CNAME-sig1._domainkey.devantler.tech. _externaldns.sig1._domainkey.devantler.tech; do + { printf '%s\n' "$noop_import" "$dkim_kept"; marker "$name"; } | reject "mail-claiming marker $name" +done +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.www cloudflare_dns_record '["no-op"]' "$www" "$www" null + marker _externaldns.cname-www.devantler.tech; } | reject "marker for a record still managed here" +# A record external-dns owns is never written here, and every write has a known +# name (OpenTofu writes an unknown value as null). +portal='{"name":"portal.devantler.tech","type":"A","content":"192.0.2.2"}' +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.portal cloudflare_dns_record '["update"]' '{"name":"portal.devantler.tech","type":"A","content":"192.0.2.1"}' "$portal" null; } | reject owned-update +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.nameless cloudflare_dns_record '["create"]' null '{"type":"TXT","content":"x"}' null; } | reject missing-name printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if check; then exit 1; fi printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi printf '%s\n' 'PASS: DNS plans admit unchanged imports, creates, updates and forgets only' From c88888782c1e7eeba24bfe0e64946c78707fe577 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 04:32:25 +0100 Subject: [PATCH 08/11] fix(dns): check ownership before updates, require upper-case types, hide private refusals - Refuse an update that moves a record external-dns owns, not only one that lands on an owned name. - Require an upper-case record type for every write and compare types case-insensitively when resolving imports, so a `txt` marker cannot skip the claims check. - Report a refusal for a record keyed by the private configuration without its address. Co-Authored-By: Claude Opus 5.5 --- scripts/apply-dns.sh | 3 ++- scripts/check-dns-plan.jq | 5 ++++- scripts/dns-imports.jq | 4 ++-- scripts/tests/test-dns-apply.sh | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 942679f..0e15bd1 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -167,7 +167,8 @@ stage=discover plan_json discover || stop jq --slurpfile snapshot "$scratch/evidence/snapshot.json" -f "$root/scripts/dns-imports.jq" \ "$scratch/evidence/discover.json" > "$scratch/evidence/resolved.json" 2>/dev/null || stop -jq -r '.refused[]|"DNS_REFUSED address=\(.address) reason=\(.refused)"' "$scratch/evidence/resolved.json" +# Refusals keyed by the private configuration are reported without their address. +jq -r '.refused[] | if (.address|startswith("cloudflare_dns_record.records[")) then "DNS_REFUSED address=private reason=\(.refused)" else "DNS_REFUSED address=\(.address) reason=\(.refused)" end' "$scratch/evidence/resolved.json" jq -e '.refused==[]' "$scratch/evidence/resolved.json" >/dev/null 2>&1 || stop jq '.imports' "$scratch/evidence/resolved.json" > "$scratch/evidence/imports.json" jq -r '.[]|"import {\n to = \(.address)\n id = \(.id|tojson)\n}\n"' "$scratch/evidence/imports.json" > "$scratch/work/imports.tf" diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq index e01166d..338ea2f 100644 --- a/scripts/check-dns-plan.jq +++ b/scripts/check-dns-plan.jq @@ -3,7 +3,8 @@ # form, so a relative or differently written name cannot slip by. # Only DNS records may change. Nothing is deleted or replaced. Every import is # known and leaves its record exactly as it is. A created or updated record -# has a known name, and external-dns does not own it. An external-dns +# has a known name and an upper-case type, and external-dns owns it neither +# before nor after the change. An external-dns # ownership record may only claim a record this plan stops managing: one that # claims a record still managed here, such as a mail record, would let # external-dns delete it. The typeless apex registry name is always refused, @@ -47,7 +48,9 @@ all(.resource_changes[]; elif writes then .change.after as $after | ($after.name|type=="string") and ($after.type|type=="string") and + $after.type==($after.type|ascii_upcase) and ($after.name|fqdn) != "_externaldns.devantler.tech" and + (if .change.actions==["update"] then (.change.before|owned($live)|not) else true end) and (if ($after.type=="TXT" and ($after.name|fqdn|startswith("_externaldns."))) then all($after|claims[]; . as $claim | all($kept[]; .name!=$claim.name or ($claim.type!=null and .type!=$claim.type))) diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq index 19cb164..6cc8261 100644 --- a/scripts/dns-imports.jq +++ b/scripts/dns-imports.jq @@ -9,7 +9,7 @@ def fqdn: ascii_downcase | rtrimstr(".") | if .=="@" then "devantler.tech" elif .=="devantler.tech" or endswith(".devantler.tech") then . else .+".devantler.tech" end; -def registry: .type=="TXT" and (.name|fqdn|startswith("_externaldns.")); +def registry: (.type|ascii_upcase)=="TXT" and (.name|fqdn|startswith("_externaldns.")); def owned($records): (.name|fqdn) as $name | (.type|ascii_downcase) as $type | any($records[]; (.name|fqdn) as $marker | @@ -19,7 +19,7 @@ $snapshot[0] as $live | [.resource_changes[] | select(.mode=="managed" and .type=="cloudflare_dns_record" and .change.actions==["create"]) | .change.after as $after | - [$live.records[] | select((.name|fqdn)==($after.name|fqdn) and .type==$after.type)] as $named | + [$live.records[] | select((.name|fqdn)==($after.name|fqdn) and (.type|ascii_upcase)==($after.type|ascii_upcase))] as $named | [$named[] | select(.id|IN($managed[])|not)] as $same | [$same[] | select(.content==$after.content)] as $exact | if ($after|registry|not) and ($after|owned($live.records)) then {address, refused:"external-dns-owned"} diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh index c64adc8..4973af6 100755 --- a/scripts/tests/test-dns-apply.sh +++ b/scripts/tests/test-dns-apply.sh @@ -74,6 +74,8 @@ jq -e '.imports==[] and .refused==[{address:"cloudflare_dns_record.mail_mx_03",r result=$(create cloudflare_dns_record.new_txt devantler.tech TXT '"new-proof"' | managed '["00000000000000000000000000000002","00000000000000000000000000000003"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") jq -e '.imports==[] and .refused==[]' <<<"$result" >/dev/null +result=$(create cloudflare_dns_record.lower devantler.tech txt '"apple-domain=x"' | resolve) +jq -e --arg zone "$zone" '.refused==[] and .imports==[{address:"cloudflare_dns_record.lower",id:($zone+"/00000000000000000000000000000003")}]' <<<"$result" >/dev/null printf '%s\n' 'PASS: declared records import only from one exact unowned live record' # 3. Plan boundary: imports must be unchanged, nothing is deleted or replaced, @@ -118,6 +120,8 @@ done # name (OpenTofu writes an unknown value as null). portal='{"name":"portal.devantler.tech","type":"A","content":"192.0.2.2"}' { printf '%s\n' "$noop_import"; change cloudflare_dns_record.portal cloudflare_dns_record '["update"]' '{"name":"portal.devantler.tech","type":"A","content":"192.0.2.1"}' "$portal" null; } | reject owned-update +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.portal cloudflare_dns_record '["update"]' "$portal" '{"name":"archive.devantler.tech","type":"A","content":"192.0.2.2"}' null; } | reject rename-away-from-owned +{ printf '%s\n' "$noop_import" "$dkim_kept"; change cloudflare_dns_record.marker cloudflare_dns_record '["create"]' null '{"name":"_externaldns.cname-sig1._domainkey.devantler.tech","type":"txt","content":"x"}' null; } | reject lower-case-marker-type { printf '%s\n' "$noop_import"; change cloudflare_dns_record.nameless cloudflare_dns_record '["create"]' null '{"type":"TXT","content":"x"}' null; } | reject missing-name printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if check; then exit 1; fi printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi From 5c8fffa0cbf6a07b85d2edf9f315000d69141599 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 04:46:26 +0100 Subject: [PATCH 09/11] fix(dns): markers may only claim live records this plan forgets From a second local review round: - An `_externaldns.` ownership record must claim at least one live record, and every live record it claims must be one the same plan forgets. It can no longer hand an unmanaged record, such as another DKIM selector, to external-dns. - Record names on writes use hostname characters only. - Import resolution stops if the plan's prior state lacks the zone, instead of treating the managed set as empty. - The apply gets a fresh 15-minute state session and a 5-minute SIGTERM bound, so its state save and unlock never run on expired credentials. - The runbook describes what to do after a failed or partial apply. Co-Authored-By: Claude Opus 5.5 --- docs/runbooks/bootstrap-and-recovery.md | 12 +++++++++-- scripts/apply-dns.sh | 16 +++++++++++---- scripts/check-dns-plan.jq | 17 +++++++++++----- scripts/dns-imports.jq | 5 ++++- scripts/tests/test-dns-apply.sh | 27 +++++++++++++++++++------ 5 files changed, 59 insertions(+), 18 deletions(-) diff --git a/docs/runbooks/bootstrap-and-recovery.md b/docs/runbooks/bootstrap-and-recovery.md index 9956bda..d3b3639 100644 --- a/docs/runbooks/bootstrap-and-recovery.md +++ b/docs/runbooks/bootstrap-and-recovery.md @@ -368,8 +368,9 @@ The saved plan must then pass `scripts/assert-no-destructive-plan.sh` and their record exactly as it is. Creates, updates and `removed` blocks with `destroy = false` are admitted; deletes and replacements are not. A created or updated record must have a known name and must not be owned by external-dns. -An `_externaldns.` ownership record may only claim a record that the same plan -stops managing, so it can never hand a mail record to external-dns. The public +An `_externaldns.` ownership record must claim at least one live record, and +only records the same plan stops managing, so it can never hand a mail record, +or any record nothing here manages, to external-dns. The public log shows `DNS_PLAN` and `DNS_DRIFT` lines, with the `dns.tf` addresses, actions and changed attribute names, plus counts for anything keyed by the private configuration and one `DNS_UNMANAGED` count, by type, of live records @@ -388,6 +389,13 @@ identity (`check-state-recovery-package.sh --identity-file`), and retain the a data-only one, or the next DNS change. The configuration does not change, so the configuration secret stays as it is. The next DNS change is possible only after that pull request merges. +If an apply fails or stops part-way, do not dispatch again. The state has +moved on, so every new run is refused until it is retained. Authenticate the +`after` package from `encrypted-dns-change`, and the `errored.tfstate` in its +evidence if OpenTofu could not save to R2. Compare the state with the live zone +and retain the package that matches what Cloudflare actually holds. Then +dispatch a plan to see what is still pending. + To hand a record to external-dns without a gap, create its ownership TXT record and replace the record's resource with a `removed` block that keeps the record (`lifecycle { destroy = false }`), in the same change. Forget the diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 0e15bd1..1c4ef08 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -75,9 +75,14 @@ request() { request GET "https://api.cloudflare.com/client/v4/accounts/$account/tokens/verify" "$scratch/parent.json" || stop parent=$(jq -er 'if .result.status=="active" and (.result.id|type=="string" and test("^[a-f0-9]{32}$")) then .result.id else error("invalid parent") end' "$scratch/parent.json" 2>/dev/null) || stop jq -n --arg parent "$parent" '{parentAccessKeyId:$parent,bucket:"cloudflare-bootstrap-state",permission:"object-read-write",ttlSeconds:900,prefixes:["cloudflare/"]}' > "$scratch/request.json" +# A 15-minute state session. The apply gets a fresh one, so its state save and +# unlock never run on expiring credentials. +new_session() { + request POST "https://api.cloudflare.com/client/v4/accounts/$account/r2/temp-access-credentials" "$scratch/session.json" "$scratch/request.json" && + backend_session "$scratch/session.json" +} stage=session -request POST "https://api.cloudflare.com/client/v4/accounts/$account/r2/temp-access-credentials" "$scratch/session.json" "$scratch/request.json" || stop -backend_session "$scratch/session.json" || stop +new_session || stop unset parent # OpenTofu gets only the short-lived state session, plus the account token for # provider steps. Nothing else from the runner environment reaches it. @@ -208,11 +213,14 @@ jq -e --slurpfile imports "$scratch/evidence/imports.json" --slurpfile snapshot if [[ $DNS_MODE == plan ]]; then unset root_token; exit 0; fi stage=before-capture capture before || stop +stage=session-refresh +new_session || stop stage=apply apply_rc=0 # No hard kill mid-apply: a killed apply leaves the R2 lock and unrecorded -# changes. The job timeout is the outer bound. -native provider timeout 600 tofu -chdir="$scratch/work" apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? +# changes. SIGTERM after 5 minutes lets OpenTofu save state and unlock within +# the fresh session. +native provider timeout 300 tofu -chdir="$scratch/work" apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? # Capture whatever the apply left behind before judging it. When OpenTofu # cannot save to R2 it writes errored.tfstate; keep that in the encrypted # evidence first, because the remote copy may then be older than the zone. diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq index 338ea2f..f90ac74 100644 --- a/scripts/check-dns-plan.jq +++ b/scripts/check-dns-plan.jq @@ -4,10 +4,11 @@ # Only DNS records may change. Nothing is deleted or replaced. Every import is # known and leaves its record exactly as it is. A created or updated record # has a known name and an upper-case type, and external-dns owns it neither -# before nor after the change. An external-dns -# ownership record may only claim a record this plan stops managing: one that -# claims a record still managed here, such as a mail record, would let -# external-dns delete it. The typeless apex registry name is always refused, +# before nor after the change. Its name uses only hostname characters. An +# external-dns ownership record must claim at least one live record, and every +# live record it claims must be one this plan stops managing. Otherwise it +# could hand a mail record, or a record nothing here manages, to external-dns, +# which would delete it. The typeless apex registry name is always refused, # because it claims every apex record, the Email MX and TXT records included. def fqdn: ascii_downcase | rtrimstr(".") | if .=="@" then "devantler.tech" @@ -23,11 +24,13 @@ def claims: (.name|fqdn|ltrimstr("_externaldns.")) as $rest | [{name:$rest, type:null}] + [$rest|capture("^(?[a-z0-9]+)-(?.+)$")|{name, type}]; +def hits($claim): (.name|fqdn)==$claim.name and ($claim.type==null or (.type|ascii_downcase)==$claim.type); def writes: .change.actions==["create"] or .change.actions==["update"]; . as $plan | $imports[0] as $wanted | $snapshot[0].records as $live | [.resource_changes[] | select(.type=="cloudflare_dns_record" and .change.actions!=["forget"]) | .change.after // .change.before | select(type=="object") | {name:(.name // "" | fqdn), type:(.type // "" | ascii_downcase)}] as $kept | +[.resource_changes[] | select(.type=="cloudflare_dns_record" and .change.actions==["forget"]) | .change.before.id // empty] as $forgotten | .errored!=true and ((.deferred_changes // [])|length)==0 and (.resource_changes|type=="array") and @@ -48,11 +51,15 @@ all(.resource_changes[]; elif writes then .change.after as $after | ($after.name|type=="string") and ($after.type|type=="string") and + ($after.name|test("^[A-Za-z0-9_.*@-]+$")) and $after.type==($after.type|ascii_upcase) and ($after.name|fqdn) != "_externaldns.devantler.tech" and (if .change.actions==["update"] then (.change.before|owned($live)|not) else true end) and (if ($after.type=="TXT" and ($after.name|fqdn|startswith("_externaldns."))) then - all($after|claims[]; . as $claim | + ($after|claims) as $claims | + [$live[] | . as $r | select(any($claims[]; . as $c | $r|hits($c)))] as $claimed | + ($claimed|length)>0 and all($claimed[]; .id|IN($forgotten[])) and + all($claims[]; . as $claim | all($kept[]; .name!=$claim.name or ($claim.type!=null and .type!=$claim.type))) else ($after|owned($live)|not) end) else true end)) diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq index 6cc8261..8f54a1a 100644 --- a/scripts/dns-imports.jq +++ b/scripts/dns-imports.jq @@ -14,8 +14,11 @@ def owned($records): (.name|fqdn) as $name | (.type|ascii_downcase) as $type | any($records[]; (.name|fqdn) as $marker | .type=="TXT" and ($marker=="_externaldns."+$name or $marker=="_externaldns."+$type+"-"+$name)); +# The zone is always in state; without it the managed set below would be empty. +if ([.prior_state.values.root_module.resources[]? | select(.type=="cloudflare_zone")]|length)!=1 +then error("dns-imports: prior state without the zone") else . end | $snapshot[0] as $live | -[.prior_state.values.root_module.resources[]? | select(.type=="cloudflare_dns_record") | .values.id] as $managed | +[.prior_state.values.root_module.resources[] | select(.type=="cloudflare_dns_record") | .values.id] as $managed | [.resource_changes[] | select(.mode=="managed" and .type=="cloudflare_dns_record" and .change.actions==["create"]) | .change.after as $after | diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh index 4973af6..f807f52 100755 --- a/scripts/tests/test-dns-apply.sh +++ b/scripts/tests/test-dns-apply.sh @@ -44,11 +44,21 @@ jq -n --arg zone "$zone" '{zone:{id:$zone}, records:[ {id:"00000000000000000000000000000005",name:"dup.devantler.tech",type:"TXT",content:"\"same\""}, {id:"00000000000000000000000000000006",name:"portal.devantler.tech",type:"A",content:"192.0.2.1"}, {id:"00000000000000000000000000000007",name:"_externaldns.a-portal.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""}, - {id:"00000000000000000000000000000008",name:"_externaldns.a-ghost.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""} + {id:"00000000000000000000000000000008",name:"_externaldns.a-ghost.devantler.tech",type:"TXT",content:"\"heritage=external-dns\""}, + {id:"00000000000000000000000000000009",name:"www.devantler.tech",type:"CNAME",content:"pages.example.test"}, + {id:"0000000000000000000000000000000a",name:"sig1._domainkey.devantler.tech",type:"CNAME",content:"dkim.example.test"}, + {id:"0000000000000000000000000000000b",name:"sig2._domainkey.devantler.tech",type:"CNAME",content:"dkim2.example.test"} ]}' > "$scratch/snapshot.json" create() { jq -n --arg address "$1" --arg name "$2" --arg type "$3" --arg content "$4" \ '{address:$address,mode:"managed",type:"cloudflare_dns_record",change:{actions:["create"],after:{name:$name,type:$type,content:$content}}}'; } -resolve() { jq -s '{resource_changes:.}' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq"; } +# A plan from the real backend always has the zone in its prior state. +state() { jq -s --argjson ids "$1" '{prior_state:{values:{root_module:{resources:([{type:"cloudflare_zone",values:{id:"z"}}] + + [$ids[]|{type:"cloudflare_dns_record",values:{id:.}}])}}},resource_changes:.}'; } +resolve() { state '[]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq"; } +# Without the zone in prior state the managed set is unproven, so resolution stops. +if create cloudflare_dns_record.mail_mx_01 devantler.tech MX mx01.mail.icloud.com | + jq -s '{prior_state:{values:{root_module:{resources:[{type:"cloudflare_dns_record",values:{id:"00000000000000000000000000000002"}}]}}},resource_changes:.}' | + jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq" >/dev/null 2>&1; then exit 1; fi result=$( { create cloudflare_dns_record.mail_mx_01 devantler.tech MX mx01.mail.icloud.com create cloudflare_dns_record.mail_spf devantler.tech TXT '"v=spf1 include:icloud.com ~all"' create cloudflare_dns_record.marker _externaldns.cname-www.devantler.tech TXT '"heritage=external-dns"'; } | resolve) @@ -67,12 +77,11 @@ result=$( { create cloudflare_dns_record.one devantler.tech MX mx01.mail.icloud. jq -e '.refused==[{address:"*",refused:"duplicate-target"}]' <<<"$result" >/dev/null # A record already managed here is never adopted a second time, and managed # siblings do not block a genuinely new record at the same name and type. -managed() { jq -s --argjson ids "$1" '{prior_state:{values:{root_module:{resources:[$ids[]|{type:"cloudflare_dns_record",values:{id:.}}]}}},resource_changes:.}'; } result=$(create cloudflare_dns_record.mail_mx_03 devantler.tech MX mx01.mail.icloud.com | - managed '["00000000000000000000000000000001"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") + state '["00000000000000000000000000000001"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") jq -e '.imports==[] and .refused==[{address:"cloudflare_dns_record.mail_mx_03",refused:"already-managed"}]' <<<"$result" >/dev/null result=$(create cloudflare_dns_record.new_txt devantler.tech TXT '"new-proof"' | - managed '["00000000000000000000000000000002","00000000000000000000000000000003"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") + state '["00000000000000000000000000000002","00000000000000000000000000000003"]' | jq --slurpfile snapshot "$scratch/snapshot.json" -f "$root/scripts/dns-imports.jq") jq -e '.imports==[] and .refused==[]' <<<"$result" >/dev/null result=$(create cloudflare_dns_record.lower devantler.tech txt '"apple-domain=x"' | resolve) jq -e --arg zone "$zone" '.refused==[] and .imports==[{address:"cloudflare_dns_record.lower",id:($zone+"/00000000000000000000000000000003")}]' <<<"$result" >/dev/null @@ -84,7 +93,7 @@ import_id="$zone/00000000000000000000000000000001" printf '[{"address":"cloudflare_dns_record.mail_mx_01","id":"%s"}]\n' "$import_id" > "$scratch/imports.json" record='{"name":"devantler.tech","type":"MX","content":"mx01.mail.icloud.com","ttl":3600}' dkim='{"name":"sig1._domainkey.devantler.tech","type":"CNAME","content":"dkim.example.test"}' -www='{"name":"www.devantler.tech","type":"CNAME","content":"pages.example.test"}' +www='{"id":"00000000000000000000000000000009","name":"www.devantler.tech","type":"CNAME","content":"pages.example.test"}' change() { jq -n --arg address "$1" --arg type "$2" --argjson actions "$3" --argjson before "$4" --argjson after "$5" --argjson importing "$6" \ '{address:$address,mode:"managed",type:$type,change:({actions:$actions,before:$before,after:$after,after_unknown:{}} + (if $importing then {importing:$importing} else {} end))}'; } plan() { jq -s '{resource_changes:.}'; } @@ -116,6 +125,11 @@ for name in _externaldns.cname-sig1._domainkey.devantler.tech _externaldns.CNAME done { printf '%s\n' "$noop_import"; change cloudflare_dns_record.www cloudflare_dns_record '["no-op"]' "$www" "$www" null marker _externaldns.cname-www.devantler.tech; } | reject "marker for a record still managed here" +# A marker must claim a live record, and only records this plan forgets. +{ printf '%s\n' "$noop_import"; marker _externaldns.cname-nothing.devantler.tech; } | reject "marker that claims nothing" +for name in _externaldns.cname-sig2._domainkey.devantler.tech _externaldns.sig2._domainkey.devantler.tech; do + { printf '%s\n' "$noop_import"; marker "$name"; } | reject "marker for an unmanaged live record $name" +done # A record external-dns owns is never written here, and every write has a known # name (OpenTofu writes an unknown value as null). portal='{"name":"portal.devantler.tech","type":"A","content":"192.0.2.2"}' @@ -123,6 +137,7 @@ portal='{"name":"portal.devantler.tech","type":"A","content":"192.0.2.2"}' { printf '%s\n' "$noop_import"; change cloudflare_dns_record.portal cloudflare_dns_record '["update"]' "$portal" '{"name":"archive.devantler.tech","type":"A","content":"192.0.2.2"}' null; } | reject rename-away-from-owned { printf '%s\n' "$noop_import" "$dkim_kept"; change cloudflare_dns_record.marker cloudflare_dns_record '["create"]' null '{"name":"_externaldns.cname-sig1._domainkey.devantler.tech","type":"txt","content":"x"}' null; } | reject lower-case-marker-type { printf '%s\n' "$noop_import"; change cloudflare_dns_record.nameless cloudflare_dns_record '["create"]' null '{"type":"TXT","content":"x"}' null; } | reject missing-name +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.spaced cloudflare_dns_record '["create"]' null '{"name":" _externaldns.devantler.tech","type":"TXT","content":"x"}' null; } | reject non-hostname-name printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if check; then exit 1; fi printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi printf '%s\n' 'PASS: DNS plans admit unchanged imports, creates, updates and forgets only' From cb717670284d82bf938bc579f0c46cb4e71392ae Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 04:55:27 +0100 Subject: [PATCH 10/11] fix(dns): anchor record names and interrupt only OpenTofu on a slow apply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the third local review round: - Match write names against the whole string (\A…\z), so a trailing newline cannot slip past the hostname pattern or the ownership check. - Stop a slow apply with SIGINT sent to OpenTofu alone (--foreground), so the provider plugin finishes its in-flight call and the result is saved. - Test that a marker is refused when the forget lacks or mismatches the claimed record's id. Co-Authored-By: Claude Opus 5.5 --- scripts/apply-dns.sh | 7 ++++--- scripts/check-dns-plan.jq | 2 +- scripts/tests/test-dns-apply.sh | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index 1c4ef08..37a0139 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -218,9 +218,10 @@ new_session || stop stage=apply apply_rc=0 # No hard kill mid-apply: a killed apply leaves the R2 lock and unrecorded -# changes. SIGTERM after 5 minutes lets OpenTofu save state and unlock within -# the fresh session. -native provider timeout 300 tofu -chdir="$scratch/work" apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? +# changes. After 5 minutes OpenTofu alone gets SIGINT, its graceful stop: it +# waits for in-flight provider calls, saves state and unlocks within the fresh +# session. --foreground keeps the signal away from the provider plugin. +native provider timeout --foreground -s INT 300 tofu -chdir="$scratch/work" apply -input=false -lock-timeout=0s -no-color change.tfplan > "$scratch/evidence/apply.log" 2>&1 || apply_rc=$? # Capture whatever the apply left behind before judging it. When OpenTofu # cannot save to R2 it writes errored.tfstate; keep that in the encrypted # evidence first, because the remote copy may then be older than the zone. diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq index f90ac74..833b8d9 100644 --- a/scripts/check-dns-plan.jq +++ b/scripts/check-dns-plan.jq @@ -51,7 +51,7 @@ all(.resource_changes[]; elif writes then .change.after as $after | ($after.name|type=="string") and ($after.type|type=="string") and - ($after.name|test("^[A-Za-z0-9_.*@-]+$")) and + ($after.name|test("\\A[A-Za-z0-9_.*@-]+\\z")) and $after.type==($after.type|ascii_upcase) and ($after.name|fqdn) != "_externaldns.devantler.tech" and (if .change.actions==["update"] then (.change.before|owned($live)|not) else true end) and diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh index f807f52..823cae6 100755 --- a/scripts/tests/test-dns-apply.sh +++ b/scripts/tests/test-dns-apply.sh @@ -125,6 +125,10 @@ for name in _externaldns.cname-sig1._domainkey.devantler.tech _externaldns.CNAME done { printf '%s\n' "$noop_import"; change cloudflare_dns_record.www cloudflare_dns_record '["no-op"]' "$www" "$www" null marker _externaldns.cname-www.devantler.tech; } | reject "marker for a record still managed here" +{ printf '%s\n' "$noop_import"; marker _externaldns.cname-www.devantler.tech + change cloudflare_dns_record.www cloudflare_dns_record '["forget"]' '{"name":"www.devantler.tech","type":"CNAME"}' null null; } | reject "forget without the record id" +{ printf '%s\n' "$noop_import"; marker _externaldns.cname-www.devantler.tech + change cloudflare_dns_record.www cloudflare_dns_record '["forget"]' '{"id":"0000000000000000000000000000000b","name":"www.devantler.tech","type":"CNAME"}' null null; } | reject "forget of a different record" # A marker must claim a live record, and only records this plan forgets. { printf '%s\n' "$noop_import"; marker _externaldns.cname-nothing.devantler.tech; } | reject "marker that claims nothing" for name in _externaldns.cname-sig2._domainkey.devantler.tech _externaldns.sig2._domainkey.devantler.tech; do @@ -138,6 +142,7 @@ portal='{"name":"portal.devantler.tech","type":"A","content":"192.0.2.2"}' { printf '%s\n' "$noop_import" "$dkim_kept"; change cloudflare_dns_record.marker cloudflare_dns_record '["create"]' null '{"name":"_externaldns.cname-sig1._domainkey.devantler.tech","type":"txt","content":"x"}' null; } | reject lower-case-marker-type { printf '%s\n' "$noop_import"; change cloudflare_dns_record.nameless cloudflare_dns_record '["create"]' null '{"type":"TXT","content":"x"}' null; } | reject missing-name { printf '%s\n' "$noop_import"; change cloudflare_dns_record.spaced cloudflare_dns_record '["create"]' null '{"name":" _externaldns.devantler.tech","type":"TXT","content":"x"}' null; } | reject non-hostname-name +{ printf '%s\n' "$noop_import"; change cloudflare_dns_record.newline cloudflare_dns_record '["create"]' null '{"name":"portal.devantler.tech\n","type":"A","content":"192.0.2.3"}' null; } | reject trailing-newline-name printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if check; then exit 1; fi printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi printf '%s\n' 'PASS: DNS plans admit unchanged imports, creates, updates and forgets only' From 994877cce627403c99dd138bec7775200afe2264 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 05:02:36 +0100 Subject: [PATCH 11/11] chore(recovery): retain the state after the DNS adoption apply The import-only apply in run 43674749476 (source 457727e) moved the authoritative state from serial 3 to 4 on the same lineage, with the configuration unchanged. Its `after` package and the fresh runner's `recovered` package both authenticate with the independent identity and carry the same state digest. The DNS workflow requires this generation before its next run. Co-Authored-By: Claude Opus 5.5 --- .../backend-source/configuration.json.age | Bin 493 -> 493 bytes recovery/backend-source/manifest.json | 14 +++++++------- recovery/backend-source/state.json.age | Bin 2399 -> 14132 bytes 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/recovery/backend-source/configuration.json.age b/recovery/backend-source/configuration.json.age index ba28326b4fb4d8b98b9f5c19c4d4ed6ab3ae639a..9d832a17b8e27348165af37f56dd38f4b537621d 100644 GIT binary patch delta 471 zcmV;|0Vw|M1MLHlAb)pIdQeM4d3tzyXfR7bZE|;IO-xx-SUFK|W;sGLPf2Dubxclc zZ)!nUV+vwLaYAfMP(pN1WiU@vWpG$gOmTNId08-GWNbxuZbwN=Q%7r3FH$sAYYHtb zEg)w$c`H*_D@Zp?a8*cFR$_TjLUKbhN=8deRC#!LLsVlyFMl#jK}Vu_V5772}FfrYtmd!D>bNO4;%i?QMhm0d=cKQ@WOwGiMtd|L-EA&lK{&u!lbZ zD~@G#UGtntGk>x^TJmIhIPmEw(f+jl4mis%famS(i7RpYoMVSPMgJ2Y_<{Es)Gl0? zSo(p#g3hcWiLG|ai+`hchH~D!J6i}}v-5Wgj&}Nb5D6wq-M^Zt)|#N41RPYJ1ktU; zw70rUC-bk|nZAA1JMBsm24xVH6HtAC0q_s&Q%{*~lTar;2gmeV?4TK>05=%c^7qJd z;`XhBW>b&lSt*ie{&~NGP1aI%#aoo{QRAeeaV=h*WgO>Q~I>4 Nsn3?b_UD})qLxP0*xUdB delta 471 zcmV;|0Vw|M1MLHlAb(|OMQ3X^c6UivMp$t;PH9R_LsVf&bWb)#a8qJMNSH#u!$R(Ew~bU`p!ZCGS%OIb-VRSGRF zEg)GmWm4O;@ zD6dPmah4B{U%?0&a-8as5kXDGAP=g?&<(r>4MRl}B#RhV6Y;Y>CVUpLGX)pJpW^JB zO~zumBCj@AI z`z3b;^AEPE(8$qr9pB_F1N7T3VNKJu*JP9h~iX diff --git a/recovery/backend-source/manifest.json b/recovery/backend-source/manifest.json index fea3bd6..732162d 100644 --- a/recovery/backend-source/manifest.json +++ b/recovery/backend-source/manifest.json @@ -1,16 +1,16 @@ { "schemaVersion": 1, "kind": "cloudflare-state-recovery", - "capturedAt": "2026-11-02T21:21:43Z", - "sourceCommit": "1628f233a82f2200e9cd07581b5ee8c755bac062", - "sourceRun": 41525197170, + "capturedAt": "2026-11-15T03:59:45Z", + "sourceCommit": "457727e207af0d4acc448eff21937b9c9b7a2e9b", + "sourceRun": 43674749476, "recipient": "age1rk6fs67kly3h4zux5za429z3grjtvs8vcunav4sa28pxk738najsk8wgs6", "lineage": "d981e029-9986-63a7-2f54-379ce10ee6a4", - "serial": 3, + "serial": 4, "accountIdSha256": "3080d90d2ff8a5d38ec8c5dae7b9207b928b353c68f4dd81e0a378cf932a5f95", "zoneNameSha256": "1fc6c364e0d592421f88ba2d2cf6cf15691bbc711113d7907c159a638f35aeb0", - "stateCiphertextSha256": "a506974a068c67698d58ae273cd34e6e718cf6fddc7f3899cb70591053230e50", - "configurationCiphertextSha256": "cd9000b57fe7fcd3fc8035bf3c4ee0da23ddf0692bb1b2e8d731bb9a3380bd66", - "statePlaintextSha256": "029455c41771dbbdfa9c43807796df543fff5faedae690267053f582c43ea835", + "stateCiphertextSha256": "d0758c769aa6f20bea6ff1f88d23e3eed70e54d92026baa895a7e3d3281ce6d6", + "configurationCiphertextSha256": "803b3e4140e92b9d8eb8691e9cdee1bcb12e1f2b44c58e999a4273cea1c7bcaa", + "statePlaintextSha256": "c4bc70c78223477b27d842eb75d2e7a8f6a3d95e74deb16e9098f016eb2a70e4", "configurationPlaintextSha256": "84c9b6ea504f82a9fd694e701287e30d54fe9b42ecf81dfccfbf2cba9873239a" } diff --git a/recovery/backend-source/state.json.age b/recovery/backend-source/state.json.age index 0022e85b9253a04145ca0d416dc3abab9f3e720d..0e5442a769b0a06c7f8693239a836d158b927aaa 100644 GIT binary patch literal 14132 zcmV-4H_OOjXJsvAZewzJaCB*JZZ2zKS6OUS za%WalX;5`xNNGYdV{tTbFGp-gF)L|rWM(loXH5z;LT6G=bT&e3W;bCrN^xQ{X*6n5 zNn&S4a7ImQaC&oaVQ*%2V^C&TPB;oJEiE8da$06}aVsxuG;44~GBbHtR&iNbb!T^U zX)kD5RbokVHD*F)QcFd7Pj?FCkqQd--Av}f_{Ii*l2@eAt7*i%p^c!q&WSUqmQ1Ib1dC!ybIvX8F?N=9Dqnm79a`ah8=Ji2iAF^}=BAmJvY4WH#P zQC@Dn49mnhDDbq7I@;9iwJa#`bcIzIG;4pjS`eSZY%(e$@_MZ1&0Q>tWx?}Sy*uq2 zn|_-xatNh)-s~09VMLX3FDH*E!5$i`N85NaZS7C7R&V5=uOEMEm4>EL5zLF2r+upQ zHecNTXcF|-UB&5bL6gKfw8TS)OZ*=+CEnPLAl2=kifZmd@%gfJvtvetTUzmuKH)uO zCCH?7Y!e_oCP;HR|E%8~X0cj1 z>5^{_+uaDXSk|q8W;FzIqg!v#sIGRSfQNzWv9SPpX`EVcx4e;TnuXG&!1jI1;y~*D z=cW22Ul!I{w(dbY+o6`hxrE9rn!Fd?f}y^_Yi$ zb7bYhdji6hu^R?c!hj$F!s}6(v5K;{w;N3E4h%+5>PvxkyRDifP_UYafrEUxI1aIY z`2qy^${SwZK5*Tboy`|B9QtvKedx_H+9}Ot*(f}Pb}gM?7cn9Oa{EO&9znR+$g<^(o_vu^Zrd+F0kKZjlq6({QSEvo9TwCAV??gzTbccFlSHEXzGQ0|?;is^x&F;}G1r0vwdJKF)5WfJFRtm>a=O{YQ1w7Gcn3M_PZrpYTEZxf zg^uU{Y5J=EIxn#Z^ue#-fY8*z#K~+c`iea(=7KC3-B+}YHEsZ+?3`}q8HJ|V4b^j1 zv&NizVJk(>*!Y3ufPZ0)`l<{G5~_wf@9~&d%+8}V0_}I%-QR=2ah7Olj&b#FfaH%# zeZDt}k~UZ9Qb70ru zGO~=CFG+zn7L@HZkFnFJ4V})^uEwH#Uw5(ys;+= z59swqC`KnZs5@h#&Ij&tx^xI5Ks+l_rW%^9)<(&E`@}2$i`d0H3A>M}eHX|$30lh| zc&t5l;}5pCSJD`ZUkXV+dE+e_`p=V#o-vLWLZU6ao z2L1FRc}QzpQ5g!$Bmpa~mOG0Ue&B2md-|= z+?KfY3JWx(NdE{ub~av!x(NbwN(yf+$ja*s@S#3zF#FQG1={|b2Te8QV-)QzcGEFVa>L6qXSWgb*dEJeT?hkVs4 zd?+zJi<(0aO|*lDGR^4*ONcPRzwNeuFS};EYd59&}lGC#ZFyG#tHZ!j{WN(&f*)o=5D4j$s z&sB{zOmuAaR+(h!<{l{2)d?2xcV-+siinBUq6iVTv(bfHOEGh}?t&y(XoPZUL@GJq zI+u;gv190Vi@PEs#gWl$g`ZIYI9N5u7zTT@anfGRIgFDqELP8*;K1N72t_HaptwXX zl!OcnIoJSthdRn1r}Q+B=D2n-}^QV|ft@zCk5B&Z?)`JPd}$_-7K zu(71pXmg%PH|j4`n;bk)?pywLh|qO&ke}y}9CZS7$pG5_&KMFvdte7oJr9@R92{$nm2y5hpKsUo@aJcX|z@|k*k3!<->YZ$wEGq0S@w_w!@_mFQrxh`@KZh0F zXP*zfddXbF0kshm#{{t=IV@_Hn;4qvm24Dcy@@najonJx5z z;o8lWt3j~9k(zRxwZE{Jz(y#EqktRm=um{nkvAu#e7xl!qR5@2>%(6}^-;<>C{cT3 zESOcEUbn`Mem#(mWI?=|p5``rk=9l~9Kcev-kNn|fW=P~=q$(KwKydr{<=SoAKbJT zbqtfv2)AN8L~8hNturP`^I+qHVW=3%n3IVr0O9W++cxXdiHu?*|DkSe5{90cJPp9P z@8~qr$V3WVy&Z6`6j!8sk_M`0bWO!wLnO%&58zXXuZM{&vI&gTWLs!ATOQ)0Mms|V zNl__`kABjjRSlz61c=fXadLIiT)pgJ@!v z#0Gr_fOU2;pg!IJ5_0vzY{&&Y?yamax@eI4C4Z@bX+-1FpA2mTUDd4bbMsWT=e`x^~tFuJpZU^H^-VGr2#VAAs@4VA9XHq1lTI7 z{FMr~lDnXtW*^!q=5yu(tT@&ztalIj9A1p{@AwYu^lqv&4P$k}TbzHqSG_J> zbZJu0x-rzwGXBtCm0s_E{ta-Z?=th1S|RC_OMVxEPOd zqY3P4oA6=*CeWW+t3eX4+vLk(TU0TMfFll*xtM*`sf9Gx-8mNmbv7z?LW%uWNt-^; zJndom?;!FEXcPO}ko`rx@1c#URhD{`DEY8yHokw~zTMZ zY-(R!+VlfTY1MNsMk*&oN}dm>CvU4+8rvM7zc!3rqRxQlUnb!7o^M@z<-6QUZ(&xs zbQ=rjW6?$Bj|o${?bn4JWHZ2`Z3O*to@R`E#f&e5Xey7SVHKCh;a$xdF2#uzSc! zf&vwfg|>(`@TL!~=Ms29Y(29#1(L~uT-`kp$P>*R{}*r`i#)pm2e7JP-ICTqT-@zQ(Al=qbepCj*m8uEzcNTeJQ;&m!G9sqq2pHgDCWb&KrlnO=b3u}r;ms;-E8I6Axj z^Vph_1i?zVkR?+eCLif`(df19*(~^{e8J`Hgw@N2VIvS<+tlP=BeKmb$)1kjAQg)x z`1mLKju{E16~Mfw_#w;8)F-8YYSD$@3}Z3?#k59sZSYx%3%IYadkcJ7H&ys*A%ouY z&>yTJEK@iV0a2E5*xTzeeID~q_Q$3LEIqK?`BHKzB9-MY4>r0OdKcE`ohQf|gOoXU zP)+KgWOBJ7NpuSnPkYw{p6LtboY2HdH(or7A}rps~grF0&iEzHb!`5Z#}*Z_URAHArFw z_1FONz-Gf1JsO%l-6UDf!N;&oXfPE5tUOxm#03>6oUSYT4B*sipIW!A_i&MZABRoD z1qoY_oRJhZpVvir6Sk0CkQs8KOECJWgSO-Oms zWynTy-1X;sNN1~LRz0`WU}$(A$CZi=@p#`r?Dea$ciN2vtSUw1tYg+6JZF5|Er4g_ z)}$ReTV-`c*PWTOng$2m+!12;pyL;KN7$V*D`fgS#5sv4R$DU^=2*As#5Xn;--<{A zVnG6g0W>Cp+14u^Fn=6R=q+2n`$P@6SXX^v_IK>h(j%ZRD8ey?#glPD$!YQH1Bf^M zDTi*aHin>#1CT3^%0jBg+~JXnPKt#{*;SE)ZGLX{dD`-^veJb5og)T2$COVVcHU>U zcSPqJiGi2#4JiP1lN#oHH?|Jf5q1}z#VNvI|K6f>tSjiQWhm~dIatsF(uCC#tj(`7 zI2+il^up69DuvXhFgEvPsG*Y3Oc`^#ua59ftMnD#ayW_})?Znlf$a9^vS44kLMH3S zKWQ+Z+S|izv#eUppBWU0qyaWWZ#IKDkzmHMw*H%`;zoYg0yt6Xl)9lr2Eo_5QltfZ z4pVR@f}gIz=2uIB_wd`{bF~!rl_W|+kDoI{v!NK$|JaKb?Z+KIvvjRUjBZzAgUpNp z7)FiwK#H%vpEWsOSwylsgnAQ!$jLMROh+64^VqWHWc2Qw4IWc2Q}G809XpGlW(}wk zLwmUKcLeZf*o4rL9KB0+Xs?uq_<0$RXrKqtw1a+2$MzeHj7$kcQ#Qdxe1)O|jP0e9 zL1q1MmWO_}c!v+n9YAW-i;4GP$r>7p-2pKf<_NZ-GK))DSqP@^W&dQhOfQz6*Y@s$ zRb+N-=7bZ%oP!%1(6hC1SH{OqOnxdoIAe@1{Ck614-SX7_9y5YNTQ$IP0q7hbWQX_ zH(!jA1RX1y)<9$L2;0|KE`4Xn2V=NEp`Azon>rLjE9z5|mDl!a$2cHWOgc2liH`)& zHsYhnViIiYm`#)pa>@K8Dvgx7eSFWbxUFMu%b4(_Dh|hlX8P*( zXl;odAV3QHbeO6BNPf9Q)CIIxI>{8Pk&IC#=prC!H z-v6;zFm|yi z&NaCPFK#1sh z%ZqCfd)rDlsABv<=4Q~Cc#K<>=kG7aEMg2wHaC&(XFWaXy9(q6sogq1aw+G5K%Re*C^1EGvLjN$#&O*W_`Lo#2U#Pld)2)r7f73hUZl0aj?jKHQB3Z`U ztq9u_rhY5^huT5m4>NM=Ky{j$57yjH{LX~3q9qik6xpQn*OX~d6?KdqVl?Bn23kli z;!f30aq8&G9YDQ#5e1{warLp`u4ZO&p8OV<=!p?(cqGLz7zhZLpf1VO0FT4yWiD-# zW9Y24u=QIARx{N9W#tA2nP?&!#ikL*WLKj7W<@Fi7e&om1*pLk8Y*;Nmh7qEP)wit zu)n#k8hyo;I81G2T>d=6@sZ+S0;#~?TyjVcerTer30=*Vdr;kyg@LGUgQMHzQ(dHu zru9Y|b1QuN{SE55zaIHsdjNXmRoUfO8>DAeURGZrcfq=3t!l(PRS#0j$c#pwj)RCK z(OFE1K~=4mxX=3zzWbtGTscAe_JpEwh*OO~gxgo6F<+>>_0$Q9(&CGdiE)7w50gun zU;3u{>u<#;@4Jj5D3jQ0jcwn=1`Bt~f)qtOeOB3C_N8topAL%t>K(~?d;Stpmed60 zh&tG04kQfl7d}4IRttG{v3(!5$wJrZ+@kwh$@y?jO*Nl`s5n-tdxB9p8tfCE`ctEbwR(mT%PHi62uWSJbLzLj+|YWcd_$k#*^f{rhLrP>_GaP&p`Hee&n9igvLAJ({QJ?iRndZlG#|wozxHb^Tlf={0dd|ybr)0=}S2= z7Jc~xtj8@U90x^MT=yL!LG(kJJ+5~Jce|EKfvz`hQ#=q4IzijD?>l!sEA+%kBFkYDFb~qgS>^+QCy>`# zI5&-JG1{f07TF%nJxW1(?zxD0_<~DZ#K*LyQ^3;NZJKf<%NT`j3TJ8{UvVS)4?8p} z^=zSgfKUjonhHG?u5C?vjB*D$cO)JkWdenYIAc9c3Sj={0|8eijB2-&)tF{{Lb5!7 z0$0DQP0+%8S?^i25MQCi5b#d#V<;TiemE2hM3#`*Ol16fuEmc$$VyT&BVmQk0HujD zR@S9;kJBp=ahWsOxs&?PspUxmPvA{uOTD|^SyYi+hEw~@jB5H$YeBY~TeyNhiXba? zk;TFXea2{!Au@`b&o5TAP+1|SmQ_Yve|IH{^2Ty=ih&va`myzYFxjR#;&^a@!@|u7 z%NBk5<2Ml9Dc#^V_G18oX7|(wmhZ4!U7I>Stw^f|X#o$>6+G4m;TuhE4ip>{XAt7h=iZ1$Ou?-h~6toNG1}yA;Y*@SZihU&FaAUE&0Zubo|*Z^YCJC zGggzG$%1VeG>l<5E5*~L0TLkSn$?ZNhd1n_3bpR#z^h0 zr4WEQCF}{*Q@kd8-q!XwQeu36yyM)QqH7`=^mtL5g=%#uKi*q(rxrmB_qoV-U0O1M z8lBvbo{`d3%$#tbsF3d67y5oB71NL>B=tujzdB!10Qao+P{~d?pJCa_E2%jG+s7z` zapQJmreke0`jRS%%%_}txzGdu-yOX5otZWW8{QfwzBauuZ&Y-1?up}hw0A838S}oa zM<%;9n`6FZP-Fz3G)8dHk}ht4QHopdIu#Aix}9Pm?n@y`ktxXQzbhxXR2JBqh^K;{ z5mevkK!YWqNZtFWGg#+`ikUJ6q}oKeM(N028zPwH%{Qo~R*L%dgh1PZUv>H!BnNW{ zC~GLmckuSy)A!Xfxr3NZaIcB4^2yg283~t`eAlCnEykf=y>9F9s2|C!KO;DCnwml( z;}f``Ff0|%DyG@0rl6MryFS@u)Vt@pn*6jk7;zrd*r7EWmZ?4gNj;90S{>MfMhcr{ zUJQHh%-qzo2E?OMb=S#C_!;y5YX;*ie6e%VH%(_2$wsH(f)JQY;m)G+*WetyR4Fe+ zQGzF?e@Ob6cENWETukyh*_r4}If>JbYHqqfT%=jG6!!l-9xLqm{a%O-Jnr~HqVE|9 z80XHg$uDwqs&8x@5&t3lZv{cZEOu;911FRd6{kC5jY{ruCc>DBwFfc_oCX;pO=>7b z0q}?o_r&cSiaN5?_EtgHi#88`X?CK>-(6c!)sTFD%9RZbJ{=W$>Ee`sXkt|kDEp20 zdG@Fva@m=oh)VrF%l4kg`+4;D*zlPaiB@h{vtQfT6zl)N9wT(oZOq5tYoGoniYYh= zh#p^l>`J-a65bQrU=s-M`kSi)vO6XJ>MI7nxR!ZvyrJ2{K6J%Xk&n1WVER)xunfBQ@0JT9w2E5j?edjq^wtF5O#mCf)UvMDltrWox7a*YVJ3r-QxrNe3 z#DYiVtHr$c)2JBv33fj!ANY#jUP*`#Ovp)5(0y_I) zdjDZDXr*wyt7w~ysPJVqznQX#&EA7VN&O49LPPiQ=lyKOO+rI|P61A;ZvE!Vet^W< zF$&7ucp`hn@{6&(oA^TU3GTR2SD@SG0~}FiTvJwyvp>@Ltzn!b1xW;>+smHD`+)Dd z!@l3Uq%YG(t^rM7D)750lU=XW;iuPpry|=$eQ zdZv?(DLW{-n5M9FZNNe@e{0DqviH$n9+l1bQY|>cZV8xHqMW(~h@PBvZcu5xJ&_3e zMd_^2rbUnUi|p@e2EZ?AIc{h01cC=>2V}3GBH4vX$vAhEWZU^OhGoL&DbzCwE?l{< zFNuf+lCNsSo3?scvdEK*)b+FL8ZEFXQE^Vt)Edss&i#=wHsM|(vet5o5vE~S95kxo z<=Nsmd%f4AZ5i90Pl)|O%xncEeReU_!TiH|1Dm7uxSfWgRIzr4V2T)kNJ{CxC%zjM zwT*{AQP6oBA3UvMq3iIRy9w@xBifUSgczj$p=v8pS4{LKuXUtywqS3wu$;pTBFa$9 zlW_VS8N*BEHC_@bWw;sa8gM~AEBd##-$D}(pGWgeI>RWdyYCucUc>CqptWF|$AZw7 zwjl#tXr!0d04Kqj11D5&^Ladof`<%KXSIbdA6$a*yT|BfzN?s+OVf8OeZVL!ttShv zo6PqKMI^#vIV%XH!v0$1^Fxd`>eC9f!#!~@@l=@!^DuG+`tidk$r>AW3=}&_D`l(41}GVYATaq*_U6CSHKqh$Vr6e}~ zKRN*$F{7-U*{@y}+HUsFk`Q9J_7x_hF}bENZJlVJHt4z!n@;*6ZS>P{VptBh#UR1> zM5kLHbOpmFVd=Fl>!6sbLUQfket)TjX4ubP>TKpb+MF?Fcb(m`c?i7U>B09l&DvlC zmTLvGvxp`bM5OgcHrse)1T5V7#kgX&dB_tGvZUR>?+7XWlsL25kNp8moZ#zl7}!R3 zGrbqaui|)&(a-LOGb_<8o8eM;*Zry1hawKHJbD!z=`nzF6_dbY?Qz=vn0HF5%rlZT z^j7W%MH8}S&FnE^$olA6a^VVMf`Xk!siQ^ zbK@ShGd_{gmGIdxi?ppbjM9M|bPJEIdeb3QKm~3G(8{RK%_6EiBbI%bqLZ z;D9V8kVVzO2hoK*$kkty9|Dm>!fdG52m90z5AMZfi#xA2;9{t3xEd2OtGdoD&W|iB zm9(v^SQ{uE%RIQ&6hpqRzzXo8k)+XJz()^QP12P@_W32QjQ9~6bF_4$5f|~KJYdC_ zjB^iCK;zt0JVP#Ypbp+9vl!!nWhtf?IIVrNTT^4?&AG_Ese0T}N{NsmD%8_#ubc;(>YuzcK*By~&MqRXV0-+nViYCcSqv%Ez2vG-jE%J( zT#|HZ_!XRaq)l^dGt8+X{;NNosIEhoH5FQ4bh1fDX$u6KQ5Izk5I4oQJy&ZHt0SKF zZz%qm_C?m{7)0s`-&c3zD-q6=eeBR_EoeSqW?1pNffwG!D<5su?4<4+IqY)vGXj?W zxgU&A>s4)=I-SKC&xD57SFm=ek3b(!ZI!Q$Bl%c1>fzu?-9ORTiOWll zl?Edc2_uaEJ(82|-9c)~)jy zP{eVSe&!^=>PVl{FU6v%H#68pXK;TJd5Mf$iwgF@wDHFCh>o69$FO`wyd-*Wy4$;4|+-<#cMHfV_?)ntdLte7i z=~OAV)V}#4$Nu$%U-HqxC!D@({>(?_z8Va*;$K+?@lQ=uLAhF@KX~Jhx4;aCi;Wq-TKUw#jG$rebQ?O;5Z&hBt?5y4h=di|L$;NG z?J}-IH%BbKN`u*h6hI4s%{ORK!XGF&2y(Oo8=0Ooj9Z@Tsr zKNR=|-FtDyrUzi8X=QVfB{6>1|Hmq^bmIO1Z8GQQxq*rRJ^IMy9?v9Px-{Ew+G*=3 zSjmV=TtHH-yy|yl`ZJ4K0x(K)Sypw3F<5I0oMWf$hm~PR#eh4zEGFmuPLmcxMO*S> zrnT_Ek8tB~ z*pPm`)#L{#)X;jCab!;6uS12C5~dJ3sXQ2Ll=S4P(XJ>mcWsO|$bT)h47gUj-Mw=@ zBw{7RW!s$~xOn{4G1)|0i{%H>!~T{4BNY@TpcO1%HQ~g9v7PoUD^Z++-ob7YU>?61 z6rIWpy}hn?S8H<|$@W(m(P4^Q*I^#Xr2{g*(?P0Gh{3o+dO8xecIwHxfiPrjE%85j z=fRFkKc;`$_(q8%iBxOcjT!4a(PEM=Ff8X(d*l6h>#8V18fZHAKu&8kvgcNCEH1T< zd%fM11Ct0@@NR8}c##qj{dZxsPTIq8hhIL)oMQf@!hFYaObh8?6MS{UCT)wy(}!Bk^X|mNphoAK89zQ+j7UMx1^Pwus z`o5&wTR}*v7Iyr|0`v-$< z*x}ZYW`0mYd&KIzPTh#MoT+5C(T@-8DwE8HsU$Cx^aU1Xq*$D*G_1Fz z!NKM1EuO`axw!rfcvX6{6}+S+3aRdzEw~h7`h(2Eu-cWs%|(iNlKN|ddmgLZ@pgqJ zBjeORkk!n&-V#*Jy+c|W=mhF@ z9GG;Yk?)XNNqKQQrOirsocs6>W8@QnZ8KqJ0?63#Cas5*8k?srL>J)j)q(?IbQhFz zvz|oSWl~Jw^Lme-~-X!t+j1(W> zhp{ItqbItq_!~+^u16el%e>S;ZM>0n$ZB){i*f=&CmZa43LpCg*P=V0`8YxdP#5Z( z@&>dqOa`DIkGWusg2_#{{!W~L2j;Ph<_Zn4ulRJR-Lp!wWpktH_$kHvb5{)jBgRTM zLSSlr#&=I}P!XJD8qwNm>sD1i5^h2MW)pRm#Hx+eHe~-dmIQtL=WE0Ezz8bdjtE_} zc`u0;$kkA=Ge%)(TI;j%qA?UUuFEhe>fBw~8iRINe1rdQj|I(5oRFB3Bsq7UbS1F}kHBv}ZqA}zu?A`Wx_ z$5xil&641qT{p05`x62>;r<(6cB^-(ZrGHU*T>J#APN{@nTNa4Q;;E($rLn7XoOLQvto<7K$EgS=q1}W zP58<4n~6U->T^-nP!~R$gy#tfdTc!a3|I|DL{n;BH}x5*R&+Crk#<+hs9K(oS2ZDN z^1_9fXQTVh6^y-ji5M7tUnQu2S}fSV%$Po#r(?M4o;7fF>4_n_-AV&A3b!u>j`8kn zh^DKJ^2ErH&vH>H>-vZF;r%D46{_7xHv2!rt_F;mIUY1B6f#tNHH~9qdC@rr#lxOm z0zJyw>CWz*;*z!Ohkk+^!Cjz&yOXY|dU^eiqE+?to!aKcy8QBPdr(8Wm`&!)$T+h% zhw$b8P@ul~w^XeOk&IE6VA_8bWJQ)M7v|?V+R^v-J+$#~%w@FYku>o~PV9a* zH~wM2l{t6GshXA!+NH-D`k{V&HJ8!36dfa3o4~9Gd^0nri|w|b*kPhI@QWUR*yEPg1MJjFl9Zooio1eBq4P;nXv`;dAd@4&}_j2o3eA!U+5zZfTXJ z^t0rq7*() z73cpqN*{j$d=60}`tp9MRIa;ff*IutUb^jbcOc^hJn?Q_)8ZTX_ckNsG zRC0O3|5$l5pFe*nOIVOoOCnDLXf1&x(;OTld)k|D&p;mhxa{$s_t^IfE3k=I8RnuC zB(t{0{hTcMc2bXyJ9sdQ(8&STNb9Sl)vo5k_0#0oBrB z$ZLC5IRp_wURL>c`FnKTy|fO+QHVoahmz;eP?0K3iNeo~jewI9BL*7}!Ht&0Lrue3 z*Dl1K&dRVuZvZ;O15z~pn{)ic$v^C+S{ga;^D3Mv4L`p==;;n8R+rlde3ESu#Kf;J zo69_oe}k>*l3@`ipfRatCQwj$+9RWxHo|8K*Oe&P9<8wsIGkQw{oP8aZ_EngQMcV%@3fm28$+_Xfl0Rq0EY1G| z3h;$pK{Q@nLw7W0*Z>5VAH%rR>`?U@IH6uAdZRH-m(LN)XCn?EmE*<@IH7BpmX$1R!2#L z6m_|W;J=4nu58|k&rSTYGydTkAq_juXt^S9$joMP`Mw&wR+lx=nTaNmnFWaXK!{`z zWGRkKhO0jA8T2LAS1IgFM{)=M4Je{5e$_%am4LBGRu*L=R#cR{CdcbreTCxZs5?lc zaS|x{$>B)Wx&#EzgjfUFV*TDE`tAUM_(7ZKs}o!aY81Q9evR}fu>J#cdJL@ z{#-5n)FILoALI@Xqo-+qYAom$jlrFU6GqI03y^Tbu948=6xg`jbiA%SPI{|a5}3;Q zcKm#xpKCSfJ>-edQ?SV9Y$8Bo2X!Idamb8&X?jh@8fSiobH`aInfZP+x*Eby=>}h($H#BAsalLh`rY;H**_AqiYIn$= zLH8E5`a@F$it1OLNU~;_AKn{MpN5bKuDRxDo58lQmn0BGx{m{FCX`MNS<&2G-i|M; zi|#<87yAtzUA)MF+N6y+o8sbtqEl6E6D0?zzW7+Ab&D=ctJrzZgMqvbxlc0Nm?~QcV;(ldS!T2OG|f4T2WCiI7($# zbxU+?YYJFJOLTE)OGRgHabjq7Vo^~|Qd(&;aYRsZRB=-|Q*ujGa#LwJZ*OlaSqd#J zEg&;YMNMo}bu~q0R6%hGQQM%&b(?CCB(HR4Rn{PdzcjdE zQ--iI=v~L_@0$}2-n{IBfe|+?s_V<(UdDN@c)f?djdcmUQb1r@l0hNQm^1xc0{<06 z3HUX2#`n5*aepu)fLq>2A%TZa;HN_aL&1V>l?y z&@D3+2eqvC0pOuq+JS9Ae>jNU-o`r8@A~1@cD0ea^&{bPF^{<&sKSrVJXx%E$G(yA=)e ze-pV8hG}}+3o-0YRWp>3g8S5@pg4lR#wn&)t6U>9Bdo~p92FJqN|2x)7eqfT(lAm# ze(rOiNGRZz!P=6!%oZNpP$Y^s&NPgCpSrQ0(+Ub?R8wYCh^_C;P5EpPz~&FS8op|` zf@$P^9)DS|4-K9N=)}lwJMDJ}HK4T`%ZVv6ke49K({>T>5Zsg1P-?PyX%dsdAy|-O zkz-+1bK6xh>Wm>-?rwCrNse}%u~s7)k8V2NG;O0OzG<0S%nEKt?;#zs281kMl;#P#!MXh)cq(>6sLp}hsj!8Wn4TytX-hBuzz|`a3`yMZsg0mt>8=-xd=x3LK|4L zS38;1}uZOp?UnI2=E|{0JtilqI_|` z7N}SOF5_ctb-s9qaTU&)ABQ<>Ep2|2X1%uRx$9TDuOpy~2D~BQOSlKTUrS8HMLm^7 zuYa*)^IWCZM%>~Ph8se8ubG0B)_R{n(i74MUV@Wf|1GpO_;0nraQFpIJ@{RYUGnso zyNjr*B$J)KaW=Fm&ZBFMPqGgU@T1OMcUxTiTXYC;YSWd6e#K80w>dqTa47Tv4|vq8 zVplyj-om6<|9T`~X6cfAu^%IdY68ntF@Gr>`poBox#K%1Fb6|@dX}Jto({(kFP#pe zPB5?M?{mnwD{#v?Ej>L;4^?+T5oMP!(n)vcaB@4NKj?&MkoGI3;{B>V2a7>f_xG(l zU0+V73}<;6?2BG6W*S^P)`dhC7-3F-TPMi4SJi^8Pk=|qkCmY)C+|aIaj%_iD1X`` z9c?B?H8Pzk3}F!@*WQL9_yL6EL#jt=nommnk(@|SeUMUe@}LUm2;88WjAThr*n;|X z)DSgQ-@5}L{6JUoiN&XLB+h3}UfzE(X3BMF$2#aLe>=1}&r>ZKWB5IA3kDe{?Uxcwd$eVFIo~Sb^Fb`kr{Cr*+s(Y8lUvl~ zk|*$311D~t*TCS4UVt$ z@Ojsz>YoDHdTx_wKXZ0U*zH$z_z^yk?_hN(6v6L&=;UH?wtaa)k_{Uo2A$po$gnRr zq`ut)(?P_bhoLDuYa@7)Ownh!7-VsnA_PqYN~;C57$IzwO&L_l?SJLYI}bJ{<$n74 zEFg{9;>4LAPmVr7JLD!nobI%7A#@Y_^VBBBeH$Wn4)@(a5J4dUXYR7I0?N0NtCO*4 ziARJoa=SW|d56eLWU@KE( z@qKwN3CGUoA6r~M;(z0jNr&$?23<_H^PIY#K#3x*!7skJA44(TJ9nF-neTA;jI_@z;uLBM6GpGND0BN9+1Iysx&PftCIXDQ=?v5jTE@r ze2q4W-L0mM>nNOG6wH!-AgS}TBjvUKJtkVMyjR2N{oa$g9SdqXu@M%dg8lt-9zv*qjqRV+bgT)uK&}2BXa^DwI+@h*%%o)!Ro}cak@}R z;BsTDACj*k<^xSx#cT`@Ibxz%Wu~b3#-0ExSY^F(fSQ67iF8QnKn9_{2SSw9nEoko zlv{!BQ9nFx5Pye)m>080A?Y21byyj4LKd%h@X-*86GhY5e!%S-_JLlbv-D3Ie!gIx z5PiyX0DX`8R3BVUYfN0)ZwG^*qEV*AiI)7zza+f8!S+CUC15_roHJ2SfZGHmq^FC& z5sRrs*c;=HgA6uzrRUls$pBC20Wsv}g9SRJmoJN;h<~i6R!=%E`uj$zTAk3dB0>v9 zO5)K{An*ZI?nFB)f*|*$nXYFgslsKo+X*O)smM6t4 z#s5Nx(9;xns@3n0{m7!lDZ9q*5&8|bv{`jar@8ELtC zJScjVOqFESQw=rbBTuMFw3zc`-V%lKYw=pri&16U`X01+B4RXjFR8mf8@Mey=p7~u z|2zM*HpVUbIo{oA&VfJT@amIjkFmK0NrG