From 1f1913995872ec77721f2d11e35009b67b6cb0b4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 10:41:50 +0100 Subject: [PATCH 1/4] feat(zone): stop Cloudflare rewriting proxied pages Since devantler.tech is proxied, Cloudflare changes 76 of its HTML pages: it obfuscates mailto links behind a JavaScript decoder, injects the Bot Fight Mode JavaScript Detections script, and adds AI Labyrinth honeypot links. Declare those settings off in zone.tf so visitors get exactly what the origin serves. On the Free plan JavaScript Detections can only be turned off with Bot Fight Mode; "Block AI bots" is left as it is. The DNS workflow now also imports the zone's single-instance settings (bot management by zone, the listed zone settings by zone and name) and admits only imports and updates of them, never creates or forgets. Part of #21. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 ++- docs/runbooks/bootstrap-and-recovery.md | 5 ++++- scripts/apply-dns.sh | 5 +++-- scripts/check-dns-plan.jq | 14 +++++++++++-- scripts/dns-imports.jq | 16 ++++++++++++++- scripts/tests/test-dns-apply.sh | 25 ++++++++++++++++++++++++ zone.tf | 26 +++++++++++++++++++++++++ 7 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 zone.tf diff --git a/README.md b/README.md index 22dcf7e..fca1bc4 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,8 @@ workflow defaults to checking independent encrypted recovery publication. Its apply mode accepts only the frozen, reviewed first proposal and requires a successful custody check from that same main revision. -DNS records are declared in [`dns.tf`](dns.tf) and change only through the +DNS records are declared in [`dns.tf`](dns.tf), and the zone settings that would +rewrite proxied pages in [`zone.tf`](zone.tf). Both change only through the manual **DNS plan and guarded apply** workflow: plan first, then apply the same reviewed revision. It imports declared records that already exist, never deletes or replaces anything, and keeps an encrypted state copy from before and diff --git a/docs/runbooks/bootstrap-and-recovery.md b/docs/runbooks/bootstrap-and-recovery.md index a7fc82a..d3bef8d 100644 --- a/docs/runbooks/bootstrap-and-recovery.md +++ b/docs/runbooks/bootstrap-and-recovery.md @@ -364,7 +364,10 @@ match, a record already managed here, or a record that external-dns owns refuses the run. The saved plan must then pass `scripts/assert-no-destructive-plan.sh` and -`scripts/check-dns-plan.jq`. Only DNS records may change. Imports must leave +`scripts/check-dns-plan.jq`. Only DNS records and the zone settings in `zone.tf` +may change. Those settings, the bot management configuration and the listed +zone settings, always exist, so they are imported by zone, never created or +forgotten, and may be updated by the change that imports them. Imports must leave their record exactly as it is. Creates, updates and `removed` blocks with `destroy = false` are admitted; deletes and replacements are not. A created or updated record must have a known name and must not be owned by external-dns. diff --git a/scripts/apply-dns.sh b/scripts/apply-dns.sh index fb62ad1..b702c33 100755 --- a/scripts/apply-dns.sh +++ b/scripts/apply-dns.sh @@ -180,9 +180,10 @@ jq -r '.[]|"import {\n to = \(.address)\n id = \(.id|tojson)\n}\n"' "$scratch/ stage=plan plan_json change || stop # Public summary: actions and changed attribute names, never values. Only the -# addresses written in dns.tf are printed; resources keyed by the private +# addresses written in dns.tf and zone.tf are printed; resources keyed by the private # configuration (tokens, buckets, configured records) appear only as a count. -jq -r 'def public: .address|startswith("cloudflare_dns_record.") and (startswith("cloudflare_dns_record.records[")|not); +jq -r 'def public: .address | (startswith("cloudflare_dns_record.") and (startswith("cloudflare_dns_record.records[")|not)) or + startswith("cloudflare_zone_setting.") or startswith("cloudflare_bot_management."); (.resource_changes[] | select(public) | (if .change.importing then "import" else (.change.actions|join("-")) end) as $action | select($action!="no-op") | diff --git a/scripts/check-dns-plan.jq b/scripts/check-dns-plan.jq index a6aa447..9957ed2 100644 --- a/scripts/check-dns-plan.jq +++ b/scripts/check-dns-plan.jq @@ -11,6 +11,14 @@ # names. Otherwise it could hand a mail record, or any other record, to # external-dns, which would delete it. The typeless apex registry name is always refused, # because it claims every apex record, the Email MX and TXT records included. +# +# Besides DNS records, only the zone settings declared in zone.tf may change: +# the bot management configuration and the zone settings listed below. They +# always exist, so they are imported, never created, and may be updated by the +# reviewed change that imports them. +def zone_settings: ["email_obfuscation"]; +def setting: .type=="cloudflare_bot_management" or + (.type=="cloudflare_zone_setting" and (((.change.after // .change.before // {}).setting_id // "")|IN(zone_settings[]))); def fqdn: ascii_downcase | rtrimstr(".") | if .=="@" then "devantler.tech" elif .=="devantler.tech" or endswith(".devantler.tech") then . @@ -51,9 +59,11 @@ all(.resource_changes[]; .mode=="managed" and (.change.actions|IN(["no-op"],["create"],["update"],["forget"])) and (if .change.importing then - any($wanted[]; .address==$resource.address) and .change.actions==["no-op"] and (changed|not) + any($wanted[]; .address==$resource.address) and + (if setting then .change.actions|IN(["no-op"],["update"]) else .change.actions==["no-op"] and (changed|not) end) else true end) and - (if .type!="cloudflare_dns_record" then .change.actions==["no-op"] + (if setting then .change.actions|IN(["no-op"],["update"]) + elif .type!="cloudflare_dns_record" then .change.actions==["no-op"] elif writes then .change.after as $after | ($after.name|type=="string") and ($after.type|type=="string") and diff --git a/scripts/dns-imports.jq b/scripts/dns-imports.jq index 8f54a1a..fa97687 100644 --- a/scripts/dns-imports.jq +++ b/scripts/dns-imports.jq @@ -5,6 +5,12 @@ # no external-dns ownership record claims it. Anything else is refused, so a # typo or drift can never add a second mail record, adopt a record twice, or # take over an external-dns record. +# +# The zone's single-instance settings in zone.tf always exist, so a planned +# create of one of them is always an import of the live setting: the bot +# management configuration by zone, and an allowed zone setting by zone and +# setting name. Any other setting is left to fail at the plan check. +def zone_settings: ["email_obfuscation"]; def fqdn: ascii_downcase | rtrimstr(".") | if .=="@" then "devantler.tech" elif .=="devantler.tech" or endswith(".devantler.tech") then . @@ -33,7 +39,15 @@ $snapshot[0] as $live | elif ($exact|length)>1 then {address, refused:"ambiguous"} elif ($same|length)>0 then {address, refused:"content-differs"} else empty end -] as $resolved | +] as $records | +[.resource_changes[] | + select(.mode=="managed" and .change.actions==["create"]) | + if .type=="cloudflare_bot_management" then {address, id:$live.zone.id} + elif .type=="cloudflare_zone_setting" and (.change.after.setting_id|IN(zone_settings[])) then + {address, id:($live.zone.id+"/"+.change.after.setting_id)} + else empty end +] as $settings | +($records + $settings) as $resolved | ([$resolved[]|select(has("id"))|.id]) as $ids | { imports:[$resolved[]|select(has("id"))]|sort_by(.address), diff --git a/scripts/tests/test-dns-apply.sh b/scripts/tests/test-dns-apply.sh index 8ffb6fb..94ee605 100755 --- a/scripts/tests/test-dns-apply.sh +++ b/scripts/tests/test-dns-apply.sh @@ -173,6 +173,31 @@ printf '%s\n' "$noop_import" | jq -s '{errored:true,resource_changes:.}' | if ch printf '%s\n' "$noop_import" | jq -s '{deferred_changes:[{}],resource_changes:.}' | if check; then exit 1; fi printf '%s\n' 'PASS: DNS plans admit unchanged imports, creates, updates and forgets only' +# 3b. The zone's single-instance settings are always imported, may then be updated, +# and are never created, forgotten or widened to other zone settings. +setting() { jq -n --arg address "$1" --arg type "$2" --arg setting "$3" '{address:$address,mode:"managed",type:$type,change:{actions:["create"],after:({zone_id:null} + (if $setting=="" then {} else {setting_id:$setting,value:"off"} end))}}'; } +result=$( { setting cloudflare_bot_management.primary cloudflare_bot_management "" + setting cloudflare_zone_setting.email_obfuscation cloudflare_zone_setting email_obfuscation + setting cloudflare_zone_setting.rocket_loader cloudflare_zone_setting rocket_loader; } | resolve) +jq -e --arg zone "$zone" '.refused==[] and .imports==[ + {address:"cloudflare_bot_management.primary",id:$zone}, + {address:"cloudflare_zone_setting.email_obfuscation",id:($zone+"/email_obfuscation")}]' <<<"$result" >/dev/null +printf '%s\n' "[{\"address\":\"cloudflare_bot_management.primary\",\"id\":\"$zone\"},{\"address\":\"cloudflare_zone_setting.email_obfuscation\",\"id\":\"$zone/email_obfuscation\"}]" > "$scratch/imports.json" +bot_import=$(change cloudflare_bot_management.primary cloudflare_bot_management '["update"]' '{"fight_mode":true,"enable_js":true}' '{"fight_mode":false,"enable_js":false}' "{\"id\":\"$zone\"}") +obfuscation_import=$(change cloudflare_zone_setting.email_obfuscation cloudflare_zone_setting '["update"]' '{"setting_id":"email_obfuscation","value":"on"}' '{"setting_id":"email_obfuscation","value":"off"}' "{\"id\":\"$zone/email_obfuscation\"}") +printf '%s\n' "$bot_import" "$obfuscation_import" | plan | check +# The import list names only the other resource, so only the action rule can refuse these. +printf '%s\n' "[{\"address\":\"cloudflare_zone_setting.email_obfuscation\",\"id\":\"$zone/email_obfuscation\"}]" > "$scratch/imports.json" +{ printf '%s\n' "$obfuscation_import"; change cloudflare_bot_management.primary cloudflare_bot_management '["create"]' null '{"fight_mode":false}' null; } | reject "created bot management" +printf '%s\n' "[{\"address\":\"cloudflare_bot_management.primary\",\"id\":\"$zone\"}]" > "$scratch/imports.json" +{ printf '%s\n' "$bot_import"; change cloudflare_zone_setting.email_obfuscation cloudflare_zone_setting '["forget"]' '{"setting_id":"email_obfuscation","value":"on"}' null null; } | reject "forgotten zone setting" +printf '%s\n' "[{\"address\":\"cloudflare_bot_management.primary\",\"id\":\"$zone\"},{\"address\":\"cloudflare_zone_setting.email_obfuscation\",\"id\":\"$zone/email_obfuscation\"}]" > "$scratch/imports.json" +{ printf '%s\n' "$bot_import" "$obfuscation_import" + change cloudflare_zone_setting.rocket_loader cloudflare_zone_setting '["update"]' '{"setting_id":"rocket_loader","value":"off"}' '{"setting_id":"rocket_loader","value":"on"}' null; } | reject "update of an unlisted zone setting" +printf '%s\n' "$bot_import" | reject "missing zone-setting import" +printf '%s\n' "[{\"address\":\"cloudflare_dns_record.mail_mx_01\",\"id\":\"$import_id\"}]" > "$scratch/imports.json" +printf '%s\n' 'PASS: zone settings are imported, may be updated, and never created, forgotten or widened' + # 4. The declared mail records are the ones the company Email depends on. grep -q 'content = "mx01.mail.icloud.com"' "$root/dns.tf" grep -q 'content = "mx02.mail.icloud.com"' "$root/dns.tf" diff --git a/zone.tf b/zone.tf new file mode 100644 index 0000000..37fafe3 --- /dev/null +++ b/zone.tf @@ -0,0 +1,26 @@ +# Zone features that rewrite the HTML of proxied pages stay off, so visitors +# get exactly the bytes the origin serves. The business site forbids injected +# scripts, and a strict Content-Security-Policy would block them anyway. +# +# - Email Address Obfuscation replaces mailto links with a JavaScript decoder, +# which hides the published contact address from visitors without JS. +# - Bot Fight Mode injects the JavaScript Detections script on every page; on +# the Free plan it cannot be turned off separately. It can also challenge the +# platform's API and sign-in traffic, which sign-in protects instead. +# - AI Labyrinth (`crawler_protection`) adds hidden honeypot links to pages. +# +# "Block AI bots" acts at the edge without changing pages and is left as it is. +# Every other bot setting keeps its live value, because the provider treats +# unset attributes as computed. +resource "cloudflare_zone_setting" "email_obfuscation" { + zone_id = cloudflare_zone.primary.id + setting_id = "email_obfuscation" + value = "off" +} + +resource "cloudflare_bot_management" "primary" { + zone_id = cloudflare_zone.primary.id + fight_mode = false + enable_js = false + crawler_protection = "disabled" +} From 7c3e5122757d96b529a8b73e7da31af750cdc1c0 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 10:44:03 +0100 Subject: [PATCH 2/4] feat(zone): let each site's robots.txt decide on AI crawlers "Block AI bots" answered GPTBot with 403 on devantler.tech although the site's robots.txt allows every crawler, keeping the business site out of AI search. Set ai_bots_protection to disabled in the same apply; every other bot setting keeps its live value. Co-Authored-By: Claude Opus 5.5 --- zone.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/zone.tf b/zone.tf index 37fafe3..6e27fc6 100644 --- a/zone.tf +++ b/zone.tf @@ -1,5 +1,5 @@ -# Zone features that rewrite the HTML of proxied pages stay off, so visitors -# get exactly the bytes the origin serves. The business site forbids injected +# Zone features that rewrite the HTML of proxied pages, or override a site's own +# crawler policy, stay off, so visitors get exactly the bytes the origin serves. The business site forbids injected # scripts, and a strict Content-Security-Policy would block them anyway. # # - Email Address Obfuscation replaces mailto links with a JavaScript decoder, @@ -8,8 +8,10 @@ # the Free plan it cannot be turned off separately. It can also challenge the # platform's API and sign-in traffic, which sign-in protects instead. # - AI Labyrinth (`crawler_protection`) adds hidden honeypot links to pages. +# - "Block AI bots" (`ai_bots_protection`) answered AI crawlers with 403 although +# the site's robots.txt allows every crawler, keeping the business site out of +# AI search. Each site states its crawler policy in its own robots.txt. # -# "Block AI bots" acts at the edge without changing pages and is left as it is. # Every other bot setting keeps its live value, because the provider treats # unset attributes as computed. resource "cloudflare_zone_setting" "email_obfuscation" { @@ -23,4 +25,5 @@ resource "cloudflare_bot_management" "primary" { fight_mode = false enable_js = false crawler_protection = "disabled" + ai_bots_protection = "disabled" } From d4a91e835956f97cc9bbb3093144c63bf29f8da6 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 10:44:18 +0100 Subject: [PATCH 3/4] docs(zone): rewrap the zone.tf header comment Co-Authored-By: Claude Opus 5.5 --- zone.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zone.tf b/zone.tf index 6e27fc6..183b2e1 100644 --- a/zone.tf +++ b/zone.tf @@ -1,6 +1,7 @@ # Zone features that rewrite the HTML of proxied pages, or override a site's own -# crawler policy, stay off, so visitors get exactly the bytes the origin serves. The business site forbids injected -# scripts, and a strict Content-Security-Policy would block them anyway. +# crawler policy, stay off, so visitors get exactly the bytes the origin serves. +# The business site forbids injected scripts, and a strict +# Content-Security-Policy would block them anyway. # # - Email Address Obfuscation replaces mailto links with a JavaScript decoder, # which hides the published contact address from visitors without JS. From 59124121b0bdcbf3226e150f34e9831d308f92ab Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 15 Nov 2026 10:56:53 +0100 Subject: [PATCH 4/4] fix(zone): keep the bot settings the provider would otherwise reset Provider 5.28.1 gives ai_bots_migration_opt_out a static false default and sends it on update, and bot_preference_sync_enabled is optional without a computed value. Ignore both, so the apply changes only the declared bot settings and leaves these at their live values. Co-Authored-By: Claude Opus 5.5 --- zone.tf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zone.tf b/zone.tf index 183b2e1..e704b79 100644 --- a/zone.tf +++ b/zone.tf @@ -13,8 +13,9 @@ # the site's robots.txt allows every crawler, keeping the business site out of # AI search. Each site states its crawler policy in its own robots.txt. # -# Every other bot setting keeps its live value, because the provider treats -# unset attributes as computed. +# Every other bot setting keeps its live value. Most unset attributes are +# computed; the two the provider would otherwise reset (a static default and an +# optional, non-computed flag) are ignored so this resource never changes them. resource "cloudflare_zone_setting" "email_obfuscation" { zone_id = cloudflare_zone.primary.id setting_id = "email_obfuscation" @@ -27,4 +28,8 @@ resource "cloudflare_bot_management" "primary" { enable_js = false crawler_protection = "disabled" ai_bots_protection = "disabled" + + lifecycle { + ignore_changes = [ai_bots_migration_opt_out, bot_preference_sync_enabled] + } }