From f3d47dc466de99ce5a58eaa1fa824c3a6df10c5d Mon Sep 17 00:00:00 2001 From: Brett Ridenour Date: Mon, 19 Oct 2026 00:34:09 -0500 Subject: [PATCH 1/5] mempool explorer: sitemap.xml, named in robots.txt and checked in CI mempool.lazarus-xbt.xyz/sitemap.xml fell through to the app shell (200 text/html). sitemap.py now holds the explorer's page list (the 19 URLs our SEO advisor's crawl found) and build.sh writes dist/sitemap.xml from it after postprocess.py. Every is the build time: the explorer is one Angular app, so a build is when any page last changed, and there are no hand-kept dates to go stale. Pages serves the file statically (_routes.json exclude, _headers cache rule) and the worker's robots.txt names it. tests/sitemap.py runs in the `sites` CI job: no duplicate or off-host URLs, nothing robots.txt disallows, and robots.txt still names the sitemap. verify-mempool.py checks the deployed robots.txt and sitemap on the candidate alone (the node's nginx serves neither, so they leave the byte-for-byte comparison) and that every listed page answers 200. Co-Authored-By: Claude Opus 5.5 --- cloudflare/mempool-site/_headers | 2 + cloudflare/mempool-site/_worker.js | 2 + cloudflare/mempool-site/build.sh | 8 ++-- cloudflare/mempool-site/sitemap.py | 58 ++++++++++++++++++++++++ cloudflare/mempool-site/tests/sitemap.py | 37 +++++++++++++++ cloudflare/verify-mempool.py | 26 ++++++++++- 6 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 cloudflare/mempool-site/sitemap.py create mode 100644 cloudflare/mempool-site/tests/sitemap.py diff --git a/cloudflare/mempool-site/_headers b/cloudflare/mempool-site/_headers index 847c06b..6dd1fd2 100644 --- a/cloudflare/mempool-site/_headers +++ b/cloudflare/mempool-site/_headers @@ -14,6 +14,8 @@ Cache-Control: public, max-age=300 /lazarus/* Cache-Control: public, max-age=300 +/sitemap.xml + Cache-Control: public, max-age=0, s-maxage=3600 # The licence files have no extension, so Pages would not call them text. /lazarus/NOTICE Content-Type: text/plain; charset=utf-8 diff --git a/cloudflare/mempool-site/_worker.js b/cloudflare/mempool-site/_worker.js index e81630c..2321729 100644 --- a/cloudflare/mempool-site/_worker.js +++ b/cloudflare/mempool-site/_worker.js @@ -59,6 +59,8 @@ Disallow: /api/ Disallow: /*/address/ Disallow: /*/tx/ Allow: / + +Sitemap: https://mempool.lazarus-xbt.xyz/sitemap.xml `; // No Content-Security-Policy on purpose: the Angular app uses inline styles and scripts, the diff --git a/cloudflare/mempool-site/build.sh b/cloudflare/mempool-site/build.sh index 379d03e..a4e6763 100755 --- a/cloudflare/mempool-site/build.sh +++ b/cloudflare/mempool-site/build.sh @@ -56,18 +56,20 @@ for f in favicon.ico favicon-16x16.png favicon-32x32.png apple-touch-icon.png; d done python3 "$HERE/postprocess.py" "$DIST" +python3 "$HERE/sitemap.py" "$DIST" for f in LICENSE NOTICE; do [ -s "$DIST/lazarus/$f" ] || { echo "lazarus/$f missing: the footer links to it" >&2; exit 1; } done cp "$HERE/_worker.js" "$HERE/_headers" "$DIST/" -# Static files skip the worker, except pool-tags.json, which it fetches live from the node. Pages -# gives exclude priority over include, so the theme files are excluded by name, not /lazarus/*. +# Static files skip the worker (sitemap.xml included), except pool-tags.json, which it fetches +# live from the node. Pages gives exclude priority over include, so the theme files are excluded +# by name, not /lazarus/*. python3 - "$DIST" <<'PY' import json, pathlib, sys dist = pathlib.Path(sys.argv[1]) names = sorted(p.name for p in (dist / "lazarus").iterdir() if p.is_file() and p.name != "pool-tags.json") -routes = {"version": 1, "include": ["/*"], "exclude": ["/resources/*"] + ["/lazarus/" + n for n in names]} +routes = {"version": 1, "include": ["/*"], "exclude": ["/resources/*", "/sitemap.xml"] + ["/lazarus/" + n for n in names]} assert len(routes["include"]) + len(routes["exclude"]) <= 100, "Pages allows 100 route rules" (dist / "_routes.json").write_text(json.dumps(routes, indent=2) + "\n") PY diff --git a/cloudflare/mempool-site/sitemap.py b/cloudflare/mempool-site/sitemap.py new file mode 100644 index 0000000..476c8e7 --- /dev/null +++ b/cloudflare/mempool-site/sitemap.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Write the explorer's sitemap.xml: the pages worth a search result, dated by this build. + +The explorer is one Angular app, so every page here is the same shell until the browser fills it +in, and the build is when any of them last changed: each is the build time. Addresses, +transactions and the API stay out (robots.txt in _worker.js disallows them; tests/sitemap.py fails +CI if this list names a disallowed path). Adding a page to the sitemap is one line in PAGES. + + python3 sitemap.py dist # build.sh runs this after postprocess.py +""" +import sys +import time +from pathlib import Path +from xml.sax.saxutils import escape + +sys.dont_write_bytecode = True +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from postprocess import SITE + +# Path and priority, most important first. +PAGES = ( + ("/", 1.0), + ("/mining", 0.64), + ("/graphs", 0.64), + ("/docs", 0.64), + ("/about", 0.64), + ("/mempool-block/0", 0.64), + ("/rbf", 0.64), + ("/mining/pool/lazarus", 0.64), + ("/blocks", 0.64), + ("/docs/api", 0.64), + ("/docs/faq", 0.64), + ("/clock/mempool/0", 0.64), + ("/clock/mined/0", 0.64), + ("/tools/calculator", 0.64), + ("/lazarus/NOTICE", 0.64), + ("/terms-of-service", 0.64), + ("/privacy-policy", 0.64), + ("/trademark-policy", 0.64), + ("/3rdpartylicenses.txt", 0.64), +) + + +def sitemap_xml(built=None): + stamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(built)) + urls = "".join( + f" \n {escape(SITE + path)}\n {stamp}\n" + f" {prio:.4f}\n \n" + for path, prio in PAGES + ) + return ( + '\n' + '\n' + urls + "\n" + ) + + +if __name__ == "__main__": + (Path(sys.argv[1]) / "sitemap.xml").write_text(sitemap_xml(), encoding="utf-8") diff --git a/cloudflare/mempool-site/tests/sitemap.py b/cloudflare/mempool-site/tests/sitemap.py new file mode 100644 index 0000000..3013292 --- /dev/null +++ b/cloudflare/mempool-site/tests/sitemap.py @@ -0,0 +1,37 @@ +"""Check the explorer's sitemap against its robots.txt. No network access, no build needed. + +Fails when the sitemap names a page twice, a page on another host or one robots.txt disallows, +or when robots.txt stops pointing crawlers at the sitemap. +""" +import re +import sys +from pathlib import Path +from urllib.parse import urlsplit +from xml.etree import ElementTree as ET + +sys.dont_write_bytecode = True +HERE = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(HERE)) +from postprocess import SITE +from sitemap import sitemap_xml + +ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} +root = ET.fromstring(sitemap_xml(0)) +assert root.tag == "{%s}urlset" % ns["s"], root.tag +urls = root.findall("s:url", ns) +locs = [u.find("s:loc", ns).text for u in urls] +assert locs and len(locs) == len(set(locs)), "duplicate sitemap URLs" +assert locs[0] == SITE + "/", locs[0] +for u, loc in zip(urls, locs): + assert loc.startswith(SITE + "/"), loc + assert u.find("s:lastmod", ns).text == "1970-01-01T00:00:00+00:00", loc + assert 0 <= float(u.find("s:priority", ns).text) <= 1, loc + +robots = re.search(r"const ROBOTS = `([^`]*)`", (HERE / "_worker.js").read_text(encoding="utf-8")).group(1) +assert f"Sitemap: {SITE}/sitemap.xml" in robots.splitlines(), "robots.txt does not name the sitemap" +# robots.txt matching: a rule is a path prefix, `*` matches any run of characters. +blocked = [re.compile(re.escape(r[len("Disallow: "):]).replace(r"\*", ".*")) for r in robots.splitlines() if r.startswith("Disallow: ")] +for loc in locs: + path = urlsplit(loc).path + assert not any(b.match(path) for b in blocked), f"{path} is in the sitemap but robots.txt disallows it" +print(f"PASS: {len(locs)} explorer sitemap URLs, none disallowed by robots.txt, which names the sitemap") diff --git a/cloudflare/verify-mempool.py b/cloudflare/verify-mempool.py index 412856e..74ed7e5 100644 --- a/cloudflare/verify-mempool.py +++ b/cloudflare/verify-mempool.py @@ -62,7 +62,7 @@ def same(path, headers=None, label=""): same("/145.402fa264d97bba59.js", hdr, label) for p in ("/de/", "/de/mining", "/zh/block/abc", "/pt-BR/", "/resources/config.js", "/resources/customize.js", "/resources/mining-pools/lazarus.svg", "/resources/mining-pools/default.svg", "/resources/favicons/favicon.ico", "/lazarus/chi-rho.svg", - "/resources/pools-v2.json", "/robots.txt", "/resources/nope.png", "/resources/favicons/site.webmanifest"): + "/resources/pools-v2.json", "/resources/nope.png", "/resources/favicons/site.webmanifest"): same(p) # The footer links to the licence files (mempool is AGPL-3.0). The build ships them whether or # not the node's nginx serves them, so they are checked on the candidate alone. @@ -82,6 +82,30 @@ def same(path, headers=None, label=""): if not (st == 200 and body.strip() and "text/plain" in (hh.get("Content-Type") or "")): bad += 1 print(f"DIFF {p}: {st}/{len(body)} {hh.get('Content-Type')}") +# The worker answers robots.txt and the build writes sitemap.xml; the node's nginx serves neither, +# so both are checked on the candidate alone, along with every page the sitemap lists. +import sitemap +from xml.etree import ElementTree as ET +st, body, hh = get(cand, "/robots.txt") +n += 1 +if not (st == 200 and f"Sitemap: {postprocess.SITE}/sitemap.xml" in body.decode().splitlines()): + bad += 1 + print(f"DIFF /robots.txt: {st}, does not name the sitemap") +st, body, hh = get(cand, "/sitemap.xml") +n += 1 +try: + locs = [e.text for e in ET.fromstring(body).iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc")] +except ET.ParseError: + locs = None +if not (st == 200 and "xml" in (hh.get("Content-Type") or "") and locs == [postprocess.SITE + p for p, _ in sitemap.PAGES]): + bad += 1 + print(f"DIFF /sitemap.xml: {st} {hh.get('Content-Type')}, {'unparseable' if locs is None else f'{len(locs)} URLs'}") +for p, _ in sitemap.PAGES: + st = get(cand, p)[0] + n += 1 + if st != 200: + bad += 1 + print(f"DIFF {p}: in the sitemap, candidate answers {st}") # The pie's gateway bands read pool-tags.json, which the worker fetches live from the node # (pools-sync rewrites it every 3 min). A build-time copy would freeze the bands at deploy. import json, time From be1247ca08db8ca6b9d4e05f67457350af2d63b5 Mon Sep 17 00:00:00 2001 From: Brett Ridenour Date: Mon, 19 Oct 2026 01:26:44 -0500 Subject: [PATCH 2/5] mempool explorer: sitemap lists the 16 real pages, everything else noindex A browser check showed four sitemap URLs are app redirects (/graphs, /docs, /docs/api, /blocks land on /graphs/mempool, /docs/faq, /docs/api/rest, /blocks/1), and two were licence text files. The list of indexable pages now lives once, as INDEXABLE in _worker.js: sitemap.py builds sitemap.xml from it, and the worker answers every other page (chart pages, other pools, // copies, address/tx/block pages, the redirect stubs) with X-Robots-Tag: noindex. The licence files get the header from _headers (static) and the worker (3rdpartylicenses.txt). INDEXING.md records what is left out and why, and the temporary bridge Worker that serves this until the Pages deploy. tests/noindex.mjs checks the header rule against a stand-in for the assets; verify-mempool.py checks it on the deployed candidate. Co-Authored-By: Claude Opus 5.5 --- cloudflare/mempool-site/INDEXING.md | 37 +++++++++++++++++++ cloudflare/mempool-site/_headers | 7 +++- cloudflare/mempool-site/_worker.js | 30 ++++++++++++++++ cloudflare/mempool-site/sitemap.py | 43 ++++++++-------------- cloudflare/mempool-site/tests/noindex.mjs | 44 +++++++++++++++++++++++ cloudflare/verify-mempool.py | 17 ++++++--- 6 files changed, 144 insertions(+), 34 deletions(-) create mode 100644 cloudflare/mempool-site/INDEXING.md create mode 100644 cloudflare/mempool-site/tests/noindex.mjs diff --git a/cloudflare/mempool-site/INDEXING.md b/cloudflare/mempool-site/INDEXING.md new file mode 100644 index 0000000..22a4dcb --- /dev/null +++ b/cloudflare/mempool-site/INDEXING.md @@ -0,0 +1,37 @@ +# What search engines may index on mempool.lazarus-xbt.xyz + +There is one rule: **a page is in the sitemap, or it answers `X-Robots-Tag: noindex`.** + +The list is `INDEXABLE` in `_worker.js`. `sitemap.py` builds `sitemap.xml` from it, and the worker +adds the noindex header to every other HTML answer. To make a page indexable, add it there, run +`python3 tests/sitemap.py` and `node tests/noindex.mjs`, and deploy (`deploy.sh mempool`). + +## Indexable (in the sitemap), 16 pages + +`/`, `/mining`, `/graphs/mempool`, `/about`, `/mempool-block/0`, `/rbf`, `/mining/pool/lazarus`, +`/blocks/1`, `/docs/api/rest`, `/docs/faq`, `/clock/mempool/0`, `/clock/mined/0`, +`/tools/calculator`, `/terms-of-service`, `/privacy-policy`, `/trademark-policy` + +## Noindex, and why (decided 19 Oct 2026) + +| What | Why it is left out | +|---|---| +| `/graphs`, `/docs`, `/docs/api`, `/blocks` | The app redirects them to `/graphs/mempool`, `/docs/faq`, `/docs/api/rest` and `/blocks/1`, which are listed. | +| Chart pages: `/graphs/mining/pools`, `/graphs/mining/hashrate-difficulty`, `/graphs/mining/pools-dominance`, `/graphs/mining/block-fees`, `/graphs/mining/block-fees-subsidy`, `/graphs/mining/block-rewards`, `/graphs/mining/block-fee-rates`, `/graphs/mining/block-sizes-weights`, `/graphs/price` | Rendered in the browser, they are a chart plus axis labels and range buttons, with no prose (checked 19 Oct). Too thin for search for now. Add them to `INDEXABLE` if they get real text. | +| `/docs/api/websocket`, `/blocks/stale`, other `/blocks/N` and `/mempool-block/N`, other `/clock/...` and `/tools/...` pages | Variants of listed pages, or thin. | +| Other pools' pages, `/mining/pool/` (except `lazarus`) | Thin, and about other pools. | +| Language copies, `//...` (`/de/mining`, `/zh/`, …) | Duplicates: the same unprefixed URL already serves each visitor's language. | +| `/api`, `/api/` (the API docs shell) | The listed API docs page is `/docs/api/rest`. | +| `/address/...`, `/tx/...`, `/block/...` | Auto-generated, one per address, transaction or block. `/address/` and `/tx/` are also disallowed in robots.txt, which stops Google from seeing the noindex. Lifting that Disallow is sequenced in GitHub issue #46 (Mike's call) and must come after this header is live. | +| `/lazarus/NOTICE`, `/lazarus/LICENSE`, `/lazarus/COPYING.md` (`_headers`), `/3rdpartylicenses.txt` (worker) | Licence files the footer links to, not pages. | + +Scripts, styles and images carry no robots header: crawlers fetch them to render the pages above. + +## Temporary state (19 Oct 2026, until this is deployed) + +The explorer is not in the merge-to-main autodeploy. Until `deploy.sh mempool` ships this +directory, a separate bridge Worker, `lazarus-mempool-seo` on Mike's account, answers +`/robots.txt` and `/sitemap.xml` and adds the noindex header to the pages under its routes +(`/graphs*`, `/docs*`, `/blocks*`, `/mining*`, `/tools*`, `/clock*`, `/mempool-block*`, the licence +files). It is generated from this directory's `_worker.js` and `sitemap.py`. Delete it right +**after** the Pages deploy, not before: `wrangler delete --name lazarus-mempool-seo`. diff --git a/cloudflare/mempool-site/_headers b/cloudflare/mempool-site/_headers index 6dd1fd2..e009a1c 100644 --- a/cloudflare/mempool-site/_headers +++ b/cloudflare/mempool-site/_headers @@ -16,11 +16,16 @@ Cache-Control: public, max-age=300 /sitemap.xml Cache-Control: public, max-age=0, s-maxage=3600 -# The licence files have no extension, so Pages would not call them text. +# The licence files have no extension, so Pages would not call them text. They are files the +# footer links to, not pages, so they stay out of search results (INDEXING.md). /lazarus/NOTICE Content-Type: text/plain; charset=utf-8 + X-Robots-Tag: noindex /lazarus/LICENSE Content-Type: text/plain; charset=utf-8 + X-Robots-Tag: noindex +/lazarus/COPYING.md + X-Robots-Tag: noindex /resources/mining-pools/lazarus.svg ! Cache-Control Cache-Control: public, max-age=300 diff --git a/cloudflare/mempool-site/_worker.js b/cloudflare/mempool-site/_worker.js index 2321729..5e515b4 100644 --- a/cloudflare/mempool-site/_worker.js +++ b/cloudflare/mempool-site/_worker.js @@ -63,6 +63,32 @@ Allow: / Sitemap: https://mempool.lazarus-xbt.xyz/sitemap.xml `; +// The pages search engines may index. This is the one list: sitemap.py builds sitemap.xml from it +// (so it stays valid JSON: double quotes, no trailing comma), and every other page the app serves +// answers with X-Robots-Tag: noindex. INDEXING.md says what is left out and why. +const INDEXABLE = [ + "/", + "/mining", + "/graphs/mempool", + "/about", + "/mempool-block/0", + "/rbf", + "/mining/pool/lazarus", + "/blocks/1", + "/docs/api/rest", + "/docs/faq", + "/clock/mempool/0", + "/clock/mined/0", + "/tools/calculator", + "/terms-of-service", + "/privacy-policy", + "/trademark-policy" +]; + +function noindexUnlisted(headers, path) { + if (!INDEXABLE.includes(path)) headers.set("X-Robots-Tag", "noindex"); +} + // No Content-Security-Policy on purpose: the Angular app uses inline styles and scripts, the // theme loads Google Fonts and calls the pool API cross-origin, and a policy tight enough to be // worth having would break all three. HSTS carries no includeSubDomains and no preload. @@ -102,6 +128,7 @@ async function shell(env, request, lang, status = 200) { const out = new Response(request.method === "HEAD" ? null : res.body, { status: res.status === 200 ? status : res.status, headers: res.headers }); out.headers.set("Cache-Control", LOCALE_CACHE); out.headers.set("Vary", "Accept-Encoding, Accept-Language, Cookie"); + noindexUnlisted(out.headers, new URL(request.url).pathname); secure(out.headers); return out; } @@ -241,6 +268,9 @@ export default { const out = new Response(hit.body, hit); out.headers.set("Cache-Control", HASHED_RE.test(path) ? HASHED_CACHE : LOCALE_CACHE); out.headers.set("Vary", "Accept-Encoding, Accept-Language, Cookie"); + // Text documents (3rdpartylicenses.txt) are files, not pages; scripts and styles are left + // alone so nothing a renderer needs carries a robots header. + if (/\.(txt|md)$/.test(path)) out.headers.set("X-Robots-Tag", "noindex"); secure(out.headers); return out; } diff --git a/cloudflare/mempool-site/sitemap.py b/cloudflare/mempool-site/sitemap.py index 476c8e7..9693571 100644 --- a/cloudflare/mempool-site/sitemap.py +++ b/cloudflare/mempool-site/sitemap.py @@ -1,52 +1,39 @@ #!/usr/bin/env python3 """Write the explorer's sitemap.xml: the pages worth a search result, dated by this build. +The list is INDEXABLE in _worker.js, the same list the worker uses to send every other page +X-Robots-Tag: noindex, so the sitemap and what may be indexed cannot drift apart. Adding a page is +one line there (INDEXING.md says what is left out and why). + The explorer is one Angular app, so every page here is the same shell until the browser fills it -in, and the build is when any of them last changed: each is the build time. Addresses, -transactions and the API stay out (robots.txt in _worker.js disallows them; tests/sitemap.py fails -CI if this list names a disallowed path). Adding a page to the sitemap is one line in PAGES. +in, and the build is when any of them last changed: each is the build time. python3 sitemap.py dist # build.sh runs this after postprocess.py """ +import json +import re import sys import time from pathlib import Path from xml.sax.saxutils import escape sys.dont_write_bytecode = True -sys.path.insert(0, str(Path(__file__).resolve().parent)) +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) from postprocess import SITE -# Path and priority, most important first. -PAGES = ( - ("/", 1.0), - ("/mining", 0.64), - ("/graphs", 0.64), - ("/docs", 0.64), - ("/about", 0.64), - ("/mempool-block/0", 0.64), - ("/rbf", 0.64), - ("/mining/pool/lazarus", 0.64), - ("/blocks", 0.64), - ("/docs/api", 0.64), - ("/docs/faq", 0.64), - ("/clock/mempool/0", 0.64), - ("/clock/mined/0", 0.64), - ("/tools/calculator", 0.64), - ("/lazarus/NOTICE", 0.64), - ("/terms-of-service", 0.64), - ("/privacy-policy", 0.64), - ("/trademark-policy", 0.64), - ("/3rdpartylicenses.txt", 0.64), -) + +def indexable(): + src = (HERE / "_worker.js").read_text(encoding="utf-8") + return json.loads(re.search(r"^const INDEXABLE = (\[.*?\]);", src, re.S | re.M).group(1)) def sitemap_xml(built=None): stamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(built)) urls = "".join( f" \n {escape(SITE + path)}\n {stamp}\n" - f" {prio:.4f}\n \n" - for path, prio in PAGES + f" {1.0 if path == '/' else 0.64:.4f}\n \n" + for path in indexable() ) return ( '\n' diff --git a/cloudflare/mempool-site/tests/noindex.mjs b/cloudflare/mempool-site/tests/noindex.mjs new file mode 100644 index 0000000..35255cf --- /dev/null +++ b/cloudflare/mempool-site/tests/noindex.mjs @@ -0,0 +1,44 @@ +// Check which explorer answers carry X-Robots-Tag: noindex, against a stand-in for the Pages +// assets. No network: the sitemap pages must stay indexable, every other page and text file must +// not. +import { readFileSync, writeFileSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const src = new URL("../_worker.js", import.meta.url); +const dir = mkdtempSync(join(tmpdir(), "mempool-worker-")); +writeFileSync(join(dir, "worker.mjs"), readFileSync(src)); +const worker = (await import(join(dir, "worker.mjs"))).default; +const indexable = JSON.parse(/^const INDEXABLE = (\[[\s\S]*?\]);/m.exec(readFileSync(src, "utf8"))[1]); + +const FILES = { "/3rdpartylicenses.txt": "text/plain", "/main.0123456789abcdef.js": "text/javascript" }; +const env = { + ASSETS: { + async fetch(req) { + const path = new URL(req.url).pathname; + if (FILES[path]) return new Response("x", { headers: { "Content-Type": FILES[path] } }); + // Pages' SPA fallback: anything else is the app shell, which // also is. + return new Response("", { headers: { "Content-Type": "text/html" } }); + }, + }, +}; + +async function robots(path) { + const res = await worker.fetch(new Request("https://mempool.lazarus-xbt.xyz" + path), env, {}); + return res.headers.get("X-Robots-Tag"); +} + +let bad = 0; +const expect = (path, want, got) => { + if (got !== want) { + bad++; + console.error(`${path}: X-Robots-Tag ${got}, want ${want}`); + } +}; +for (const p of indexable) expect(p, null, await robots(p)); +for (const p of ["/graphs", "/docs", "/blocks", "/graphs/mining/pools", "/mining/pool/convoy", "/de/mining", "/zh/", "/block/abc", "/address/bc1qtest", "/tx/abc", "/api", "/3rdpartylicenses.txt"]) { + expect(p, "noindex", await robots(p)); +} +expect("/main.0123456789abcdef.js", null, await robots("/main.0123456789abcdef.js")); +if (bad) process.exit(1); +console.log(`PASS: ${indexable.length} sitemap pages indexable; other pages, language copies and text files noindex`); diff --git a/cloudflare/verify-mempool.py b/cloudflare/verify-mempool.py index 74ed7e5..070fdf6 100644 --- a/cloudflare/verify-mempool.py +++ b/cloudflare/verify-mempool.py @@ -97,15 +97,22 @@ def same(path, headers=None, label=""): locs = [e.text for e in ET.fromstring(body).iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc")] except ET.ParseError: locs = None -if not (st == 200 and "xml" in (hh.get("Content-Type") or "") and locs == [postprocess.SITE + p for p, _ in sitemap.PAGES]): +if not (st == 200 and "xml" in (hh.get("Content-Type") or "") and locs == [postprocess.SITE + p for p in sitemap.indexable()]): bad += 1 print(f"DIFF /sitemap.xml: {st} {hh.get('Content-Type')}, {'unparseable' if locs is None else f'{len(locs)} URLs'}") -for p, _ in sitemap.PAGES: - st = get(cand, p)[0] +# Sitemap pages are indexable; everything else answers noindex (INDEXING.md). +for p in sitemap.indexable(): + st, _, hh = get(cand, p) n += 1 - if st != 200: + if st != 200 or hh.get("X-Robots-Tag"): bad += 1 - print(f"DIFF {p}: in the sitemap, candidate answers {st}") + print(f"DIFF {p}: in the sitemap, candidate answers {st} X-Robots-Tag {hh.get('X-Robots-Tag')}") +for p in ("/graphs", "/graphs/mining/pools", "/de/mining", "/lazarus/NOTICE", "/lazarus/LICENSE", "/3rdpartylicenses.txt"): + st, _, hh = get(cand, p) + n += 1 + if st != 200 or "noindex" not in (hh.get("X-Robots-Tag") or ""): + bad += 1 + print(f"DIFF {p}: not in the sitemap, should be noindex, got {st} X-Robots-Tag {hh.get('X-Robots-Tag')}") # The pie's gateway bands read pool-tags.json, which the worker fetches live from the node # (pools-sync rewrites it every 3 min). A build-time copy would freeze the bands at deploy. import json, time From 2ab20f92b06aa8114808620f26494d8b98c6c89f Mon Sep 17 00:00:00 2001 From: Brett Ridenour Date: Mon, 19 Oct 2026 01:31:06 -0500 Subject: [PATCH 3/5] mempool robots.txt: drop the redundant "Allow: /" Everything not disallowed is allowed already. Google and Bing (RFC 9309) pick the longest matching rule, so order never mattered to them, but a first-match parser reading "Allow: /" above the Disallows would allow every URL, address and tx pages included. Without the line, no Allow sits below a Disallow and every parser reads the same rules. Co-Authored-By: Claude Opus 5.5 --- cloudflare/mempool-site/_worker.js | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudflare/mempool-site/_worker.js b/cloudflare/mempool-site/_worker.js index 5e515b4..1921a3d 100644 --- a/cloudflare/mempool-site/_worker.js +++ b/cloudflare/mempool-site/_worker.js @@ -58,7 +58,6 @@ Disallow: /tx/ Disallow: /api/ Disallow: /*/address/ Disallow: /*/tx/ -Allow: / Sitemap: https://mempool.lazarus-xbt.xyz/sitemap.xml `; From 213a40c3af4c8a030112ce36be3190deec3c894e Mon Sep 17 00:00:00 2001 From: Brett Ridenour Date: Mon, 19 Oct 2026 01:43:19 -0500 Subject: [PATCH 4/5] mempool robots.txt: "Allow: /" on the second line Brett's call, matching the pool's robots.txt: the Allow goes right under User-agent, above the Disallows. Google and Bing pick the longest matching rule, so the Disallows still apply. A first-match parser would read it as allow-all; the worker's 429 on crawler /api/address/ lookups stays the backstop for the electrs load the Disallows exist for. Co-Authored-By: Claude Opus 5.5 --- cloudflare/mempool-site/_worker.js | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudflare/mempool-site/_worker.js b/cloudflare/mempool-site/_worker.js index 1921a3d..400bffa 100644 --- a/cloudflare/mempool-site/_worker.js +++ b/cloudflare/mempool-site/_worker.js @@ -53,6 +53,7 @@ const API_TTL = [ // until a crawler rereads that, its address lookups are turned away here. const CRAWLER_RE = /bot|crawl|spider|slurp|bingpreview|facebookexternalhit|embedly|headless/i; const ROBOTS = `User-agent: * +Allow: / Disallow: /address/ Disallow: /tx/ Disallow: /api/ From 11aa42f23cbaa1f9186ecda67a1a986519cc0ff2 Mon Sep 17 00:00:00 2001 From: Brett Ridenour Date: Mon, 19 Oct 2026 02:33:36 -0500 Subject: [PATCH 5/5] mempool sitemap: 5 pool pages in /sitemap-en.xml, /sitemap.xml indexes it On IT's advice the sitemap keeps only the pages about this chain and this pool: /, /mining, /mining/pool/lazarus, /blocks/1 and /graphs/mempool (the pages /blocks and /graphs redirect to). The stock mempool.space pages (/about, /docs/faq, API docs, terms, privacy, trademark, calculator) and the live widgets (/clock/*, /mempool-block/0, /rbf) leave INDEXABLE and so get noindex. is gone; Google ignores it. The pages are listed in /sitemap-en.xml, which robots.txt names, and /sitemap.xml is an index of it, as on the pool site and the hub: Search Console reads a new sitemap URL on submit, while /sitemap.xml was never fetched. robots.txt is otherwise unchanged. Co-Authored-By: Claude Opus 5.5 --- cloudflare/mempool-site/INDEXING.md | 39 +++++++++++++++-------- cloudflare/mempool-site/_headers | 2 ++ cloudflare/mempool-site/_worker.js | 22 ++++--------- cloudflare/mempool-site/build.sh | 4 +-- cloudflare/mempool-site/sitemap.py | 27 +++++++++++++--- cloudflare/mempool-site/tests/noindex.mjs | 2 +- cloudflare/mempool-site/tests/sitemap.py | 14 +++++--- cloudflare/verify-mempool.py | 30 +++++++++-------- 8 files changed, 84 insertions(+), 56 deletions(-) diff --git a/cloudflare/mempool-site/INDEXING.md b/cloudflare/mempool-site/INDEXING.md index 22a4dcb..4868447 100644 --- a/cloudflare/mempool-site/INDEXING.md +++ b/cloudflare/mempool-site/INDEXING.md @@ -2,36 +2,49 @@ There is one rule: **a page is in the sitemap, or it answers `X-Robots-Tag: noindex`.** -The list is `INDEXABLE` in `_worker.js`. `sitemap.py` builds `sitemap.xml` from it, and the worker -adds the noindex header to every other HTML answer. To make a page indexable, add it there, run -`python3 tests/sitemap.py` and `node tests/noindex.mjs`, and deploy (`deploy.sh mempool`). +The list is `INDEXABLE` in `_worker.js`. `sitemap.py` builds `sitemap-en.xml` from it, plus +`sitemap.xml`, an index that names `sitemap-en.xml` (the layout the pool site and the hub use). +robots.txt names `sitemap-en.xml`. The worker adds the noindex header to every other HTML answer. +To make a page indexable, add it there, run `python3 tests/sitemap.py` and `node tests/noindex.mjs`, +and deploy (`deploy.sh mempool`). -## Indexable (in the sitemap), 16 pages +## Indexable (in the sitemap), 5 pages -`/`, `/mining`, `/graphs/mempool`, `/about`, `/mempool-block/0`, `/rbf`, `/mining/pool/lazarus`, -`/blocks/1`, `/docs/api/rest`, `/docs/faq`, `/clock/mempool/0`, `/clock/mined/0`, -`/tools/calculator`, `/terms-of-service`, `/privacy-policy`, `/trademark-policy` +`/`, `/mining`, `/mining/pool/lazarus`, `/blocks/1`, `/graphs/mempool` -## Noindex, and why (decided 19 Oct 2026) +These are the pages about this chain and this pool. `/blocks/1` and `/graphs/mempool` are where the +app sends `/blocks` and `/graphs`, so the sitemap names the page a crawler lands on. The entries +carry `` (the build time) and no ``, which Google ignores. + +## Noindex, and why (19 Oct 2026; trimmed to pool pages the same day, on IT's advice) | What | Why it is left out | |---|---| -| `/graphs`, `/docs`, `/docs/api`, `/blocks` | The app redirects them to `/graphs/mempool`, `/docs/faq`, `/docs/api/rest` and `/blocks/1`, which are listed. | +| `/about`, `/docs/faq`, `/docs/api/rest`, `/docs/api/websocket`, `/terms-of-service`, `/privacy-policy`, `/trademark-policy`, `/tools/calculator` | Stock mempool.space pages, word for word (`/about` gains one attribution paragraph from the theme). A copy of another site's page earns no result of its own. | +| `/clock/...`, `/mempool-block/N`, `/rbf` | Live widgets: numbers that change by the second, no text to rank. | +| `/graphs`, `/docs`, `/docs/api`, `/blocks` | The app redirects them to `/graphs/mempool`, `/docs/faq`, `/docs/api/rest` and `/blocks/1`. | | Chart pages: `/graphs/mining/pools`, `/graphs/mining/hashrate-difficulty`, `/graphs/mining/pools-dominance`, `/graphs/mining/block-fees`, `/graphs/mining/block-fees-subsidy`, `/graphs/mining/block-rewards`, `/graphs/mining/block-fee-rates`, `/graphs/mining/block-sizes-weights`, `/graphs/price` | Rendered in the browser, they are a chart plus axis labels and range buttons, with no prose (checked 19 Oct). Too thin for search for now. Add them to `INDEXABLE` if they get real text. | -| `/docs/api/websocket`, `/blocks/stale`, other `/blocks/N` and `/mempool-block/N`, other `/clock/...` and `/tools/...` pages | Variants of listed pages, or thin. | +| `/blocks/stale`, other `/blocks/N` | Variants of a listed page. | | Other pools' pages, `/mining/pool/` (except `lazarus`) | Thin, and about other pools. | | Language copies, `//...` (`/de/mining`, `/zh/`, …) | Duplicates: the same unprefixed URL already serves each visitor's language. | -| `/api`, `/api/` (the API docs shell) | The listed API docs page is `/docs/api/rest`. | +| `/api`, `/api/` (the API docs shell) | Stock API docs, as above. | | `/address/...`, `/tx/...`, `/block/...` | Auto-generated, one per address, transaction or block. `/address/` and `/tx/` are also disallowed in robots.txt, which stops Google from seeing the noindex. Lifting that Disallow is sequenced in GitHub issue #46 (Mike's call) and must come after this header is live. | | `/lazarus/NOTICE`, `/lazarus/LICENSE`, `/lazarus/COPYING.md` (`_headers`), `/3rdpartylicenses.txt` (worker) | Licence files the footer links to, not pages. | Scripts, styles and images carry no robots header: crawlers fetch them to render the pages above. +## robots.txt + +Unchanged apart from the `Sitemap:` line: `Allow: /`, then Disallow `/address/`, `/tx/`, `/api/` +and their language-prefixed forms (`/*/address/`, `/*/tx/`). Nothing in the sitemap may match a +Disallow; `tests/sitemap.py` checks it. + ## Temporary state (19 Oct 2026, until this is deployed) The explorer is not in the merge-to-main autodeploy. Until `deploy.sh mempool` ships this directory, a separate bridge Worker, `lazarus-mempool-seo` on Mike's account, answers -`/robots.txt` and `/sitemap.xml` and adds the noindex header to the pages under its routes -(`/graphs*`, `/docs*`, `/blocks*`, `/mining*`, `/tools*`, `/clock*`, `/mempool-block*`, the licence +`/robots.txt`, `/sitemap.xml` and `/sitemap-en.xml` and adds the noindex header to the pages under +its routes (`/graphs*`, `/docs*`, `/blocks*`, `/mining*`, `/tools*`, `/clock*`, `/mempool-block*`, +`/about*`, `/rbf*`, `/terms-of-service*`, `/privacy-policy*`, `/trademark-policy*`, the licence files). It is generated from this directory's `_worker.js` and `sitemap.py`. Delete it right **after** the Pages deploy, not before: `wrangler delete --name lazarus-mempool-seo`. diff --git a/cloudflare/mempool-site/_headers b/cloudflare/mempool-site/_headers index e009a1c..5a2a555 100644 --- a/cloudflare/mempool-site/_headers +++ b/cloudflare/mempool-site/_headers @@ -16,6 +16,8 @@ Cache-Control: public, max-age=300 /sitemap.xml Cache-Control: public, max-age=0, s-maxage=3600 +/sitemap-en.xml + Cache-Control: public, max-age=0, s-maxage=3600 # The licence files have no extension, so Pages would not call them text. They are files the # footer links to, not pages, so they stay out of search results (INDEXING.md). /lazarus/NOTICE diff --git a/cloudflare/mempool-site/_worker.js b/cloudflare/mempool-site/_worker.js index 400bffa..854e9fb 100644 --- a/cloudflare/mempool-site/_worker.js +++ b/cloudflare/mempool-site/_worker.js @@ -60,29 +60,19 @@ Disallow: /api/ Disallow: /*/address/ Disallow: /*/tx/ -Sitemap: https://mempool.lazarus-xbt.xyz/sitemap.xml +Sitemap: https://mempool.lazarus-xbt.xyz/sitemap-en.xml `; -// The pages search engines may index. This is the one list: sitemap.py builds sitemap.xml from it -// (so it stays valid JSON: double quotes, no trailing comma), and every other page the app serves -// answers with X-Robots-Tag: noindex. INDEXING.md says what is left out and why. +// The pages search engines may index: the ones about this chain and this pool, not the stock +// mempool.space pages or the live widgets. This is the one list: sitemap.py builds sitemap-en.xml +// from it (so it stays valid JSON: double quotes, no trailing comma), and every other page the app +// serves answers with X-Robots-Tag: noindex. INDEXING.md says what is left out and why. const INDEXABLE = [ "/", "/mining", - "/graphs/mempool", - "/about", - "/mempool-block/0", - "/rbf", "/mining/pool/lazarus", "/blocks/1", - "/docs/api/rest", - "/docs/faq", - "/clock/mempool/0", - "/clock/mined/0", - "/tools/calculator", - "/terms-of-service", - "/privacy-policy", - "/trademark-policy" + "/graphs/mempool" ]; function noindexUnlisted(headers, path) { diff --git a/cloudflare/mempool-site/build.sh b/cloudflare/mempool-site/build.sh index a4e6763..c8c34f8 100755 --- a/cloudflare/mempool-site/build.sh +++ b/cloudflare/mempool-site/build.sh @@ -62,14 +62,14 @@ for f in LICENSE NOTICE; do done cp "$HERE/_worker.js" "$HERE/_headers" "$DIST/" -# Static files skip the worker (sitemap.xml included), except pool-tags.json, which it fetches +# Static files skip the worker (both sitemaps included), except pool-tags.json, which it fetches # live from the node. Pages gives exclude priority over include, so the theme files are excluded # by name, not /lazarus/*. python3 - "$DIST" <<'PY' import json, pathlib, sys dist = pathlib.Path(sys.argv[1]) names = sorted(p.name for p in (dist / "lazarus").iterdir() if p.is_file() and p.name != "pool-tags.json") -routes = {"version": 1, "include": ["/*"], "exclude": ["/resources/*", "/sitemap.xml"] + ["/lazarus/" + n for n in names]} +routes = {"version": 1, "include": ["/*"], "exclude": ["/resources/*", "/sitemap.xml", "/sitemap-en.xml"] + ["/lazarus/" + n for n in names]} assert len(routes["include"]) + len(routes["exclude"]) <= 100, "Pages allows 100 route rules" (dist / "_routes.json").write_text(json.dumps(routes, indent=2) + "\n") PY diff --git a/cloudflare/mempool-site/sitemap.py b/cloudflare/mempool-site/sitemap.py index 9693571..55e6b0f 100644 --- a/cloudflare/mempool-site/sitemap.py +++ b/cloudflare/mempool-site/sitemap.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -"""Write the explorer's sitemap.xml: the pages worth a search result, dated by this build. +"""Write the explorer's sitemaps: sitemap-en.xml, the pages worth a search result dated by this +build, and sitemap.xml, an index naming it (the same layout as the pool site and the hub). The list is INDEXABLE in _worker.js, the same list the worker uses to send every other page X-Robots-Tag: noindex, so the sitemap and what may be indexed cannot drift apart. Adding a page is one line there (INDEXING.md says what is left out and why). The explorer is one Angular app, so every page here is the same shell until the browser fills it -in, and the build is when any of them last changed: each is the build time. +in, and the build is when any of them last changed: each is the build time. There is no +: Google ignores it. python3 sitemap.py dist # build.sh runs this after postprocess.py """ @@ -22,6 +24,9 @@ sys.path.insert(0, str(HERE)) from postprocess import SITE +# The file that lists the pages, and the one robots.txt names. sitemap.xml only points at it. +PAGES = "sitemap-en.xml" + def indexable(): src = (HERE / "_worker.js").read_text(encoding="utf-8") @@ -29,10 +34,10 @@ def indexable(): def sitemap_xml(built=None): + """sitemap-en.xml: one per indexable page.""" stamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(built)) urls = "".join( - f" \n {escape(SITE + path)}\n {stamp}\n" - f" {1.0 if path == '/' else 0.64:.4f}\n \n" + f" \n {escape(SITE + path)}\n {stamp}\n \n" for path in indexable() ) return ( @@ -41,5 +46,17 @@ def sitemap_xml(built=None): ) +def index_xml(): + """sitemap.xml: an index naming sitemap-en.xml, for crawlers that look at the usual place.""" + return ( + '\n' + '\n' + f" {escape(SITE)}/{PAGES}\n" + "\n" + ) + + if __name__ == "__main__": - (Path(sys.argv[1]) / "sitemap.xml").write_text(sitemap_xml(), encoding="utf-8") + out = Path(sys.argv[1]) + (out / PAGES).write_text(sitemap_xml(), encoding="utf-8") + (out / "sitemap.xml").write_text(index_xml(), encoding="utf-8") diff --git a/cloudflare/mempool-site/tests/noindex.mjs b/cloudflare/mempool-site/tests/noindex.mjs index 35255cf..c5aa7dc 100644 --- a/cloudflare/mempool-site/tests/noindex.mjs +++ b/cloudflare/mempool-site/tests/noindex.mjs @@ -36,7 +36,7 @@ const expect = (path, want, got) => { } }; for (const p of indexable) expect(p, null, await robots(p)); -for (const p of ["/graphs", "/docs", "/blocks", "/graphs/mining/pools", "/mining/pool/convoy", "/de/mining", "/zh/", "/block/abc", "/address/bc1qtest", "/tx/abc", "/api", "/3rdpartylicenses.txt"]) { +for (const p of ["/graphs", "/docs", "/blocks", "/about", "/docs/faq", "/terms-of-service", "/tools/calculator", "/clock/mempool/0", "/rbf", "/graphs/mining/pools", "/mining/pool/convoy", "/de/mining", "/zh/", "/block/abc", "/address/bc1qtest", "/tx/abc", "/api", "/3rdpartylicenses.txt"]) { expect(p, "noindex", await robots(p)); } expect("/main.0123456789abcdef.js", null, await robots("/main.0123456789abcdef.js")); diff --git a/cloudflare/mempool-site/tests/sitemap.py b/cloudflare/mempool-site/tests/sitemap.py index 3013292..9529a85 100644 --- a/cloudflare/mempool-site/tests/sitemap.py +++ b/cloudflare/mempool-site/tests/sitemap.py @@ -1,7 +1,8 @@ """Check the explorer's sitemap against its robots.txt. No network access, no build needed. Fails when the sitemap names a page twice, a page on another host or one robots.txt disallows, -or when robots.txt stops pointing crawlers at the sitemap. +when sitemap.xml stops being an index of sitemap-en.xml, or when robots.txt stops pointing +crawlers at sitemap-en.xml. """ import re import sys @@ -13,9 +14,12 @@ HERE = Path(__file__).resolve().parent.parent sys.path.insert(0, str(HERE)) from postprocess import SITE -from sitemap import sitemap_xml +from sitemap import PAGES, index_xml, sitemap_xml ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} +index = ET.fromstring(index_xml()) +assert index.tag == "{%s}sitemapindex" % ns["s"], index.tag +assert [e.text for e in index.findall("s:sitemap/s:loc", ns)] == [f"{SITE}/{PAGES}"], "sitemap.xml must index sitemap-en.xml" root = ET.fromstring(sitemap_xml(0)) assert root.tag == "{%s}urlset" % ns["s"], root.tag urls = root.findall("s:url", ns) @@ -25,13 +29,13 @@ for u, loc in zip(urls, locs): assert loc.startswith(SITE + "/"), loc assert u.find("s:lastmod", ns).text == "1970-01-01T00:00:00+00:00", loc - assert 0 <= float(u.find("s:priority", ns).text) <= 1, loc + assert u.find("s:priority", ns) is None, f"{loc}: is ignored by Google, leave it out" robots = re.search(r"const ROBOTS = `([^`]*)`", (HERE / "_worker.js").read_text(encoding="utf-8")).group(1) -assert f"Sitemap: {SITE}/sitemap.xml" in robots.splitlines(), "robots.txt does not name the sitemap" +assert f"Sitemap: {SITE}/{PAGES}" in robots.splitlines(), f"robots.txt does not name {PAGES}" # robots.txt matching: a rule is a path prefix, `*` matches any run of characters. blocked = [re.compile(re.escape(r[len("Disallow: "):]).replace(r"\*", ".*")) for r in robots.splitlines() if r.startswith("Disallow: ")] for loc in locs: path = urlsplit(loc).path assert not any(b.match(path) for b in blocked), f"{path} is in the sitemap but robots.txt disallows it" -print(f"PASS: {len(locs)} explorer sitemap URLs, none disallowed by robots.txt, which names the sitemap") +print(f"PASS: {len(locs)} explorer sitemap URLs in {PAGES}, none disallowed by robots.txt, which names it; sitemap.xml indexes it") diff --git a/cloudflare/verify-mempool.py b/cloudflare/verify-mempool.py index 070fdf6..310cb6e 100644 --- a/cloudflare/verify-mempool.py +++ b/cloudflare/verify-mempool.py @@ -82,24 +82,26 @@ def same(path, headers=None, label=""): if not (st == 200 and body.strip() and "text/plain" in (hh.get("Content-Type") or "")): bad += 1 print(f"DIFF {p}: {st}/{len(body)} {hh.get('Content-Type')}") -# The worker answers robots.txt and the build writes sitemap.xml; the node's nginx serves neither, -# so both are checked on the candidate alone, along with every page the sitemap lists. +# The worker answers robots.txt and the build writes the sitemaps; the node's nginx serves none of +# them, so they are checked on the candidate alone, along with every page the sitemap lists. import sitemap from xml.etree import ElementTree as ET st, body, hh = get(cand, "/robots.txt") n += 1 -if not (st == 200 and f"Sitemap: {postprocess.SITE}/sitemap.xml" in body.decode().splitlines()): +if not (st == 200 and f"Sitemap: {postprocess.SITE}/{sitemap.PAGES}" in body.decode().splitlines()): bad += 1 - print(f"DIFF /robots.txt: {st}, does not name the sitemap") -st, body, hh = get(cand, "/sitemap.xml") -n += 1 -try: - locs = [e.text for e in ET.fromstring(body).iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc")] -except ET.ParseError: - locs = None -if not (st == 200 and "xml" in (hh.get("Content-Type") or "") and locs == [postprocess.SITE + p for p in sitemap.indexable()]): - bad += 1 - print(f"DIFF /sitemap.xml: {st} {hh.get('Content-Type')}, {'unparseable' if locs is None else f'{len(locs)} URLs'}") + print(f"DIFF /robots.txt: {st}, does not name {sitemap.PAGES}") +for path, want in (("/sitemap.xml", [f"{postprocess.SITE}/{sitemap.PAGES}"]), + ("/" + sitemap.PAGES, [postprocess.SITE + p for p in sitemap.indexable()])): + st, body, hh = get(cand, path) + n += 1 + try: + locs = [e.text for e in ET.fromstring(body).iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc")] + except ET.ParseError: + locs = None + if not (st == 200 and "xml" in (hh.get("Content-Type") or "") and locs == want): + bad += 1 + print(f"DIFF {path}: {st} {hh.get('Content-Type')}, {'unparseable' if locs is None else f'{len(locs)} URLs'}") # Sitemap pages are indexable; everything else answers noindex (INDEXING.md). for p in sitemap.indexable(): st, _, hh = get(cand, p) @@ -107,7 +109,7 @@ def same(path, headers=None, label=""): if st != 200 or hh.get("X-Robots-Tag"): bad += 1 print(f"DIFF {p}: in the sitemap, candidate answers {st} X-Robots-Tag {hh.get('X-Robots-Tag')}") -for p in ("/graphs", "/graphs/mining/pools", "/de/mining", "/lazarus/NOTICE", "/lazarus/LICENSE", "/3rdpartylicenses.txt"): +for p in ("/graphs", "/graphs/mining/pools", "/about", "/docs/faq", "/rbf", "/de/mining", "/lazarus/NOTICE", "/lazarus/LICENSE", "/3rdpartylicenses.txt"): st, _, hh = get(cand, p) n += 1 if st != 200 or "noindex" not in (hh.get("X-Robots-Tag") or ""):