diff --git a/cloudflare/mempool-site/INDEXING.md b/cloudflare/mempool-site/INDEXING.md new file mode 100644 index 0000000..4868447 --- /dev/null +++ b/cloudflare/mempool-site/INDEXING.md @@ -0,0 +1,50 @@ +# What search engines may index on mempool.lazarus-xbt.xyz + +There is one rule: **a page is in the sitemap, or it answers `X-Robots-Tag: noindex`.** + +The list is `INDEXABLE` in `_worker.js`. `sitemap.py` builds `sitemap-en.xml` from it, plus +`sitemap.xml`, an index that names `sitemap-en.xml` (the layout the pool site and the hub use). +robots.txt names `sitemap-en.xml`. The worker adds the noindex header to every other HTML answer. +To make a page indexable, add it there, run `python3 tests/sitemap.py` and `node tests/noindex.mjs`, +and deploy (`deploy.sh mempool`). + +## Indexable (in the sitemap), 5 pages + +`/`, `/mining`, `/mining/pool/lazarus`, `/blocks/1`, `/graphs/mempool` + +These are the pages about this chain and this pool. `/blocks/1` and `/graphs/mempool` are where the +app sends `/blocks` and `/graphs`, so the sitemap names the page a crawler lands on. The entries +carry `` (the build time) and no ``, which Google ignores. + +## Noindex, and why (19 Oct 2026; trimmed to pool pages the same day, on IT's advice) + +| What | Why it is left out | +|---|---| +| `/about`, `/docs/faq`, `/docs/api/rest`, `/docs/api/websocket`, `/terms-of-service`, `/privacy-policy`, `/trademark-policy`, `/tools/calculator` | Stock mempool.space pages, word for word (`/about` gains one attribution paragraph from the theme). A copy of another site's page earns no result of its own. | +| `/clock/...`, `/mempool-block/N`, `/rbf` | Live widgets: numbers that change by the second, no text to rank. | +| `/graphs`, `/docs`, `/docs/api`, `/blocks` | The app redirects them to `/graphs/mempool`, `/docs/faq`, `/docs/api/rest` and `/blocks/1`. | +| Chart pages: `/graphs/mining/pools`, `/graphs/mining/hashrate-difficulty`, `/graphs/mining/pools-dominance`, `/graphs/mining/block-fees`, `/graphs/mining/block-fees-subsidy`, `/graphs/mining/block-rewards`, `/graphs/mining/block-fee-rates`, `/graphs/mining/block-sizes-weights`, `/graphs/price` | Rendered in the browser, they are a chart plus axis labels and range buttons, with no prose (checked 19 Oct). Too thin for search for now. Add them to `INDEXABLE` if they get real text. | +| `/blocks/stale`, other `/blocks/N` | Variants of a listed page. | +| Other pools' pages, `/mining/pool/` (except `lazarus`) | Thin, and about other pools. | +| Language copies, `//...` (`/de/mining`, `/zh/`, …) | Duplicates: the same unprefixed URL already serves each visitor's language. | +| `/api`, `/api/` (the API docs shell) | Stock API docs, as above. | +| `/address/...`, `/tx/...`, `/block/...` | Auto-generated, one per address, transaction or block. `/address/` and `/tx/` are also disallowed in robots.txt, which stops Google from seeing the noindex. Lifting that Disallow is sequenced in GitHub issue #46 (Mike's call) and must come after this header is live. | +| `/lazarus/NOTICE`, `/lazarus/LICENSE`, `/lazarus/COPYING.md` (`_headers`), `/3rdpartylicenses.txt` (worker) | Licence files the footer links to, not pages. | + +Scripts, styles and images carry no robots header: crawlers fetch them to render the pages above. + +## robots.txt + +Unchanged apart from the `Sitemap:` line: `Allow: /`, then Disallow `/address/`, `/tx/`, `/api/` +and their language-prefixed forms (`/*/address/`, `/*/tx/`). Nothing in the sitemap may match a +Disallow; `tests/sitemap.py` checks it. + +## Temporary state (19 Oct 2026, until this is deployed) + +The explorer is not in the merge-to-main autodeploy. Until `deploy.sh mempool` ships this +directory, a separate bridge Worker, `lazarus-mempool-seo` on Mike's account, answers +`/robots.txt`, `/sitemap.xml` and `/sitemap-en.xml` and adds the noindex header to the pages under +its routes (`/graphs*`, `/docs*`, `/blocks*`, `/mining*`, `/tools*`, `/clock*`, `/mempool-block*`, +`/about*`, `/rbf*`, `/terms-of-service*`, `/privacy-policy*`, `/trademark-policy*`, the licence +files). It is generated from this directory's `_worker.js` and `sitemap.py`. Delete it right +**after** the Pages deploy, not before: `wrangler delete --name lazarus-mempool-seo`. diff --git a/cloudflare/mempool-site/_headers b/cloudflare/mempool-site/_headers index 847c06b..5a2a555 100644 --- a/cloudflare/mempool-site/_headers +++ b/cloudflare/mempool-site/_headers @@ -14,11 +14,20 @@ Cache-Control: public, max-age=300 /lazarus/* Cache-Control: public, max-age=300 -# The licence files have no extension, so Pages would not call them text. +/sitemap.xml + Cache-Control: public, max-age=0, s-maxage=3600 +/sitemap-en.xml + Cache-Control: public, max-age=0, s-maxage=3600 +# The licence files have no extension, so Pages would not call them text. They are files the +# footer links to, not pages, so they stay out of search results (INDEXING.md). /lazarus/NOTICE Content-Type: text/plain; charset=utf-8 + X-Robots-Tag: noindex /lazarus/LICENSE Content-Type: text/plain; charset=utf-8 + X-Robots-Tag: noindex +/lazarus/COPYING.md + X-Robots-Tag: noindex /resources/mining-pools/lazarus.svg ! Cache-Control Cache-Control: public, max-age=300 diff --git a/cloudflare/mempool-site/_worker.js b/cloudflare/mempool-site/_worker.js index e81630c..854e9fb 100644 --- a/cloudflare/mempool-site/_worker.js +++ b/cloudflare/mempool-site/_worker.js @@ -53,14 +53,32 @@ const API_TTL = [ // until a crawler rereads that, its address lookups are turned away here. const CRAWLER_RE = /bot|crawl|spider|slurp|bingpreview|facebookexternalhit|embedly|headless/i; const ROBOTS = `User-agent: * +Allow: / Disallow: /address/ Disallow: /tx/ Disallow: /api/ Disallow: /*/address/ Disallow: /*/tx/ -Allow: / + +Sitemap: https://mempool.lazarus-xbt.xyz/sitemap-en.xml `; +// The pages search engines may index: the ones about this chain and this pool, not the stock +// mempool.space pages or the live widgets. This is the one list: sitemap.py builds sitemap-en.xml +// from it (so it stays valid JSON: double quotes, no trailing comma), and every other page the app +// serves answers with X-Robots-Tag: noindex. INDEXING.md says what is left out and why. +const INDEXABLE = [ + "/", + "/mining", + "/mining/pool/lazarus", + "/blocks/1", + "/graphs/mempool" +]; + +function noindexUnlisted(headers, path) { + if (!INDEXABLE.includes(path)) headers.set("X-Robots-Tag", "noindex"); +} + // No Content-Security-Policy on purpose: the Angular app uses inline styles and scripts, the // theme loads Google Fonts and calls the pool API cross-origin, and a policy tight enough to be // worth having would break all three. HSTS carries no includeSubDomains and no preload. @@ -100,6 +118,7 @@ async function shell(env, request, lang, status = 200) { const out = new Response(request.method === "HEAD" ? null : res.body, { status: res.status === 200 ? status : res.status, headers: res.headers }); out.headers.set("Cache-Control", LOCALE_CACHE); out.headers.set("Vary", "Accept-Encoding, Accept-Language, Cookie"); + noindexUnlisted(out.headers, new URL(request.url).pathname); secure(out.headers); return out; } @@ -239,6 +258,9 @@ export default { const out = new Response(hit.body, hit); out.headers.set("Cache-Control", HASHED_RE.test(path) ? HASHED_CACHE : LOCALE_CACHE); out.headers.set("Vary", "Accept-Encoding, Accept-Language, Cookie"); + // Text documents (3rdpartylicenses.txt) are files, not pages; scripts and styles are left + // alone so nothing a renderer needs carries a robots header. + if (/\.(txt|md)$/.test(path)) out.headers.set("X-Robots-Tag", "noindex"); secure(out.headers); return out; } diff --git a/cloudflare/mempool-site/build.sh b/cloudflare/mempool-site/build.sh index 379d03e..c8c34f8 100755 --- a/cloudflare/mempool-site/build.sh +++ b/cloudflare/mempool-site/build.sh @@ -56,18 +56,20 @@ for f in favicon.ico favicon-16x16.png favicon-32x32.png apple-touch-icon.png; d done python3 "$HERE/postprocess.py" "$DIST" +python3 "$HERE/sitemap.py" "$DIST" for f in LICENSE NOTICE; do [ -s "$DIST/lazarus/$f" ] || { echo "lazarus/$f missing: the footer links to it" >&2; exit 1; } done cp "$HERE/_worker.js" "$HERE/_headers" "$DIST/" -# Static files skip the worker, except pool-tags.json, which it fetches live from the node. Pages -# gives exclude priority over include, so the theme files are excluded by name, not /lazarus/*. +# Static files skip the worker (both sitemaps included), except pool-tags.json, which it fetches +# live from the node. Pages gives exclude priority over include, so the theme files are excluded +# by name, not /lazarus/*. python3 - "$DIST" <<'PY' import json, pathlib, sys dist = pathlib.Path(sys.argv[1]) names = sorted(p.name for p in (dist / "lazarus").iterdir() if p.is_file() and p.name != "pool-tags.json") -routes = {"version": 1, "include": ["/*"], "exclude": ["/resources/*"] + ["/lazarus/" + n for n in names]} +routes = {"version": 1, "include": ["/*"], "exclude": ["/resources/*", "/sitemap.xml", "/sitemap-en.xml"] + ["/lazarus/" + n for n in names]} assert len(routes["include"]) + len(routes["exclude"]) <= 100, "Pages allows 100 route rules" (dist / "_routes.json").write_text(json.dumps(routes, indent=2) + "\n") PY diff --git a/cloudflare/mempool-site/sitemap.py b/cloudflare/mempool-site/sitemap.py new file mode 100644 index 0000000..55e6b0f --- /dev/null +++ b/cloudflare/mempool-site/sitemap.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Write the explorer's sitemaps: sitemap-en.xml, the pages worth a search result dated by this +build, and sitemap.xml, an index naming it (the same layout as the pool site and the hub). + +The list is INDEXABLE in _worker.js, the same list the worker uses to send every other page +X-Robots-Tag: noindex, so the sitemap and what may be indexed cannot drift apart. Adding a page is +one line there (INDEXING.md says what is left out and why). + +The explorer is one Angular app, so every page here is the same shell until the browser fills it +in, and the build is when any of them last changed: each is the build time. There is no +: Google ignores it. + + python3 sitemap.py dist # build.sh runs this after postprocess.py +""" +import json +import re +import sys +import time +from pathlib import Path +from xml.sax.saxutils import escape + +sys.dont_write_bytecode = True +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +from postprocess import SITE + +# The file that lists the pages, and the one robots.txt names. sitemap.xml only points at it. +PAGES = "sitemap-en.xml" + + +def indexable(): + src = (HERE / "_worker.js").read_text(encoding="utf-8") + return json.loads(re.search(r"^const INDEXABLE = (\[.*?\]);", src, re.S | re.M).group(1)) + + +def sitemap_xml(built=None): + """sitemap-en.xml: one per indexable page.""" + stamp = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime(built)) + urls = "".join( + f" \n {escape(SITE + path)}\n {stamp}\n \n" + for path in indexable() + ) + return ( + '\n' + '\n' + urls + "\n" + ) + + +def index_xml(): + """sitemap.xml: an index naming sitemap-en.xml, for crawlers that look at the usual place.""" + return ( + '\n' + '\n' + f" {escape(SITE)}/{PAGES}\n" + "\n" + ) + + +if __name__ == "__main__": + out = Path(sys.argv[1]) + (out / PAGES).write_text(sitemap_xml(), encoding="utf-8") + (out / "sitemap.xml").write_text(index_xml(), encoding="utf-8") diff --git a/cloudflare/mempool-site/tests/noindex.mjs b/cloudflare/mempool-site/tests/noindex.mjs new file mode 100644 index 0000000..c5aa7dc --- /dev/null +++ b/cloudflare/mempool-site/tests/noindex.mjs @@ -0,0 +1,44 @@ +// Check which explorer answers carry X-Robots-Tag: noindex, against a stand-in for the Pages +// assets. No network: the sitemap pages must stay indexable, every other page and text file must +// not. +import { readFileSync, writeFileSync, mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const src = new URL("../_worker.js", import.meta.url); +const dir = mkdtempSync(join(tmpdir(), "mempool-worker-")); +writeFileSync(join(dir, "worker.mjs"), readFileSync(src)); +const worker = (await import(join(dir, "worker.mjs"))).default; +const indexable = JSON.parse(/^const INDEXABLE = (\[[\s\S]*?\]);/m.exec(readFileSync(src, "utf8"))[1]); + +const FILES = { "/3rdpartylicenses.txt": "text/plain", "/main.0123456789abcdef.js": "text/javascript" }; +const env = { + ASSETS: { + async fetch(req) { + const path = new URL(req.url).pathname; + if (FILES[path]) return new Response("x", { headers: { "Content-Type": FILES[path] } }); + // Pages' SPA fallback: anything else is the app shell, which // also is. + return new Response("", { headers: { "Content-Type": "text/html" } }); + }, + }, +}; + +async function robots(path) { + const res = await worker.fetch(new Request("https://mempool.lazarus-xbt.xyz" + path), env, {}); + return res.headers.get("X-Robots-Tag"); +} + +let bad = 0; +const expect = (path, want, got) => { + if (got !== want) { + bad++; + console.error(`${path}: X-Robots-Tag ${got}, want ${want}`); + } +}; +for (const p of indexable) expect(p, null, await robots(p)); +for (const p of ["/graphs", "/docs", "/blocks", "/about", "/docs/faq", "/terms-of-service", "/tools/calculator", "/clock/mempool/0", "/rbf", "/graphs/mining/pools", "/mining/pool/convoy", "/de/mining", "/zh/", "/block/abc", "/address/bc1qtest", "/tx/abc", "/api", "/3rdpartylicenses.txt"]) { + expect(p, "noindex", await robots(p)); +} +expect("/main.0123456789abcdef.js", null, await robots("/main.0123456789abcdef.js")); +if (bad) process.exit(1); +console.log(`PASS: ${indexable.length} sitemap pages indexable; other pages, language copies and text files noindex`); diff --git a/cloudflare/mempool-site/tests/sitemap.py b/cloudflare/mempool-site/tests/sitemap.py new file mode 100644 index 0000000..9529a85 --- /dev/null +++ b/cloudflare/mempool-site/tests/sitemap.py @@ -0,0 +1,41 @@ +"""Check the explorer's sitemap against its robots.txt. No network access, no build needed. + +Fails when the sitemap names a page twice, a page on another host or one robots.txt disallows, +when sitemap.xml stops being an index of sitemap-en.xml, or when robots.txt stops pointing +crawlers at sitemap-en.xml. +""" +import re +import sys +from pathlib import Path +from urllib.parse import urlsplit +from xml.etree import ElementTree as ET + +sys.dont_write_bytecode = True +HERE = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(HERE)) +from postprocess import SITE +from sitemap import PAGES, index_xml, sitemap_xml + +ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"} +index = ET.fromstring(index_xml()) +assert index.tag == "{%s}sitemapindex" % ns["s"], index.tag +assert [e.text for e in index.findall("s:sitemap/s:loc", ns)] == [f"{SITE}/{PAGES}"], "sitemap.xml must index sitemap-en.xml" +root = ET.fromstring(sitemap_xml(0)) +assert root.tag == "{%s}urlset" % ns["s"], root.tag +urls = root.findall("s:url", ns) +locs = [u.find("s:loc", ns).text for u in urls] +assert locs and len(locs) == len(set(locs)), "duplicate sitemap URLs" +assert locs[0] == SITE + "/", locs[0] +for u, loc in zip(urls, locs): + assert loc.startswith(SITE + "/"), loc + assert u.find("s:lastmod", ns).text == "1970-01-01T00:00:00+00:00", loc + assert u.find("s:priority", ns) is None, f"{loc}: is ignored by Google, leave it out" + +robots = re.search(r"const ROBOTS = `([^`]*)`", (HERE / "_worker.js").read_text(encoding="utf-8")).group(1) +assert f"Sitemap: {SITE}/{PAGES}" in robots.splitlines(), f"robots.txt does not name {PAGES}" +# robots.txt matching: a rule is a path prefix, `*` matches any run of characters. +blocked = [re.compile(re.escape(r[len("Disallow: "):]).replace(r"\*", ".*")) for r in robots.splitlines() if r.startswith("Disallow: ")] +for loc in locs: + path = urlsplit(loc).path + assert not any(b.match(path) for b in blocked), f"{path} is in the sitemap but robots.txt disallows it" +print(f"PASS: {len(locs)} explorer sitemap URLs in {PAGES}, none disallowed by robots.txt, which names it; sitemap.xml indexes it") diff --git a/cloudflare/verify-mempool.py b/cloudflare/verify-mempool.py index 412856e..310cb6e 100644 --- a/cloudflare/verify-mempool.py +++ b/cloudflare/verify-mempool.py @@ -62,7 +62,7 @@ def same(path, headers=None, label=""): same("/145.402fa264d97bba59.js", hdr, label) for p in ("/de/", "/de/mining", "/zh/block/abc", "/pt-BR/", "/resources/config.js", "/resources/customize.js", "/resources/mining-pools/lazarus.svg", "/resources/mining-pools/default.svg", "/resources/favicons/favicon.ico", "/lazarus/chi-rho.svg", - "/resources/pools-v2.json", "/robots.txt", "/resources/nope.png", "/resources/favicons/site.webmanifest"): + "/resources/pools-v2.json", "/resources/nope.png", "/resources/favicons/site.webmanifest"): same(p) # The footer links to the licence files (mempool is AGPL-3.0). The build ships them whether or # not the node's nginx serves them, so they are checked on the candidate alone. @@ -82,6 +82,39 @@ def same(path, headers=None, label=""): if not (st == 200 and body.strip() and "text/plain" in (hh.get("Content-Type") or "")): bad += 1 print(f"DIFF {p}: {st}/{len(body)} {hh.get('Content-Type')}") +# The worker answers robots.txt and the build writes the sitemaps; the node's nginx serves none of +# them, so they are checked on the candidate alone, along with every page the sitemap lists. +import sitemap +from xml.etree import ElementTree as ET +st, body, hh = get(cand, "/robots.txt") +n += 1 +if not (st == 200 and f"Sitemap: {postprocess.SITE}/{sitemap.PAGES}" in body.decode().splitlines()): + bad += 1 + print(f"DIFF /robots.txt: {st}, does not name {sitemap.PAGES}") +for path, want in (("/sitemap.xml", [f"{postprocess.SITE}/{sitemap.PAGES}"]), + ("/" + sitemap.PAGES, [postprocess.SITE + p for p in sitemap.indexable()])): + st, body, hh = get(cand, path) + n += 1 + try: + locs = [e.text for e in ET.fromstring(body).iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc")] + except ET.ParseError: + locs = None + if not (st == 200 and "xml" in (hh.get("Content-Type") or "") and locs == want): + bad += 1 + print(f"DIFF {path}: {st} {hh.get('Content-Type')}, {'unparseable' if locs is None else f'{len(locs)} URLs'}") +# Sitemap pages are indexable; everything else answers noindex (INDEXING.md). +for p in sitemap.indexable(): + st, _, hh = get(cand, p) + n += 1 + if st != 200 or hh.get("X-Robots-Tag"): + bad += 1 + print(f"DIFF {p}: in the sitemap, candidate answers {st} X-Robots-Tag {hh.get('X-Robots-Tag')}") +for p in ("/graphs", "/graphs/mining/pools", "/about", "/docs/faq", "/rbf", "/de/mining", "/lazarus/NOTICE", "/lazarus/LICENSE", "/3rdpartylicenses.txt"): + st, _, hh = get(cand, p) + n += 1 + if st != 200 or "noindex" not in (hh.get("X-Robots-Tag") or ""): + bad += 1 + print(f"DIFF {p}: not in the sitemap, should be noindex, got {st} X-Robots-Tag {hh.get('X-Robots-Tag')}") # The pie's gateway bands read pool-tags.json, which the worker fetches live from the node # (pools-sync rewrites it every 3 min). A build-time copy would freeze the bands at deploy. import json, time