From 2f72279c24e5891830407fd267a0e23822f031fc Mon Sep 17 00:00:00 2001 From: Brett Ridenour Date: Mon, 19 Oct 2026 02:05:16 -0500 Subject: [PATCH] Pool robots.txt opens /api/*; hub sitemap moves to /sitemap-en.xml Pool: the pages fill their figures, the hardware catalog and the blocks table from /api/*, and Google does not render a fetch to a blocked URL, so Disallow: /api/ left /hardware and /blocks empty for Googlebot. Only the per-address and per-block lookups (/api/miner/, /api/solo/, /api/found/) stay closed. The indexing check now fails if robots.txt blocks an endpoint the pages render from. Hub: Search Console never fetched https://lazarus-xbt.xyz/sitemap.xml ("Couldn't fetch") while the pool's new sitemap-en.xml / sitemap-zh.xml were read on submit. The URL list is now /sitemap-en.xml, robots.txt points at it, and /sitemap.xml is an index of that one file, as on the pool site. Co-Authored-By: Claude Opus 5.5 --- cloudflare/map-site/public/robots.txt | 2 +- cloudflare/map-site/public/sitemap-en.xml | 13 +++++++++++++ cloudflare/map-site/public/sitemap.xml | 15 +++------------ cloudflare/map-site/scripts/indexnow.mjs | 4 ++-- cloudflare/map-site/scripts/sitemap-lastmod.mjs | 12 ++++++------ cloudflare/map-site/scripts/test-indexing.mjs | 5 ++++- cloudflare/pool-site/tests/indexing.py | 7 ++++++- pool/server.py | 11 ++++++----- 8 files changed, 41 insertions(+), 28 deletions(-) create mode 100644 cloudflare/map-site/public/sitemap-en.xml diff --git a/cloudflare/map-site/public/robots.txt b/cloudflare/map-site/public/robots.txt index 9d2fd1e..986d0b7 100644 --- a/cloudflare/map-site/public/robots.txt +++ b/cloudflare/map-site/public/robots.txt @@ -2,4 +2,4 @@ User-agent: * Allow: / Disallow: /api/ -Sitemap: https://lazarus-xbt.xyz/sitemap.xml +Sitemap: https://lazarus-xbt.xyz/sitemap-en.xml diff --git a/cloudflare/map-site/public/sitemap-en.xml b/cloudflare/map-site/public/sitemap-en.xml new file mode 100644 index 0000000..3ccb0c6 --- /dev/null +++ b/cloudflare/map-site/public/sitemap-en.xml @@ -0,0 +1,13 @@ + + + https://lazarus-xbt.xyz/2026-10-121.0 + https://lazarus-xbt.xyz/learn/2026-10-120.9 + https://lazarus-xbt.xyz/ecosystem/2026-10-120.8 + https://lazarus-xbt.xyz/key/2026-10-120.8 + https://lazarus-xbt.xyz/faq/2026-10-120.7 + https://lazarus-xbt.xyz/map/2026-10-060.6 + https://lazarus-xbt.xyz/naughtylist/2026-10-120.6 + https://lazarus-xbt.xyz/map/table/2026-10-110.4 + https://lazarus-xbt.xyz/stream/2026-10-180.8 + https://lazarus-xbt.xyz/blake2b/2026-10-120.8 + diff --git a/cloudflare/map-site/public/sitemap.xml b/cloudflare/map-site/public/sitemap.xml index 3ccb0c6..644083c 100644 --- a/cloudflare/map-site/public/sitemap.xml +++ b/cloudflare/map-site/public/sitemap.xml @@ -1,13 +1,4 @@ - - https://lazarus-xbt.xyz/2026-10-121.0 - https://lazarus-xbt.xyz/learn/2026-10-120.9 - https://lazarus-xbt.xyz/ecosystem/2026-10-120.8 - https://lazarus-xbt.xyz/key/2026-10-120.8 - https://lazarus-xbt.xyz/faq/2026-10-120.7 - https://lazarus-xbt.xyz/map/2026-10-060.6 - https://lazarus-xbt.xyz/naughtylist/2026-10-120.6 - https://lazarus-xbt.xyz/map/table/2026-10-110.4 - https://lazarus-xbt.xyz/stream/2026-10-180.8 - https://lazarus-xbt.xyz/blake2b/2026-10-120.8 - + + https://lazarus-xbt.xyz/sitemap-en.xml + diff --git a/cloudflare/map-site/scripts/indexnow.mjs b/cloudflare/map-site/scripts/indexnow.mjs index 1749218..457dce9 100644 --- a/cloudflare/map-site/scripts/indexnow.mjs +++ b/cloudflare/map-site/scripts/indexnow.mjs @@ -3,7 +3,7 @@ // Google does not use IndexNow; it reads the sitemap. // // node scripts/indexnow.mjs dry run: print what would be sent -// node scripts/indexnow.mjs --live submit every URL in public/sitemap.xml +// node scripts/indexnow.mjs --live submit every URL in public/sitemap-en.xml // node scripts/indexnow.mjs --live /blake2b/ /learn/ submit only these paths // // The key is the public/.txt file served at the site root; it is public by design. @@ -21,7 +21,7 @@ const key = readFileSync(join(pub, keyFile), "utf8").trim(); const args = process.argv.slice(2); const live = args.includes("--live"); const paths = args.filter((a) => a.startsWith("/")); -const fromSitemap = [...readFileSync(join(pub, "sitemap.xml"), "utf8").matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1]); +const fromSitemap = [...readFileSync(join(pub, "sitemap-en.xml"), "utf8").matchAll(/([^<]+)<\/loc>/g)].map((m) => m[1]); const urlList = paths.length ? paths.map((p) => `https://${HOST}${p}`) : fromSitemap; const body = { host: HOST, key, keyLocation: `https://${HOST}/${keyFile}`, urlList }; diff --git a/cloudflare/map-site/scripts/sitemap-lastmod.mjs b/cloudflare/map-site/scripts/sitemap-lastmod.mjs index eeb2477..c97a1aa 100644 --- a/cloudflare/map-site/scripts/sitemap-lastmod.mjs +++ b/cloudflare/map-site/scripts/sitemap-lastmod.mjs @@ -1,8 +1,8 @@ -// Set each in public/sitemap.xml to the date of the last commit that changed that +// Set each in public/sitemap-en.xml to the date of the last commit that changed that // page's HTML (or, for Worker-rendered pages, its source files). Run it in the commit that changes a page (npm run sitemap), so the sitemap never // says "today" for pages that did not change: search engines only use lastmod while it stays true. // -// node scripts/sitemap-lastmod.mjs rewrite public/sitemap.xml +// node scripts/sitemap-lastmod.mjs rewrite public/sitemap-en.xml // node scripts/sitemap-lastmod.mjs --check exit 1 and list the URLs whose date is stale import { execFileSync } from "node:child_process"; import { readFileSync, writeFileSync } from "node:fs"; @@ -10,7 +10,7 @@ import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; const here = dirname(dirname(fileURLToPath(import.meta.url))); -const sitemapPath = join(here, "public", "sitemap.xml"); +const sitemapPath = join(here, "public", "sitemap-en.xml"); const check = process.argv.includes("--check"); // Pages the Worker renders have no HTML file; their words live in these sources. @@ -46,11 +46,11 @@ const out = xml.replace(/([^<]+)<\/loc>([^<]*)<\/lastmod>/g, if (check) { if (stale.length) { - console.error(`sitemap.xml lastmod is out of date (run: npm run sitemap):\n ${stale.join("\n ")}`); + console.error(`sitemap-en.xml lastmod is out of date (run: npm run sitemap):\n ${stale.join("\n ")}`); process.exit(1); } - console.log("sitemap.xml lastmod matches git"); + console.log("sitemap-en.xml lastmod matches git"); } else { writeFileSync(sitemapPath, out); - console.log(stale.length ? `updated:\n ${stale.join("\n ")}` : "sitemap.xml already current"); + console.log(stale.length ? `updated:\n ${stale.join("\n ")}` : "sitemap-en.xml already current"); } diff --git a/cloudflare/map-site/scripts/test-indexing.mjs b/cloudflare/map-site/scripts/test-indexing.mjs index 8e0a355..d9f4f7a 100644 --- a/cloudflare/map-site/scripts/test-indexing.mjs +++ b/cloudflare/map-site/scripts/test-indexing.mjs @@ -3,7 +3,10 @@ import assert from 'node:assert/strict'; import {readFileSync} from 'node:fs'; import worker from '../src/worker.js'; const publicDir = new URL('../public/', import.meta.url); -const xml = readFileSync(new URL('sitemap.xml', publicDir), 'utf8'); +const index = readFileSync(new URL('sitemap.xml', publicDir), 'utf8'); +assert.deepEqual([...index.matchAll(/([^<]+)<\/loc>/g)].map(m=>m[1]), ['https://lazarus-xbt.xyz/sitemap-en.xml']); +assert.ok(readFileSync(new URL('robots.txt',publicDir),'utf8').includes('Sitemap: https://lazarus-xbt.xyz/sitemap-en.xml\n')); +const xml = readFileSync(new URL('sitemap-en.xml', publicDir), 'utf8'); const urls = [...xml.matchAll(/([^<]+)<\/loc>/g)].map(m=>m[1]); assert.equal(urls.length, 10); assert.equal(new Set(urls).size, 10); diff --git a/cloudflare/pool-site/tests/indexing.py b/cloudflare/pool-site/tests/indexing.py index 6a1a24f..38c5abd 100644 --- a/cloudflare/pool-site/tests/indexing.py +++ b/cloudflare/pool-site/tests/indexing.py @@ -53,7 +53,12 @@ def handle_starttag(self, tag, attrs): count += 1 assert count == 30, count # 15 documents in two languages assert not (out/'static/index.html').exists(), 'raw template is publicly exported' -assert 'Disallow: /\n' not in (out/'robots.txt').read_text() +robots = (out/'robots.txt').read_text() +assert 'Disallow: /\n' not in robots +# The pages render from these; Google does not run a fetch to a blocked URL. +disallowed = [l.split(':', 1)[1].strip() for l in robots.splitlines() if l.startswith('Disallow:')] +for api in ['/api/pool', '/api/coinbaser', '/api/price', '/api/solo', '/api/miners', '/api/hardware', '/api/rentals', '/api/blocks', '/api/payouts', '/api/history']: + assert not any(api.startswith(d) for d in disallowed), (api, disallowed) for route in ['/static/', '/static/index.html', '/static/index']: assert f'{route} / 301' in (out/'_redirects').read_text() assert (out/'404.html').exists(), 'Pages would enable SPA fallback without 404.html' diff --git a/pool/server.py b/pool/server.py index 16d93ae..409fdcb 100644 --- a/pool/server.py +++ b/pool/server.py @@ -7184,11 +7184,12 @@ def robots_txt(): "User-agent: *\n" "Allow: /\n" "\n" - "# The API reference is a page a developer should be able to find; the JSON endpoints under\n" - "# it have nothing to read and would only burn crawl budget. Longest match wins, so /api\n" - "# stays allowed while everything below it is blocked.\n" - "Allow: /api\n" - "Disallow: /api/\n" + "# The pages fill their figures, the hardware catalog and the blocks table from /api/*, and\n" + "# Google does not render a fetch to a blocked URL, so those endpoints stay open. Only the\n" + "# lookups are closed: one URL per address or block, without end, and each slow to build.\n" + "Disallow: /api/miner/\n" + "Disallow: /api/solo/\n" + "Disallow: /api/found/\n" "\n" f"Sitemap: {_PUBLIC_SITE}/sitemap.xml\n" )